Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Headlines feed. ETF Feed.

Browse Source
This commit is contained in:
Sean Kessler
2024-03-14 20:20:48 -04:00
parent 24d9a26880
commit e39e7ca840
4 changed files with 104 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MarketDataLib/Helper/HeadlinesMarketDataHelper.cs
View File

@@ -12,6 +12,7 @@ namespace MarketData.Helper
public class HeadlinesMarketDataHelper
{
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
private static int WAIT_BETWEEN_REQUESTS_MS = 500; // wait 500 ms between requests
private List<String> symbols;
private int currentIndex = 0;
@@ -69,6 +70,7 @@ namespace MarketData.Helper
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);

101
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1407,7 +1407,6 @@ namespace MarketData.Helper
ETFHoldings etfHoldings = new ETFHoldings();
HttpNetResponse httpNetResponse=null;
DateTime modified=DateTime.Now;
int TIMEOUT_MS=1000*30;
try
{
@@ -1419,7 +1418,8 @@ namespace MarketData.Helper
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS , webProxy);
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS,webProxy);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1429,8 +1429,13 @@ namespace MarketData.Helper
List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
if(null == sections || 1!=sections.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unexpected items in section");
return null;
etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString);
if(null==etfHoldings)
{
MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response.");
return null;
}
return etfHoldings;
}
String marker="<table class=\"W(100%) M(0) BdB Bdc($seperatorColor)";
if(!sections[0].StartsWith(marker))
@@ -1473,6 +1478,55 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString)
{
ETFHoldings etfHoldings = new ETFHoldings();
try
{
DateTime modified = DateTime.Now;
int groupBy=3;
List<String> sections = Sections.GetAllItemsInSections(responseString,"section");
if(null==sections || 0==sections.Count)return null;
String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault();
if(String.IsNullOrEmpty(sectionItem))return null;
List<String> spans =Sections.GetAllItemsInSections(sectionItem,"span");
if(null==spans || 0==spans.Count || spans.Count<groupBy)return null;
for(int index=0;index<spans.Count;index+=groupBy)
{
String strElement = spans[index];
if(0==index)
{
String symbolNameHeading=Utility.BetweenString(spans[index],">","<");
String companyNameHeading=Utility.BetweenString(spans[index+1],">","<");
String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<");
if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets"))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading.");
return null;
}
}
else
{
ETFHolding etfHolding = new ETFHolding();
etfHolding.ETFSymbol = etfSymbol;
etfHolding.HoldingSymbolShareClass = null;
etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<");
etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<");
if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue;
etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<"));
etfHolding.Modified = modified;
etfHoldings.Add(etfHolding);
}
}
return 0==etfHoldings.Count?null:etfHoldings;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString()));
return null;
}
}
// ******************************************************************************************************************************************************************************
// **************************************************************************C O M P A N Y P R O F I L E **********************************************************************
// ***************************************************************** P R O F I L E : M O R N I N G S T A R *******************************************************************
@@ -1664,7 +1718,13 @@ namespace MarketData.Helper
// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S - M A R K E T W A T C H *************************************************************
// ***************************************************************************************************************************************************************************
public static Headlines GetCompanyHeadlinesMarketWatch(String symbol)
{
return null;
}
// turning this off until MarketWatch quote resets 03/14/2023. we'll try it again in some hours.
public static Headlines GetCompanyHeadlinesMarketWatch_Org(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -2034,6 +2094,11 @@ namespace MarketData.Helper
// ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
// ****************************************************************************************************************************************************************************
// If this breaks then it may be necessary to run the developer tools and point to the operatingPerformance page on MorningStar website.
// Set a filter in the network tab in the developer tools to "operatingPerformance". You should see the sal api information in the filter list.
// Match the salVersion to the latest sal version in the actual request.
// Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there.
// Not sure how long the RequestId lives for. I updated it on 03/12/2024
public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
{
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
@@ -2041,9 +2106,10 @@ namespace MarketData.Helper
String nasdaq = "xnas";
String nyse = "xnyse";
String nys="xnys";
String salVersion="3.79.0";
String salVersion="4.30.0";
int TIMEOUT_BETWEEN_REQUESTS_MS=500;
String exchange=nasdaq;
// String url="https://www.morningstar.com/stocks/";
String url="https://www.morningstar.com/api/v2/stocks/";
try
@@ -2081,7 +2147,7 @@ namespace MarketData.Helper
}
}
}
// This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier.
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
if(null==securityId)
{
@@ -2094,10 +2160,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString);
@@ -2108,10 +2175,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
@@ -2123,10 +2191,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
@@ -2137,10 +2206,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
@@ -2150,10 +2220,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
@@ -2164,10 +2235,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
@@ -4188,7 +4260,8 @@ namespace MarketData.Helper
sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower());
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz");
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
if(!httpNetResponse.Success)
{

15
MarketDataLib/Integration/HttpNetRequest.cs
View File

@@ -1062,7 +1062,8 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV7(String strRequest)
public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
@@ -1076,6 +1077,7 @@ namespace MarketData.Integration
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.KeepAlive=true;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
@@ -1084,6 +1086,7 @@ namespace MarketData.Integration
webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream responseStream = webResponse.GetResponseStream();
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
@@ -1151,12 +1154,16 @@ namespace MarketData.Integration
webRequest.Headers.Add("Sec-Fetch-Mode:cors");
webRequest.Headers.Add("Sec-Fetch-Site:same-site");
webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU=");
webRequest.Headers.Add("X-API-RequestId: 4b919fb3-6dc8-750d-e50f-639f73030527");
// webRequest.Headers.Add("X-API-RequestId: 4b919fb3-6dc8-750d-e50f-639f73030527");
webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb");
webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T");
webRequest.Accept = "*/*";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0";
// webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
webRequest.Host="api-global.morningstar.com";
webRequest.Referer="https://www.morningstar.com";
// webRequest.Referer="https://www.morningstar.com";
webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance";
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.CookieContainer = new CookieContainer();

4
MarketDataLib/Utility/Profiler.cs
View File

@@ -38,6 +38,10 @@ namespace MarketData.Utils
{
return totalTime = GetTickCount() - totalTime;
}
public uint Split()
{
return GetTickCount() - elapsedTime;
}
}
}