From e39e7ca8407308904854136cb76b6d21c0b3f700 Mon Sep 17 00:00:00 2001 From: Sean Kessler Date: Thu, 14 Mar 2024 20:20:48 -0400 Subject: [PATCH] Headlines feed. ETF Feed. --- .../Helper/HeadlinesMarketDataHelper.cs | 2 + MarketDataLib/Helper/MarketDataHelper.cs | 101 +++++++++++++++--- MarketDataLib/Integration/HttpNetRequest.cs | 15 ++- MarketDataLib/Utility/Profiler.cs | 4 + 4 files changed, 104 insertions(+), 18 deletions(-) diff --git a/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs b/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs index 0327a0a..2162de8 100644 --- a/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs +++ b/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs @@ -12,6 +12,7 @@ namespace MarketData.Helper public class HeadlinesMarketDataHelper { private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads; + private static int WAIT_BETWEEN_REQUESTS_MS = 500; // wait 500 ms between requests private List symbols; private int currentIndex = 0; @@ -69,6 +70,7 @@ namespace MarketData.Helper { ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper); + try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;} } MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index 2215ce9..58d3d09 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1407,7 +1407,6 @@ namespace MarketData.Helper ETFHoldings etfHoldings = new ETFHoldings(); HttpNetResponse httpNetResponse=null; DateTime modified=DateTime.Now; - int TIMEOUT_MS=1000*30; try { @@ -1419,7 +1418,8 @@ namespace MarketData.Helper WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS , webProxy); +// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1429,8 +1429,13 @@ namespace MarketData.Helper List sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table"); if(null == sections || 1!=sections.Count) { - MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unexpected items in section"); - return null; + etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString); + if(null==etfHoldings) + { + MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response."); + return null; + } + return etfHoldings; } String marker=" sections = Sections.GetAllItemsInSections(responseString,"section"); + if(null==sections || 0==sections.Count)return null; + String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault(); + if(String.IsNullOrEmpty(sectionItem))return null; + List spans =Sections.GetAllItemsInSections(sectionItem,"span"); + if(null==spans || 0==spans.Count || spans.Count","<"); + String companyNameHeading=Utility.BetweenString(spans[index+1],">","<"); + String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<"); + if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets")) + { + MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading."); + return null; + } + } + else + { + ETFHolding etfHolding = new ETFHolding(); + etfHolding.ETFSymbol = etfSymbol; + etfHolding.HoldingSymbolShareClass = null; + etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<"); + etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<"); + if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue; + etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<")); + etfHolding.Modified = modified; + etfHoldings.Add(etfHolding); + } + } + return 0==etfHoldings.Count?null:etfHoldings; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString())); + return null; + } + } // ****************************************************************************************************************************************************************************** // **************************************************************************C O M P A N Y P R O F I L E ********************************************************************** // ***************************************************************** P R O F I L E : M O R N I N G S T A R ******************************************************************* @@ -1664,7 +1718,13 @@ namespace MarketData.Helper // *************************************************************************************************************************************************************************** //***************************************************************** H E A D L I N E S - M A R K E T W A T C H ************************************************************* // *************************************************************************************************************************************************************************** + public static Headlines GetCompanyHeadlinesMarketWatch(String symbol) + { + return null; + } + // turning this off until MarketWatch quote resets 03/14/2023. we'll try it again in some hours. + public static Headlines GetCompanyHeadlinesMarketWatch_Org(String symbol) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); @@ -2034,6 +2094,11 @@ namespace MarketData.Helper // **************************************************************************************************************************************************************************** // ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N ******************************************* // **************************************************************************************************************************************************************************** +// If this breaks then it may be necessary to run the developer tools and point to the operatingPerformance page on MorningStar website. +// Set a filter in the network tab in the developer tools to "operatingPerformance". You should see the sal api information in the filter list. +// Match the salVersion to the latest sal version in the actual request. +// Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there. +// Not sure how long the RequestId lives for. I updated it on 03/12/2024 public static Dictionary GetHistoricalValues(String symbol) { Dictionary values = new Dictionary(); @@ -2041,9 +2106,10 @@ namespace MarketData.Helper String nasdaq = "xnas"; String nyse = "xnyse"; String nys="xnys"; - String salVersion="3.79.0"; + String salVersion="4.30.0"; + int TIMEOUT_BETWEEN_REQUESTS_MS=500; + String exchange=nasdaq; -// String url="https://www.morningstar.com/stocks/"; String url="https://www.morningstar.com/api/v2/stocks/"; try @@ -2081,7 +2147,7 @@ namespace MarketData.Helper } } } - +// This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier. String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString); if(null==securityId) { @@ -2094,10 +2160,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString); @@ -2108,10 +2175,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } List> items=LocateJSONItems(httpNetResponse.ResponseString); @@ -2123,10 +2191,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsAnnuals=GetData(httpNetResponse.ResponseString); @@ -2137,10 +2206,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString); @@ -2150,10 +2220,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); @@ -2164,10 +2235,11 @@ namespace MarketData.Helper sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString); @@ -4188,7 +4260,8 @@ namespace MarketData.Helper sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower()); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest)); - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); + WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz"); + httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); if(!httpNetResponse.Success) { diff --git a/MarketDataLib/Integration/HttpNetRequest.cs b/MarketDataLib/Integration/HttpNetRequest.cs index c762f14..bfe4787 100644 --- a/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketDataLib/Integration/HttpNetRequest.cs @@ -1062,7 +1062,8 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]"); } } - public static HttpNetResponse GetRequestNoEncodingV7(String strRequest) + + public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null) { HttpWebResponse webResponse = null; try @@ -1076,6 +1077,7 @@ namespace MarketData.Integration ServicePointManager.Expect100Continue = true; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; webRequest.KeepAlive=true; webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); @@ -1084,6 +1086,7 @@ namespace MarketData.Integration webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0"; webRequest.KeepAlive = true; webRequest.CookieContainer = new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); webResponse = (HttpWebResponse)webRequest.GetResponse(); Stream responseStream = webResponse.GetResponseStream(); if (webResponse.ContentEncoding.ToLower().Contains("gzip")) @@ -1151,12 +1154,16 @@ namespace MarketData.Integration webRequest.Headers.Add("Sec-Fetch-Mode:cors"); webRequest.Headers.Add("Sec-Fetch-Site:same-site"); webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU="); - webRequest.Headers.Add("X-API-RequestId: 4b919fb3-6dc8-750d-e50f-639f73030527"); +// webRequest.Headers.Add("X-API-RequestId: 4b919fb3-6dc8-750d-e50f-639f73030527"); + webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb"); webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T"); webRequest.Accept = "*/*"; - webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0"; +// webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0"; + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; webRequest.Host="api-global.morningstar.com"; - webRequest.Referer="https://www.morningstar.com"; +// webRequest.Referer="https://www.morningstar.com"; + webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance"; + if(null!=webProxy)webRequest.Proxy=webProxy; webRequest.CookieContainer = new CookieContainer(); diff --git a/MarketDataLib/Utility/Profiler.cs b/MarketDataLib/Utility/Profiler.cs index 5f46a76..dedf07c 100644 --- a/MarketDataLib/Utility/Profiler.cs +++ b/MarketDataLib/Utility/Profiler.cs @@ -38,6 +38,10 @@ namespace MarketData.Utils { return totalTime = GetTickCount() - totalTime; } + public uint Split() + { + return GetTickCount() - elapsedTime; + } } }