From 0fb1d194810302cc79e51c4c9644c2300458e49b Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 3 Nov 2025 21:13:14 -0500 Subject: [PATCH] Fix Morningstar feeds --- .../MarketDataLib/Helper/MarketDataHelper.cs | 31 ++- .../MarketDataUnitTestClass.cs | 185 +++++++++--------- 2 files changed, 113 insertions(+), 103 deletions(-) diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 4efd833..42c6fc7 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -52,7 +52,11 @@ namespace MarketData.Helper { public static int DEFAULT_TIMEOUT_MS=30000; public static String SEC_BASE_URL="https://www.sec.gov"; + public static String SAL_VERSION="4.71.0"; // This is for morningstar and may have to be changed periodically until I find a way to get automatically + private MarketDataHelper() + { + } public static bool GetWorldTime() { @@ -2127,8 +2131,7 @@ namespace MarketData.Helper String nasdaq = "xnas"; String nyse = "xnyse"; String nys="xnys"; - String salVersion="4.30.0"; - String salVerson45100="4.51.0"; + String strDefinition = default; String strColumnDefinition = default; int TIMEOUT_BETWEEN_REQUESTS_MS=500; @@ -2182,7 +2185,7 @@ namespace MarketData.Helper // PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2190,6 +2193,7 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } List> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString); @@ -2198,7 +2202,7 @@ namespace MarketData.Helper // KETSTATS - FINANCIALHEALTH sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2206,6 +2210,7 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } List> items=LocateJSONItems(httpNetResponse.ResponseString); @@ -2214,7 +2219,7 @@ namespace MarketData.Helper // NEWFINANCIALS - ANNUAL/SUMMARY sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2222,6 +2227,7 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } Dictionary dataSetsAnnuals=GetData(httpNetResponse.ResponseString); @@ -2229,7 +2235,7 @@ namespace MarketData.Helper // NEWFINANCIALS - INCOMESTATEMENT sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2237,6 +2243,7 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } Dictionary dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString); @@ -2244,7 +2251,7 @@ namespace MarketData.Helper // NEWFINANCIALS - CASHFLOW sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2252,6 +2259,7 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } Dictionary dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); @@ -2259,7 +2267,7 @@ namespace MarketData.Helper //NEWFINANCIALS - BALANCESHEET sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2267,11 +2275,13 @@ namespace MarketData.Helper if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return null; } Dictionary dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); +// ASSEMBLE DATA SETS // RETURN ON ASSETS strDefinition="roa"; strColumnDefinition="columnDefs"; @@ -3014,7 +3024,9 @@ namespace MarketData.Helper int waitBetweenAttemptsMS=500; StringBuilder sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=3.74.0"); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details"); + sb.Append("&version=").Append(SAL_VERSION); + String strRequest=sb.ToString(); try { @@ -3029,6 +3041,7 @@ namespace MarketData.Helper retry++; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest)); + MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****"); return httpNetResponse; } catch(Exception exception) diff --git a/MarketDataUnitTests/MarketDataUnitTestClass.cs b/MarketDataUnitTests/MarketDataUnitTestClass.cs index 7e37bf7..a58632b 100644 --- a/MarketDataUnitTests/MarketDataUnitTestClass.cs +++ b/MarketDataUnitTests/MarketDataUnitTestClass.cs @@ -1,5 +1,3 @@ -using System.Reflection; -using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.Extensions.Configuration; using MarketData.DataAccess; using MarketData.Generator.CMMomentum; @@ -12,8 +10,6 @@ using System.Diagnostics; using System.Net.Http.Headers; -// https://www.google.com/search?q=launch.json+for+debugging+c%23+unit+test&client=firefox-b-1-d&sca_esv=f0447ad3867c1f39&ei=OMQHaYOlB9GFwbkPgJbUiAs&ved=0ahUKEwiDocWOrNSQAxXRQjABHQALFbEQ4dUDCBE&uact=5&oq=launch.json+for+debugging+c%23+unit+test&gs_lp=Egxnd3Mtd2l6LXNlcnAiJmxhdW5jaC5qc29uIGZvciBkZWJ1Z2dpbmcgYyMgdW5pdCB0ZXN0MgUQABjvBTIIEAAYgAQYogQyCBAAGKIEGIkFMgUQABjvBUjvFFDdCFjtDXABeAGQAQCYAX6gAdICqgEDMC4zuAEDyAEA-AEBmAIEoALkAsICChAAGLADGNYEGEfCAgoQIRigARjDBBgKwgIIECEYoAEYwwSYAwCIBgGQBgiSBwMxLjOgB_AHsgcDMC4zuAffAsIHBTAuMi4yyAcJ&sclient=gws-wiz-serp - namespace MarketDataUnitTests; [TestClass] @@ -51,6 +47,18 @@ public class MarketDataUnitTestClass return true; } + /// + /// This is not working because the website has changed formats + /// + [TestMethod] + [Ignore] + public void AnalystRatingsMarketBeatRetrieval() + { + String symbol = "AAPL"; + AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol); + Assert.IsTrue(null != analystRatings && analystRatings.Count > 0); + } + [TestMethod] public void GetDailyPricesYahooRetrieval() { @@ -273,18 +281,6 @@ public class MarketDataUnitTestClass Assert.IsTrue(null != analystRatings && analystRatings.Count > 0); } - /// - /// This is not working because the website has changed formats - /// - [TestMethod] - [Ignore] - public void AnalystRatingsMarketBeatRetrieval() - { - String symbol = "AAPL"; - AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol); - Assert.IsTrue(null != analystRatings && analystRatings.Count > 0); - } - [TestMethod] public void SECCIKRetrieval() { @@ -310,7 +306,7 @@ public class MarketDataUnitTestClass DateTime analysisDate = dateGenerator.FindPrevBusinessDay(DateTime.Now); YieldCurve yieldCurve = MarketDataHelper.GetYieldCurve(analysisDate.Year); Assert.IsTrue(null != yieldCurve && yieldCurve.Count > 0); - Assert.IsTrue(yieldCurve[yieldCurve.Count - 1].Date.Month.Equals(analysisDate.Month)); + Assert.IsTrue(yieldCurve[yieldCurve.Count - 1].Date.Month>=(analysisDate.Month)); } [TestMethod] @@ -344,20 +340,6 @@ public class MarketDataUnitTestClass Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); } - // Test MarketWatch feed - // https://www.marketwatch.com/investing/stock/AAPL?mod=search_symbol - /// - /// MarketWatch/BigCharts is no longer working - /// - [TestMethod] - [Ignore] - public void HeadlinesMarketWatchRetrieval() - { - String symbol = "MIDD"; - Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol); - Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); - } - // Test NASDAQ Headlines feed [TestMethod] public void HeadlinesNASDAQRetrieval() @@ -367,30 +349,6 @@ public class MarketDataUnitTestClass Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); } - // Test SEEKING ALPHA Headlines feed - /// - /// SeekingAlpha headlines feed is not working - /// - [TestMethod] - [Ignore] - public void HeadlinesSeekingAlphaRetrieval() - { - String symbol = "GLD"; - Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol); - Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); - } - - /// - /// SeekingAlpha feed is not working - /// - [TestMethod] - [Ignore] - public void HeadlinesSeekingAlphaV3Retrieval() - { - String symbol = "MIDD"; - Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV3(symbol, true); - Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); - } [TestMethod] public void AnalystPriceTargetMarketBeatRetrieval() @@ -405,7 +363,8 @@ public class MarketDataUnitTestClass } /// - /// Need to figure out why this is not working + /// This is the MorningStar historical value retrieval + /// If this breaks then try updating the salVersion in the helper method /// [TestMethod] public void HistoricalRetrieval() @@ -427,6 +386,44 @@ public class MarketDataUnitTestClass Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.OperatingCashflow), "Missing OperatingCashflow"); } + /// + /// This is the Morningstar Cashflow statement retrieval + /// If this breaks then try updating the salVersion in the helper method + /// + [TestMethod] + public void CashflowStatementMorningStarRetrieval() + { + String[] symbols = { "AZEK", "CPRT", "DOCU", "ESTC", "HLNE" }; + + Dictionary> cashflowStatementsDict = new Dictionary>(); + + foreach (String symbol in symbols) + { + List cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol, CashflowStatement.PeriodType.Annual); + if (null == cashflowStatements) continue; + cashflowStatementsDict.Add(symbol, cashflowStatements); + } + + Assert.IsTrue(cashflowStatementsDict.Count != 0, "Error retrieving cashflow statements."); + + List keys = cashflowStatementsDict.Keys.ToList(); + + foreach (String key in keys) + { + List cashflowStatements = cashflowStatementsDict[key]; + CashflowStatement cashflowStatement = cashflowStatements[0]; + Assert.IsTrue(!double.IsNaN(cashflowStatement.DepreciationAndAmortization), $"DepreciationAndAmortization for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.DeferredIncomeTaxes), $"DeferredIncomeTaxes for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.StockBasedCompensation), $"StockBasedCompensation for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsReceivable), $"AccountsReceivable for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.Inventory), $"Inventory for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsPayable), $"AccountsPayable for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.AccruedLiabilities), $"AccruedLiabilities for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.OperatingCashflow), $"OperatingCashflow for {key}"); + Assert.IsTrue(!double.IsNaN(cashflowStatement.FreeCashflow), $"FreeCashflow for {key}"); + } + } + [TestMethod] public void IncomeStatementNASDAQRetrieval() { @@ -465,44 +462,6 @@ public class MarketDataUnitTestClass Assert.IsTrue(!double.IsNaN(balanceSheet.TotalLiabilities), "TotalLiabilities"); } - /// - /// Need to figure out why this is not working. - /// - [TestMethod] - public void CashflowStatementMorningStarRetrieval() - { - String[] symbols = { "AZEK", "CPRT", "DOCU", "ESTC", "HLNE" }; - - Dictionary> cashflowStatementsDict = new Dictionary>(); - - foreach (String symbol in symbols) - { - List cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol, CashflowStatement.PeriodType.Annual); - if (null == cashflowStatements) continue; - cashflowStatementsDict.Add(symbol, cashflowStatements); - } - - Assert.IsTrue(cashflowStatementsDict.Count != 0, "Error retrieving cashflow statements."); - - List keys = cashflowStatementsDict.Keys.ToList(); - - foreach (String key in keys) - { - List cashflowStatements = cashflowStatementsDict[key]; - CashflowStatement cashflowStatement = cashflowStatements[0]; - Assert.IsTrue(!double.IsNaN(cashflowStatement.DepreciationAndAmortization), $"DepreciationAndAmortization for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.DeferredIncomeTaxes), $"DeferredIncomeTaxes for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.StockBasedCompensation), $"StockBasedCompensation for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsReceivable), $"AccountsReceivable for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.Inventory), $"Inventory for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsPayable), $"AccountsPayable for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.AccruedLiabilities), $"AccruedLiabilities for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.OperatingCashflow), $"OperatingCashflow for {key}"); - Assert.IsTrue(!double.IsNaN(cashflowStatement.FreeCashflow), $"FreeCashflow for {key}"); - } - } - - [TestMethod] public void CurrencyConversionXERetrieval() { @@ -553,5 +512,43 @@ public class MarketDataUnitTestClass Assert.IsTrue(!double.IsNaN(fundamental.DebtToEquity), "DebtToEquity"); } + // Test SEEKING ALPHA Headlines feed + /// + /// SeekingAlpha headlines feed is not working + /// + [TestMethod] + [Ignore] + public void HeadlinesSeekingAlphaRetrieval() + { + String symbol = "GLD"; + Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol); + Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); + } + + /// + /// SeekingAlpha feed is not working + /// + [TestMethod] + [Ignore] + public void HeadlinesSeekingAlphaV3Retrieval() + { + String symbol = "MIDD"; + Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV3(symbol, true); + Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); + } + + // Test MarketWatch feed + // https://www.marketwatch.com/investing/stock/AAPL?mod=search_symbol + /// + /// MarketWatch/BigCharts is no longer working + /// + [TestMethod] + [Ignore] + public void HeadlinesMarketWatchRetrieval() + { + String symbol = "MIDD"; + Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol); + Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0); + } } \ No newline at end of file