Sean/ARM64
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix Morningstar feeds

Browse Source
This commit is contained in:
Sean
2025-11-03 21:13:14 -05:00
parent 58e8664176
commit 0fb1d19481
2 changed files with 113 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
MarketData/MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -52,7 +52,11 @@ namespace MarketData.Helper
{
public static int DEFAULT_TIMEOUT_MS=30000;
public static String SEC_BASE_URL="https://www.sec.gov";
public static String SAL_VERSION="4.71.0"; // This is for morningstar and may have to be changed periodically until I find a way to get automatically
private MarketDataHelper()
{
}
public static bool GetWorldTime()
{
@@ -2127,8 +2131,7 @@ namespace MarketData.Helper
String nasdaq = "xnas";
String nyse = "xnyse";
String nys="xnys";
String salVersion="4.30.0";
String salVerson45100="4.51.0";
String strDefinition = default;
String strColumnDefinition = default;
int TIMEOUT_BETWEEN_REQUESTS_MS=500;
@@ -2182,7 +2185,7 @@ namespace MarketData.Helper
// PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2190,6 +2193,7 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
List<List<String>> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString);
@@ -2198,7 +2202,7 @@ namespace MarketData.Helper
// KETSTATS - FINANCIALHEALTH
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2206,6 +2210,7 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
@@ -2214,7 +2219,7 @@ namespace MarketData.Helper
// NEWFINANCIALS - ANNUAL/SUMMARY
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2222,6 +2227,7 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
@@ -2229,7 +2235,7 @@ namespace MarketData.Helper
// NEWFINANCIALS - INCOMESTATEMENT
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2237,6 +2243,7 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
@@ -2244,7 +2251,7 @@ namespace MarketData.Helper
// NEWFINANCIALS - CASHFLOW
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2252,6 +2259,7 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
@@ -2259,7 +2267,7 @@ namespace MarketData.Helper
//NEWFINANCIALS - BALANCESHEET
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2267,11 +2275,13 @@ namespace MarketData.Helper
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return null;
}
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// ASSEMBLE DATA SETS
// RETURN ON ASSETS
strDefinition="roa";
strColumnDefinition="columnDefs";
@@ -3014,7 +3024,9 @@ namespace MarketData.Helper
int waitBetweenAttemptsMS=500;
StringBuilder sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=3.74.0");
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details");
sb.Append("&version=").Append(SAL_VERSION);
String strRequest=sb.ToString();
try
{
@@ -3029,6 +3041,7 @@ namespace MarketData.Helper
retry++;
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
return httpNetResponse;
}
catch(Exception exception)

185
MarketDataUnitTests/MarketDataUnitTestClass.cs
View File

@@ -1,5 +1,3 @@
using System.Reflection;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Microsoft.Extensions.Configuration;
using MarketData.DataAccess;
using MarketData.Generator.CMMomentum;
@@ -12,8 +10,6 @@ using System.Diagnostics;
using System.Net.Http.Headers;
// https://www.google.com/search?q=launch.json+for+debugging+c%23+unit+test&client=firefox-b-1-d&sca_esv=f0447ad3867c1f39&ei=OMQHaYOlB9GFwbkPgJbUiAs&ved=0ahUKEwiDocWOrNSQAxXRQjABHQALFbEQ4dUDCBE&uact=5&oq=launch.json+for+debugging+c%23+unit+test&gs_lp=Egxnd3Mtd2l6LXNlcnAiJmxhdW5jaC5qc29uIGZvciBkZWJ1Z2dpbmcgYyMgdW5pdCB0ZXN0MgUQABjvBTIIEAAYgAQYogQyCBAAGKIEGIkFMgUQABjvBUjvFFDdCFjtDXABeAGQAQCYAX6gAdICqgEDMC4zuAEDyAEA-AEBmAIEoALkAsICChAAGLADGNYEGEfCAgoQIRigARjDBBgKwgIIECEYoAEYwwSYAwCIBgGQBgiSBwMxLjOgB_AHsgcDMC4zuAffAsIHBTAuMi4yyAcJ&sclient=gws-wiz-serp
namespace MarketDataUnitTests;
[TestClass]
@@ -51,6 +47,18 @@ public class MarketDataUnitTestClass
return true;
}
/// <summary>
/// This is not working because the website has changed formats
/// </summary>
[TestMethod]
[Ignore]
public void AnalystRatingsMarketBeatRetrieval()
{
String symbol = "AAPL";
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
Assert.IsTrue(null != analystRatings && analystRatings.Count > 0);
}
[TestMethod]
public void GetDailyPricesYahooRetrieval()
{
@@ -273,18 +281,6 @@ public class MarketDataUnitTestClass
Assert.IsTrue(null != analystRatings && analystRatings.Count > 0);
}
/// <summary>
/// This is not working because the website has changed formats
/// </summary>
[TestMethod]
[Ignore]
public void AnalystRatingsMarketBeatRetrieval()
{
String symbol = "AAPL";
AnalystRatings analystRatings = MarketDataHelper.GetAnalystRatingsMarketBeat(symbol);
Assert.IsTrue(null != analystRatings && analystRatings.Count > 0);
}
[TestMethod]
public void SECCIKRetrieval()
{
@@ -310,7 +306,7 @@ public class MarketDataUnitTestClass
DateTime analysisDate = dateGenerator.FindPrevBusinessDay(DateTime.Now);
YieldCurve yieldCurve = MarketDataHelper.GetYieldCurve(analysisDate.Year);
Assert.IsTrue(null != yieldCurve && yieldCurve.Count > 0);
Assert.IsTrue(yieldCurve[yieldCurve.Count - 1].Date.Month.Equals(analysisDate.Month));
Assert.IsTrue(yieldCurve[yieldCurve.Count - 1].Date.Month>=(analysisDate.Month));
}
[TestMethod]
@@ -344,20 +340,6 @@ public class MarketDataUnitTestClass
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
// Test MarketWatch feed
// https://www.marketwatch.com/investing/stock/AAPL?mod=search_symbol
/// <summary>
/// MarketWatch/BigCharts is no longer working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesMarketWatchRetrieval()
{
String symbol = "MIDD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
// Test NASDAQ Headlines feed
[TestMethod]
public void HeadlinesNASDAQRetrieval()
@@ -367,30 +349,6 @@ public class MarketDataUnitTestClass
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
// Test SEEKING ALPHA Headlines feed
/// <summary>
/// SeekingAlpha headlines feed is not working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesSeekingAlphaRetrieval()
{
String symbol = "GLD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
/// <summary>
/// SeekingAlpha feed is not working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesSeekingAlphaV3Retrieval()
{
String symbol = "MIDD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV3(symbol, true);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
[TestMethod]
public void AnalystPriceTargetMarketBeatRetrieval()
@@ -405,7 +363,8 @@ public class MarketDataUnitTestClass
}
/// <summary>
/// Need to figure out why this is not working
/// This is the MorningStar historical value retrieval
/// If this breaks then try updating the salVersion in the helper method
/// </summary>
[TestMethod]
public void HistoricalRetrieval()
@@ -427,6 +386,44 @@ public class MarketDataUnitTestClass
Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.OperatingCashflow), "Missing OperatingCashflow");
}
/// <summary>
/// This is the Morningstar Cashflow statement retrieval
/// If this breaks then try updating the salVersion in the helper method
/// </summary>
[TestMethod]
public void CashflowStatementMorningStarRetrieval()
{
String[] symbols = { "AZEK", "CPRT", "DOCU", "ESTC", "HLNE" };
Dictionary<String, List<CashflowStatement>> cashflowStatementsDict = new Dictionary<String, List<CashflowStatement>>();
foreach (String symbol in symbols)
{
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol, CashflowStatement.PeriodType.Annual);
if (null == cashflowStatements) continue;
cashflowStatementsDict.Add(symbol, cashflowStatements);
}
Assert.IsTrue(cashflowStatementsDict.Count != 0, "Error retrieving cashflow statements.");
List<String> keys = cashflowStatementsDict.Keys.ToList();
foreach (String key in keys)
{
List<CashflowStatement> cashflowStatements = cashflowStatementsDict[key];
CashflowStatement cashflowStatement = cashflowStatements[0];
Assert.IsTrue(!double.IsNaN(cashflowStatement.DepreciationAndAmortization), $"DepreciationAndAmortization for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.DeferredIncomeTaxes), $"DeferredIncomeTaxes for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.StockBasedCompensation), $"StockBasedCompensation for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsReceivable), $"AccountsReceivable for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.Inventory), $"Inventory for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsPayable), $"AccountsPayable for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccruedLiabilities), $"AccruedLiabilities for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.OperatingCashflow), $"OperatingCashflow for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.FreeCashflow), $"FreeCashflow for {key}");
}
}
[TestMethod]
public void IncomeStatementNASDAQRetrieval()
{
@@ -465,44 +462,6 @@ public class MarketDataUnitTestClass
Assert.IsTrue(!double.IsNaN(balanceSheet.TotalLiabilities), "TotalLiabilities");
}
/// <summary>
/// Need to figure out why this is not working.
/// </summary>
[TestMethod]
public void CashflowStatementMorningStarRetrieval()
{
String[] symbols = { "AZEK", "CPRT", "DOCU", "ESTC", "HLNE" };
Dictionary<String, List<CashflowStatement>> cashflowStatementsDict = new Dictionary<String, List<CashflowStatement>>();
foreach (String symbol in symbols)
{
List<CashflowStatement> cashflowStatements = MarketDataHelper.GetCashflowStatement(symbol, CashflowStatement.PeriodType.Annual);
if (null == cashflowStatements) continue;
cashflowStatementsDict.Add(symbol, cashflowStatements);
}
Assert.IsTrue(cashflowStatementsDict.Count != 0, "Error retrieving cashflow statements.");
List<String> keys = cashflowStatementsDict.Keys.ToList();
foreach (String key in keys)
{
List<CashflowStatement> cashflowStatements = cashflowStatementsDict[key];
CashflowStatement cashflowStatement = cashflowStatements[0];
Assert.IsTrue(!double.IsNaN(cashflowStatement.DepreciationAndAmortization), $"DepreciationAndAmortization for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.DeferredIncomeTaxes), $"DeferredIncomeTaxes for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.StockBasedCompensation), $"StockBasedCompensation for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsReceivable), $"AccountsReceivable for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.Inventory), $"Inventory for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccountsPayable), $"AccountsPayable for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.AccruedLiabilities), $"AccruedLiabilities for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.OperatingCashflow), $"OperatingCashflow for {key}");
Assert.IsTrue(!double.IsNaN(cashflowStatement.FreeCashflow), $"FreeCashflow for {key}");
}
}
[TestMethod]
public void CurrencyConversionXERetrieval()
{
@@ -553,5 +512,43 @@ public class MarketDataUnitTestClass
Assert.IsTrue(!double.IsNaN(fundamental.DebtToEquity), "DebtToEquity");
}
// Test SEEKING ALPHA Headlines feed
/// <summary>
/// SeekingAlpha headlines feed is not working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesSeekingAlphaRetrieval()
{
String symbol = "GLD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
/// <summary>
/// SeekingAlpha feed is not working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesSeekingAlphaV3Retrieval()
{
String symbol = "MIDD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV3(symbol, true);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
// Test MarketWatch feed
// https://www.marketwatch.com/investing/stock/AAPL?mod=search_symbol
/// <summary>
/// MarketWatch/BigCharts is no longer working
/// </summary>
[TestMethod]
[Ignore]
public void HeadlinesMarketWatchRetrieval()
{
String symbol = "MIDD";
Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
Assert.IsTrue(null != companyHeadlines && companyHeadlines.Count > 0);
}
}