Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated some feeds that were failing in the unit tests.

Browse Source
This commit is contained in:
Sean
2025-02-07 10:00:06 -05:00
parent 135ed0ca3f
commit 4100ba6db5
1 changed files with 59 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1556,14 +1556,16 @@ namespace MarketData.Helper
WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo");
sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile");
strRequest = sb.ToString();
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
// httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest);
if (!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
// Locate Industry
String strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry", 4);
@@ -1589,7 +1591,16 @@ namespace MarketData.Helper
}
}
if (null == strIndustry && null == strSector && null == strDescription) return null;
if(null!=strDescription && strDescription.Equals("Description Information Not Available"))
{
strDescription = null;
}
if(null == strIndustry || null == strSector || null == strDescription)
{
return null;
}
CompanyProfile companyProfile = new CompanyProfile();
companyProfile.Symbol = symbol;
companyProfile.Sector = strSector;
@@ -1599,7 +1610,8 @@ namespace MarketData.Helper
if (null != companyProfile.Industry) companyProfile.Industry = companyProfile.Industry.Trim();
if (null != companyProfile.Description) companyProfile.Description = Utility.RemoveHtml(companyProfile.Description.Trim());
return companyProfile;
} catch (Exception exception)
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, exception);
return null;
@@ -1654,8 +1666,6 @@ namespace MarketData.Helper
if(null==strIndustry && null==strSector)return null;
if(null!=strDescription && strDescription.Equals("Description Information Not Available"))return null;
strSector=strSector.Trim();
strIndustry=strIndustry.Trim();
CompanyProfile companyProfile = new CompanyProfile();
@@ -2011,7 +2021,8 @@ namespace MarketData.Helper
WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy);
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,300000,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -2266,6 +2277,7 @@ namespace MarketData.Helper
// Match the salVersion to the latest sal version in the actual request.
// Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there.
// Not sure how long the RequestId lives for. I updated it on 03/12/2024
// 02-07-2025 : Morningstar moved the ROA and the ROIC so I had to point to a new url to source that information
public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
{
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
@@ -2274,6 +2286,9 @@ namespace MarketData.Helper
String nyse = "xnyse";
String nys="xnys";
String salVersion="4.30.0";
String salVerson45100="4.51.0";
String strDefinition = default;
String strColumnDefinition = default;
int TIMEOUT_BETWEEN_REQUESTS_MS=500;
String exchange=nasdaq;
@@ -2323,8 +2338,9 @@ namespace MarketData.Helper
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));
// PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion);
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
@@ -2334,10 +2350,12 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString);
List<List<String>> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString);
Dictionary<String,MStarDataSet> dataSetsProfitabilityAndEfficiency=GetData("fiscalPeriodYear", profitabilityItems, httpNetResponse.ResponseString);
//Dictionary<String,MStarDataSet> dataSetsProfitabilityAndEfficiency=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// KETSTATS - FINANCIALHEALTH
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
strRequest=sb.ToString();
@@ -2353,7 +2371,7 @@ namespace MarketData.Helper
Dictionary<String,MStarDataSet> dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// NEWFINANCIALS - ANNUAL/SUMMARY
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
strRequest=sb.ToString();
@@ -2368,7 +2386,7 @@ namespace MarketData.Helper
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// NEWFINANCIALS - INCOMESTATEMENT
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
@@ -2383,6 +2401,7 @@ namespace MarketData.Helper
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// NEWFINANCIALS - CASHFLOW
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
@@ -2397,7 +2416,7 @@ namespace MarketData.Helper
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
//NEWFINANCIALS - BALANCESHEET
sb=new StringBuilder();
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
@@ -2412,13 +2431,13 @@ namespace MarketData.Helper
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
httpNetResponse.Dispose();
// check result sets
String strDefinition="Return on Assets %";
String strColumnDefinition="columnDefs";
if(dataSetsOperatingPerformance.ContainsKey(strDefinition))
// RETURN ON ASSETS
strDefinition="roa";
strColumnDefinition="columnDefs";
if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
{
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA;
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType);
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
{
values.Add(elementType, timeSeriesCollection);
@@ -2427,12 +2446,13 @@ namespace MarketData.Helper
}
}
strDefinition="Return on Invested Capital %";
// RETURN ON INVESTED CAPITAL
strDefinition="roic";
strColumnDefinition="columnDefs";
if(dataSetsOperatingPerformance.ContainsKey(strDefinition))
if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
{
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC;
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType);
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
{
values.Add(elementType, timeSeriesCollection);
@@ -2441,6 +2461,7 @@ namespace MarketData.Helper
}
}
// BOOK VALUE PER SHARE
strDefinition="bookValuePerShare";
strColumnDefinition="columnDefs";
if(dataSetsFinancialHealth.ContainsKey(strDefinition))
@@ -2455,6 +2476,7 @@ namespace MarketData.Helper
}
}
// INVENTORIES
strDefinition="Inventories";
strColumnDefinition="columnDefs";
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
@@ -2470,6 +2492,7 @@ namespace MarketData.Helper
}
}
// ACCOUNTS RECEIVABLES
strDefinition="Trade/Accounts Receivable, Current";
strColumnDefinition="columnDefs";
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
@@ -2485,6 +2508,7 @@ namespace MarketData.Helper
}
}
// COST OF GOOD AND SERVICES
strDefinition="Cost of Goods and Services";
strColumnDefinition="columnDefs";
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
@@ -2500,6 +2524,7 @@ namespace MarketData.Helper
}
}
// TOTAL OPERATING PROFIT/LOSS
strDefinition="Total Operating Profit/Loss";
strColumnDefinition="columnDefs";
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
@@ -2515,6 +2540,7 @@ namespace MarketData.Helper
}
}
// INTEREST EXPENSE
strDefinition="Interest Expense Net of Capitalized Interest";
strColumnDefinition="columnDefs";
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
@@ -2530,6 +2556,7 @@ namespace MarketData.Helper
}
}
// REPORTED EFFECTIVE TAX RATE
strDefinition="Reported Effective Tax Rate";
strColumnDefinition="columnDefs";
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
@@ -2545,6 +2572,7 @@ namespace MarketData.Helper
}
}
// TOTAL REVENUE
strDefinition="Total Revenue";
strColumnDefinition="columnDefs";
if(dataSetsAnnuals.ContainsKey(strDefinition))
@@ -2560,6 +2588,7 @@ namespace MarketData.Helper
}
}
// NET INCOME AVAILABLE TO COMMON STOCKHOLDERS
strDefinition="Net Income Available to Common Stockholders";
strColumnDefinition="columnDefs";
if(dataSetsAnnuals.ContainsKey(strDefinition))
@@ -2576,6 +2605,7 @@ namespace MarketData.Helper
}
}
// DILUTED EPS
strDefinition="Diluted EPS";
strColumnDefinition="columnDefs";
if(dataSetsAnnuals.ContainsKey(strDefinition))
@@ -2590,6 +2620,7 @@ namespace MarketData.Helper
}
}
// WORKING CAPITAL
strDefinition="Working Capital";
strColumnDefinition="columnDefs";
if(dataSetsAnnuals.ContainsKey(strDefinition))
@@ -2605,6 +2636,7 @@ namespace MarketData.Helper
}
}
// FREE CASHFLOW
strDefinition="Free Cash Flow";
strColumnDefinition="columnDefs";
if(dataSetsAnnuals.ContainsKey(strDefinition))
@@ -2620,6 +2652,7 @@ namespace MarketData.Helper
}
}
// OPERATING CASHFLOW
strDefinition="Cash Generated from Operating Activities";
strColumnDefinition="columnDefs";
if(dataSetsCashflowStatement.ContainsKey(strDefinition))
@@ -2960,6 +2993,10 @@ namespace MarketData.Helper
foreach(String strDate in strDates)
{
Dictionary<String,String> inner=dataDictionary[strDate];
if(!inner.ContainsKey(elementName))
{
continue;
}
elementValues.Add(inner[elementName]);
}
data.Add(elementName,new MStarDataSet(elementName,elementValues));
@@ -4999,6 +5036,7 @@ namespace MarketData.Helper
if(sectionText.Contains("regularMarketPreviousClose"))
{
itemValue = Sections.GetSections(sections[index])[0];
price.PrevClose=FeedParser.ParseValue(itemValue);
}
else if(sectionText.Contains("regularMarketOpen"))
{