diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index 287dd3b..9a665f6 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1556,14 +1556,16 @@ namespace MarketData.Helper WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo"); sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile"); strRequest = sb.ToString(); - httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); + MDTrace.WriteLine(LogLevel.DEBUG,strRequest); + +// httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); // Locate Industry String strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry", 4); @@ -1589,7 +1591,16 @@ namespace MarketData.Helper } } - if (null == strIndustry && null == strSector && null == strDescription) return null; + if(null!=strDescription && strDescription.Equals("Description Information Not Available")) + { + strDescription = null; + } + + if(null == strIndustry || null == strSector || null == strDescription) + { + return null; + } + CompanyProfile companyProfile = new CompanyProfile(); companyProfile.Symbol = symbol; companyProfile.Sector = strSector; @@ -1599,7 +1610,8 @@ namespace MarketData.Helper if (null != companyProfile.Industry) companyProfile.Industry = companyProfile.Industry.Trim(); if (null != companyProfile.Description) companyProfile.Description = Utility.RemoveHtml(companyProfile.Description.Trim()); return companyProfile; - } catch (Exception exception) + } + catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, exception); return null; @@ -1654,8 +1666,6 @@ namespace MarketData.Helper if(null==strIndustry && null==strSector)return null; - if(null!=strDescription && strDescription.Equals("Description Information Not Available"))return null; - strSector=strSector.Trim(); strIndustry=strIndustry.Trim(); CompanyProfile companyProfile = new CompanyProfile(); @@ -2011,7 +2021,8 @@ namespace MarketData.Helper WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest)); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy); +// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,300000,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -2266,6 +2277,7 @@ namespace MarketData.Helper // Match the salVersion to the latest sal version in the actual request. // Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there. // Not sure how long the RequestId lives for. I updated it on 03/12/2024 +// 02-07-2025 : Morningstar moved the ROA and the ROIC so I had to point to a new url to source that information public static Dictionary GetHistoricalValues(String symbol) { Dictionary values = new Dictionary(); @@ -2274,6 +2286,9 @@ namespace MarketData.Helper String nyse = "xnyse"; String nys="xnys"; String salVersion="4.30.0"; + String salVerson45100="4.51.0"; + String strDefinition = default; + String strColumnDefinition = default; int TIMEOUT_BETWEEN_REQUESTS_MS=500; String exchange=nasdaq; @@ -2323,8 +2338,9 @@ namespace MarketData.Helper } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId)); +// PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC sb=new StringBuilder(); - sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion); + sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} @@ -2334,10 +2350,12 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } - Dictionary dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString); + List> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString); + Dictionary dataSetsProfitabilityAndEfficiency=GetData("fiscalPeriodYear", profitabilityItems, httpNetResponse.ResponseString); + //Dictionary dataSetsProfitabilityAndEfficiency=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); - +// KETSTATS - FINANCIALHEALTH sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion); strRequest=sb.ToString(); @@ -2353,7 +2371,7 @@ namespace MarketData.Helper Dictionary dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString); httpNetResponse.Dispose(); - +// NEWFINANCIALS - ANNUAL/SUMMARY sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion); strRequest=sb.ToString(); @@ -2368,7 +2386,7 @@ namespace MarketData.Helper Dictionary dataSetsAnnuals=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); - +// NEWFINANCIALS - INCOMESTATEMENT sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); @@ -2383,6 +2401,7 @@ namespace MarketData.Helper Dictionary dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); +// NEWFINANCIALS - CASHFLOW sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); @@ -2397,7 +2416,7 @@ namespace MarketData.Helper Dictionary dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); - +//NEWFINANCIALS - BALANCESHEET sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); @@ -2412,13 +2431,13 @@ namespace MarketData.Helper Dictionary dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); -// check result sets - String strDefinition="Return on Assets %"; - String strColumnDefinition="columnDefs"; - if(dataSetsOperatingPerformance.ContainsKey(strDefinition)) +// RETURN ON ASSETS + strDefinition="roa"; + strColumnDefinition="columnDefs"; + if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA; - TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType); + TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); @@ -2427,12 +2446,13 @@ namespace MarketData.Helper } } - strDefinition="Return on Invested Capital %"; +// RETURN ON INVESTED CAPITAL + strDefinition="roic"; strColumnDefinition="columnDefs"; - if(dataSetsOperatingPerformance.ContainsKey(strDefinition)) + if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC; - TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType); + TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); @@ -2441,6 +2461,7 @@ namespace MarketData.Helper } } +// BOOK VALUE PER SHARE strDefinition="bookValuePerShare"; strColumnDefinition="columnDefs"; if(dataSetsFinancialHealth.ContainsKey(strDefinition)) @@ -2455,6 +2476,7 @@ namespace MarketData.Helper } } +// INVENTORIES strDefinition="Inventories"; strColumnDefinition="columnDefs"; if(dataSetsBalanceSheet.ContainsKey(strDefinition)) @@ -2470,6 +2492,7 @@ namespace MarketData.Helper } } +// ACCOUNTS RECEIVABLES strDefinition="Trade/Accounts Receivable, Current"; strColumnDefinition="columnDefs"; if(dataSetsBalanceSheet.ContainsKey(strDefinition)) @@ -2485,6 +2508,7 @@ namespace MarketData.Helper } } +// COST OF GOOD AND SERVICES strDefinition="Cost of Goods and Services"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) @@ -2500,6 +2524,7 @@ namespace MarketData.Helper } } +// TOTAL OPERATING PROFIT/LOSS strDefinition="Total Operating Profit/Loss"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) @@ -2515,6 +2540,7 @@ namespace MarketData.Helper } } +// INTEREST EXPENSE strDefinition="Interest Expense Net of Capitalized Interest"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) @@ -2530,6 +2556,7 @@ namespace MarketData.Helper } } +// REPORTED EFFECTIVE TAX RATE strDefinition="Reported Effective Tax Rate"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) @@ -2545,6 +2572,7 @@ namespace MarketData.Helper } } +// TOTAL REVENUE strDefinition="Total Revenue"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) @@ -2560,6 +2588,7 @@ namespace MarketData.Helper } } +// NET INCOME AVAILABLE TO COMMON STOCKHOLDERS strDefinition="Net Income Available to Common Stockholders"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) @@ -2576,6 +2605,7 @@ namespace MarketData.Helper } } +// DILUTED EPS strDefinition="Diluted EPS"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) @@ -2590,6 +2620,7 @@ namespace MarketData.Helper } } +// WORKING CAPITAL strDefinition="Working Capital"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) @@ -2605,6 +2636,7 @@ namespace MarketData.Helper } } +// FREE CASHFLOW strDefinition="Free Cash Flow"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) @@ -2620,6 +2652,7 @@ namespace MarketData.Helper } } +// OPERATING CASHFLOW strDefinition="Cash Generated from Operating Activities"; strColumnDefinition="columnDefs"; if(dataSetsCashflowStatement.ContainsKey(strDefinition)) @@ -2960,6 +2993,10 @@ namespace MarketData.Helper foreach(String strDate in strDates) { Dictionary inner=dataDictionary[strDate]; + if(!inner.ContainsKey(elementName)) + { + continue; + } elementValues.Add(inner[elementName]); } data.Add(elementName,new MStarDataSet(elementName,elementValues)); @@ -4999,6 +5036,7 @@ namespace MarketData.Helper if(sectionText.Contains("regularMarketPreviousClose")) { itemValue = Sections.GetSections(sections[index])[0]; + price.PrevClose=FeedParser.ParseValue(itemValue); } else if(sectionText.Contains("regularMarketOpen")) {