Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Headlines feed. ETF Feed.

Browse Source
This commit is contained in:
Sean Kessler
2024-03-14 20:20:48 -04:00
parent 24d9a26880
commit e39e7ca840
4 changed files with 104 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
MarketDataLib/Helper/HeadlinesMarketDataHelper.cs
View File

@@ -12,6 +12,7 @@ namespace MarketData.Helper
public class HeadlinesMarketDataHelper
{
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
private static int WAIT_BETWEEN_REQUESTS_MS = 500; // wait 500 ms between requests
private List<String> symbols;
private int currentIndex = 0;
@@ -69,6 +70,7 @@ namespace MarketData.Helper
{
ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;}
}
MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
WaitHandle.WaitAll(resetEvents);

101
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1407,7 +1407,6 @@ namespace MarketData.Helper
ETFHoldings etfHoldings = new ETFHoldings();
HttpNetResponse httpNetResponse=null;
DateTime modified=DateTime.Now;
int TIMEOUT_MS=1000*30;
try
{
@@ -1419,7 +1418,8 @@ namespace MarketData.Helper
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS , webProxy);
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS,webProxy);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1429,8 +1429,13 @@ namespace MarketData.Helper
List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
if(null == sections || 1!=sections.Count)
{
MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unexpected items in section");
return null;
etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString);
if(null==etfHoldings)
{
MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response.");
return null;
}
return etfHoldings;
}
String marker="<table class=\"W(100%) M(0) BdB Bdc($seperatorColor)";
if(!sections[0].StartsWith(marker))
@@ -1473,6 +1478,55 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString)
{
ETFHoldings etfHoldings = new ETFHoldings();
try
{
DateTime modified = DateTime.Now;
int groupBy=3;
List<String> sections = Sections.GetAllItemsInSections(responseString,"section");
if(null==sections || 0==sections.Count)return null;
String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault();
if(String.IsNullOrEmpty(sectionItem))return null;
List<String> spans =Sections.GetAllItemsInSections(sectionItem,"span");
if(null==spans || 0==spans.Count || spans.Count<groupBy)return null;
for(int index=0;index<spans.Count;index+=groupBy)
{
String strElement = spans[index];
if(0==index)
{
String symbolNameHeading=Utility.BetweenString(spans[index],">","<");
String companyNameHeading=Utility.BetweenString(spans[index+1],">","<");
String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<");
if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets"))
{
MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading.");
return null;
}
}
else
{
ETFHolding etfHolding = new ETFHolding();
etfHolding.ETFSymbol = etfSymbol;
etfHolding.HoldingSymbolShareClass = null;
etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<");
etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<");
if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue;
etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<"));
etfHolding.Modified = modified;
etfHoldings.Add(etfHolding);
}
}
return 0==etfHoldings.Count?null:etfHoldings;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString()));
return null;
}
}
// ******************************************************************************************************************************************************************************
// **************************************************************************C O M P A N Y P R O F I L E **********************************************************************
// ***************************************************************** P R O F I L E : M O R N I N G S T A R *******************************************************************
@@ -1664,7 +1718,13 @@ namespace MarketData.Helper
// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S - M A R K E T W A T C H *************************************************************
// ***************************************************************************************************************************************************************************
public static Headlines GetCompanyHeadlinesMarketWatch(String symbol)
{
return null;
}
// turning this off until MarketWatch quote resets 03/14/2023. we'll try it again in some hours.
public static Headlines GetCompanyHeadlinesMarketWatch_Org(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -2034,6 +2094,11 @@ namespace MarketData.Helper
// ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
// ****************************************************************************************************************************************************************************
// If this breaks then it may be necessary to run the developer tools and point to the operatingPerformance page on MorningStar website.
// Set a filter in the network tab in the developer tools to "operatingPerformance". You should see the sal api information in the filter list.
// Match the salVersion to the latest sal version in the actual request.
// Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there.
// Not sure how long the RequestId lives for. I updated it on 03/12/2024
public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
{
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
@@ -2041,9 +2106,10 @@ namespace MarketData.Helper
String nasdaq = "xnas";
String nyse = "xnyse";
String nys="xnys";
String salVersion="3.79.0";
String salVersion="4.30.0";
int TIMEOUT_BETWEEN_REQUESTS_MS=500;
String exchange=nasdaq;
// String url="https://www.morningstar.com/stocks/";
String url="https://www.morningstar.com/api/v2/stocks/";
try
@@ -2081,7 +2147,7 @@ namespace MarketData.Helper
}
}
}
// This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier.
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
if(null==securityId)
{
@@ -2094,10 +2160,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString);
@@ -2108,10 +2175,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
@@ -2123,10 +2191,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
@@ -2137,10 +2206,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
@@ -2150,10 +2220,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
@@ -2164,10 +2235,11 @@ namespace MarketData.Helper
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
strRequest=sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
return null;
}
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
@@ -4188,7 +4260,8 @@ namespace MarketData.Helper
sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower());
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz");
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
if(!httpNetResponse.Success)
{