using System; using System.IO; using System.Collections.Generic; using System.Text; using System.Linq; using System.Data; using System.Threading; using System.Net; using System.Reflection; using System.Xml.Linq; using Newtonsoft.Json.Linq; using HtmlAgilityPack; using MarketData.MarketDataModel; using MarketData.Utils; using MarketData.CSVHelper; using MarketData.DataAccess; using MarketData.Integration; using System.Globalization; using MarketDataLib.Utility; //Zacks Rank - Zacks //Splits - EODDATA //GDP Data - WorldBank //Dividend History - NASDAQ //Options - NASDAQ //Analyst Ratings - Briefing.com //SEC Filings - SEC.GOV (EDGAR) //Yield Curve - Treasury.Gov //ETF Holdings - Yahoo Finance //Insider Transactions - Insider Tracking.com //Company Profile - MorningStar + Reuters //Headlines - Seeking Alpha + NASDAQ //CIK Codes - SEC.GOV //Analyst Price Target - Yahoo Finance , Market Beat as backup //Historical - MorningStar //Income Statement - NASDAQ //Balance Sheet - NASDAQ //Statement of Cashflows - MorningStar //Fundamentals - 2 Sources....Yahoo Finance & FINVIZ // a) Key Statistics // b) Financials //Intraday Pricing - Yahoo Finance + BigCharts //Historical Pricing - BigCharts + Yahoo Finance //CompanyDescription - Reuters //Premarket - CNN namespace MarketData.Helper { public class MarketDataHelper { public static int DEFAULT_TIMEOUT_MS=30000; public static String SEC_BASE_URL="https://www.sec.gov"; public static bool GetWorldTime() { Profiler profiler = new Profiler(); HttpNetResponse httpNetResponse=null; try { StringBuilder sb = new StringBuilder(); sb.Append("https://www.timeanddate.com/worldclock/"); String strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime:{0}",strRequest)); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4A(strRequest, null, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return false; } if(String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest)); return false; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length)); return true; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return false; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetWorldTime: Done, took {0}(ms)", profiler.End())); } } // ****************************************************************************************************************************************************************************** // ************************************************************************************ P R E M A R K E T D A T A *********************************************************** // ****************************************************************************************************************************************************************************** public static PremarketElements GetPremarketData() { Profiler profiler = new Profiler(); HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; PremarketElements premarketElements=new PremarketElements(); DateTime timestamp=DateTime.Now; try { StringBuilder sb=new StringBuilder(); String strRequest=null; sb.Append("https://production.dataviz.cnn.io/markets/futures/summary"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest)); httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. Failed with {1}",strRequest,httpNetResponse.ErrorMessage)); return null; } List> items=LocateJSONItems(httpNetResponse.ResponseString); if(null==items || 0==items.Count) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. No Data",strRequest)); return premarketElements; } for(int index=0;index pair=items[index]; if(pair[1].Equals("S&P 500")) { PremarketElement premarketElement=new PremarketElement(); premarketElement.Market="S&P"; premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]); premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00; premarketElement.Timestamp=timestamp; premarketElements.Add(premarketElement); } else if(pair[1].Equals("NASDAQ")) { PremarketElement premarketElement=new PremarketElement(); premarketElement.Market="NASDAQ"; premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]); premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00; premarketElement.Timestamp=timestamp; premarketElements.Add(premarketElement); } else if(pair[1].Equals("DOW")) { PremarketElement premarketElement=new PremarketElement(); premarketElement.Market="DOW"; premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]); premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00; premarketElement.Timestamp=timestamp; premarketElements.Add(premarketElement); } } return premarketElements; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if(null!=memoryStream) memoryStream.Close(); if(null!=httpNetResponse) httpNetResponse.Dispose(); MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetPremarketData: Done, took {0}(ms)", profiler.End())); } } // ****************************************************************************************************************************************************************************** // ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S ******************************************* // ****************************************************************************************************************************************************************************** public static EarningsAnnouncements GetEarningsAnnouncements(String symbol) { Profiler profiler = new Profiler(); HttpNetResponse httpNetResponse=null; EarningsAnnouncements earningsAnnouncements=new EarningsAnnouncements(); MemoryStream memoryStream=null; try { StringBuilder sb=new StringBuilder(); sb.Append("https://www.zacks.com/stock/research/").Append(symbol).Append("/earnings-announcements"); String strRequest=sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } String responseString=httpNetResponse.ResponseString; // get next earnings date byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"key-expected-earnings-data-module\"]"); if (null != table && 0 != table.Count) { HtmlNodeCollection rows = table[0].SelectNodes(".//tr"); String[] requiredElements={"Report Date","Period Ending","Zacks Consensus Estimate","Earnings ESP"}; if(2==rows.Count) { HtmlNodeCollection headerColumns = rows[0].SelectNodes(".//th"); if (null!=headerColumns && 0!=headerColumns.Count && HeaderContains(headerColumns,requiredElements)) { HtmlNodeCollection dataColumns = rows[1].ChildNodes; String[] reportElements=dataColumns[1].InnerText.Split(' '); String reportDate=reportElements[0]; if(!"NA".Equals(reportDate)) { String[] formats=new[] {"MM/yyyy","M/yyyy"}; String periodEnding=dataColumns[3].InnerText; EarningsAnnouncement earningsAnnouncement=new EarningsAnnouncement(); earningsAnnouncement.Symbol=symbol; earningsAnnouncement.Date=Utility.ParseDate(reportDate); earningsAnnouncement.PeriodEnding=DateTime.ParseExact(periodEnding, formats, new System.Globalization.CultureInfo("en-US"),DateTimeStyles.AssumeLocal); if(reportElements[1].Equals("(AMC)"))earningsAnnouncement.Time="After Close"; else if(reportElements[1].Equals("(NONE)"))earningsAnnouncement.Time="Before Open"; else earningsAnnouncement.Time=reportElements[1]; earningsAnnouncements.Add(earningsAnnouncement); } } } } // get historical responseString=Utility.KeepAfter(responseString,"earnings_announcements_earnings_table"); if(null==responseString)return null; responseString=Utility.KeepBefore(responseString,"] ]"); if(null==responseString)return null; responseString=responseString+"]"; responseString=Utility.KeepAfter(responseString,"[ ["); if(null==responseString)return null; responseString="["+responseString; responseString=Utility.RemoveDivs(responseString); if(null==responseString)return null; String[] items=responseString.Split(']'); for(int index=0;index 0) continue; HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td"); if(dataColumns.Count<4)continue; if(null!=dataColumns[0])split.Exchange=dataColumns[0].InnerText.Trim().ToUpper(); // exchange if(null!=dataColumns[1])split.Symbol=dataColumns[1].InnerText.Trim().ToUpper(); if(null!=dataColumns[2])split.EffectiveDate=Utility.ParseDate(dataColumns[2].InnerText.Trim().ToUpper()); if(null!=dataColumns[3])split.StrRatio=dataColumns[3].InnerText.Trim().ToUpper(); splits.Add(split); } memoryStream.Close(); memoryStream.Dispose(); return splits; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ************************************************************************** G D P P E R C A P I T A D A T A W O R L D B A N K ************************************************* // *************************************************************************************************************************************************************************************** public static EconomicIndicators GetGDPPerCapita(bool debug=false) { HttpNetResponse httpNetResponse=null; try { // Retrieve compressed CSV from World Bank web site and write to disk StringBuilder sb=new StringBuilder(); String strRequest; String currentWorkingDirectory=Directory.GetCurrentDirectory(); String strExtractFolder=currentWorkingDirectory+"/"+"extracts"; String strFileName="API_NY.GDP.MKTP.CD_DS2_V2_USD.zip"; String strPathFileName=currentWorkingDirectory+"/"+strFileName; sb.Append("http://api.worldbank.org/v2/en/indicator/NY.GDP.MKTP.CD?downloadformat=csv"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Downloading {0} from {1}",strFileName,strRequest)); httpNetResponse=HttpNetRequest.GetRequestStreamZIP(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } if(File.Exists(strPathFileName))File.Delete(strPathFileName); FileStream outStream=new FileStream(strPathFileName,FileMode.Create); byte[] streamBytes=httpNetResponse.ResponseStream.GetBuffer(); outStream.Write(streamBytes,0,streamBytes.Length); outStream.Flush(); outStream.Close(); outStream.Dispose(); return EconomicIndicators.FromZipFile(strPathFileName,strExtractFolder,debug); } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ************************************************************************** D I V I D E N D H I S T O R Y N A S D A Q *********************************************************** // *************************************************************************************************************************************************************************************** public static DividendHistory GetDividendHistory(String symbol) { HttpNetResponse httpNetResponse=null; DividendHistory dividendHistory = new DividendHistory(); try { StringBuilder sb = new StringBuilder(); String strRequest; CookieCollection cookieCollection = new CookieCollection(); WebProxy webProxy=HttpNetRequest.GetProxy("GetDividendHistory"); CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf"); else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest)); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); if(httpNetResponse.StatusCode.Equals(System.Net.HttpStatusCode.Forbidden)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request)); try{Thread.Sleep(15000);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest, cookieCollection, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } } else return null; } int groupBy=6; List> items=LocateJSONItems(httpNetResponse.ResponseString); if(null==items || 0==items.Count || items.Count uniqueSymbols = new Dictionary(); AnalystRatings analystRatings = new AnalystRatings(); AnalystRatings ratings = null; ratings = GetLatestAnalystRatingsBriefing("Upgrades"); if (null != ratings) { foreach (AnalystRating analystRating in ratings) { if (!uniqueSymbols.ContainsKey(analystRating.Symbol)) { analystRatings.Add(analystRating); uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol); } } } ratings = GetLatestAnalystRatingsBriefing("Downgrades"); if (null != ratings) { foreach (AnalystRating analystRating in ratings) { if (!uniqueSymbols.ContainsKey(analystRating.Symbol)) { analystRatings.Add(analystRating); uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol); } } } ratings = GetLatestAnalystRatingsBriefing("Reiterated"); if (null != ratings) { foreach (AnalystRating analystRating in ratings) { if (!uniqueSymbols.ContainsKey(analystRating.Symbol)) { analystRatings.Add(analystRating); uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol); } } } ratings = GetLatestAnalystRatingsBriefing("Initiated"); if (null != ratings) { foreach (AnalystRating analystRating in ratings) { if (!uniqueSymbols.ContainsKey(analystRating.Symbol)) { analystRatings.Add(analystRating); uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol); } } } return analystRatings; } private static AnalystRatings GetLatestAnalystRatingsBriefing(String type) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse = null; AnalystRatings analystRatings = new AnalystRatings(); try { String strRequest = null; if ("Upgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/upgrade.htm"; else if ("Downgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/downgrade.htm"; else if ("Reiterated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/reiterated.htm"; else if ("Initiated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/initiated.htm"; else return null; httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest); if (!httpNetResponse.Success || null == httpNetResponse.ResponseString) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection divSectionDate = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calDATE\"]"); HtmlNodeCollection divSectionHeader = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calHDR2\"]"); HtmlNodeCollection divSections = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calEVENT\"]"); if (null == divSectionDate || 0 == divSectionDate.Count) return null; if (null == divSectionHeader || 0 == divSectionHeader.Count) return null; if (null == divSections || 0 == divSections.Count) return null; DateTime ratingsDate = Utility.ParseDate(divSectionDate[0].InnerText); String ratingsType = divSectionHeader[0].InnerText; if (!ratingsType.StartsWith(type.ToUpper())) return null; // calDATA for (int index = 0; index < divSections.Count; index++) { List items = Sections.GetSections(divSections[index].InnerHtml); if (null == items || 0 == items.Count) continue; AnalystRating analystRating = new AnalystRating(); analystRating.CompanyName = items.Count > 0 ? items[0] : null; analystRating.Date = ratingsDate; analystRating.Symbol = items.Count > 4 ? items[4] : null; analystRating.Type = type; if (items.Count > 10 && items[10].StartsWith("Brokerage Firm")) analystRating.BrokerageFirm = items.Count > 12 ? items[12] : null; if (items.Count > 22 && items[22].StartsWith("Ratings Change")) analystRating.RatingsChange = items.Count > 24 ? items[24] : null; if (null != analystRating.RatingsChange & analystRating.RatingsChange.Contains("»")) analystRating.RatingsChange = analystRating.RatingsChange.Replace("»", "->"); if (items.Count > 28 && items[28].StartsWith("Price Tgt")) { String priceTarget = items.Count > 31 ? items[31] : null; if (null != priceTarget && priceTarget.Contains("»")) priceTarget = priceTarget.Replace("»", "-"); if (priceTarget.Contains("-")) { String[] prices = priceTarget.Split('-'); priceTarget = prices[1]; } analystRating.PriceTarget = FeedParser.ParseValue(priceTarget); } analystRatings.Add(analystRating); } return analystRatings; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, exception); return analystRatings; } finally { if (null != memoryStream) memoryStream.Close(); if (null != httpNetResponse) httpNetResponse.Dispose(); } } // ****************************************************************************************************************************************************************************************** // ****************************************************************************** A N A L Y S T R A T I N G S M A R K E T B E A T ******************************************************* // ****************************************************************************************************************************************************************************************** public static AnalystRatings GetAnalystRatingsMarketBeat(String symbol) { AnalystRatings analystRatings=new AnalystRatings(); String nasdaq="NASDAQ"; String nyse="NYSE"; try { StringBuilder sb=new StringBuilder(); CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); HtmlNodeCollection ratingsTable=null; ratingsTable=GetRatingsTableOnMarketBeat(symbol,nyse); if(null==ratingsTable) ratingsTable=GetRatingsTableOnMarketBeat(symbol,nasdaq); if(null==ratingsTable) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No ratings for {0} on MarketBeat",symbol)); return null; } HtmlNodeCollection header=ratingsTable[0].SelectNodes(".//th"); string[] requiredElements=new string[]{"Date","Brokerage","Action","Rating","Price Target","Details"}; if(!HeaderContains(header,requiredElements))return null; HtmlNodeCollection data=ratingsTable[0].SelectNodes(".//tr//td"); if(null==data)return analystRatings; for(int index=0,rowIndex=0;index"); analystRating.RatingsChange=strRating; if(null!=strPriceTarget) { if(strPriceTarget.Contains("➝")) strPriceTarget=strPriceTarget.Replace("➝","-"); if(strPriceTarget.Contains("-")) { String[] prices=strPriceTarget.Split('-'); strPriceTarget=prices[1]; } analystRating.PriceTarget=FeedParser.ParseValue(strPriceTarget); } if(analystRating.Type.Equals("Boost Price Target")) analystRating.Type="Upgrades"; if(analystRating.Type.Equals("Upgrade")) analystRating.Type="Upgrades"; if(analystRating.Type.Equals("Downgrade")) analystRating.Type="Downgrades"; if(analystRating.Type.StartsWith("Reiterated")) analystRating.Type="Reiterated"; if(analystRating.Type.StartsWith("Initiated")) analystRating.Type="Initiated"; if(analystRating.Type.StartsWith("Set Price Target"))analystRating.Type="Initiated"; if(!analystRating.Type.Equals("Upgrades")&&!analystRating.Type.Equals("Downgrades")&&!analystRating.Type.Equals("Reiterated")&&!analystRating.Type.Equals("Initiated")) continue; analystRatings.Add(analystRating); } return new AnalystRatings(analystRatings.Distinct().ToList()); } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return analystRatings; } finally { } } private static bool HeaderContains(HtmlNodeCollection header,String[] requiredElements) { List headerElements=new List(); if(null==header)return false; for(int index=0;index exceptions=requiredElements.Except(headerElements).ToList(); return exceptions.Count()>0?false:true; } private static int HeaderIndexOf(HtmlNodeCollection header,String identifier) { if(null==header)return -1; for(int index=0;index sections=Sections.GetSections(data[indexer].InnerHtml); if(null==sections || 0==sections.Count)return data[indexer].InnerText; return sections[0]; } private static HtmlNodeCollection GetRatingsTableOnMarketBeat(String symbol,String exchange) { MemoryStream memoryStream=null; HttpNetResponse httpNetResponse=null; try { String strRequest=null; StringBuilder sb=new StringBuilder(); sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3(strRequest, "www.marketbeat.com"); if(!httpNetResponse.Success||String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream=new MemoryStream(streamBytes); HtmlDocument htmlDocument=new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection ratingsTable=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"scroll-table sort-table\"]"); if(null==ratingsTable||0==ratingsTable.Count) return null; return ratingsTable; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally { if(null!=memoryStream) memoryStream.Close(); if(null!=httpNetResponse) httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************** //***************************************************************** C I K C O D E S - S E C . G O V *********************************************************************** // *************************************************************************************************************************************************************************** public static String GetCIK(String symbol) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; try { StringBuilder sb = new StringBuilder(); String strRequest; symbol = symbol.ToUpper(); sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany"); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"companyInfo\"]"); // yfnc_tablehead1,yfnc_modtitle1,yfnc_tabledata1 if (null == rows || 1!=rows.Count) { if (null == rows) MDTrace.WriteLine(LogLevel.DEBUG,"[GetCIK] Received no rows for '" + symbol + "'"); else MDTrace.WriteLine(LogLevel.DEBUG,"Expected 1 row, got " + rows.Count + " for '" + symbol + "'"); return null; } HtmlNode htmlNodeTitle = rows[0]; HtmlNodeCollection nodes = htmlNodeTitle.SelectNodes(".//a"); if (null == nodes || 0==nodes.Count) { MDTrace.WriteLine(LogLevel.DEBUG,"Expected tag for symbol '" + symbol + "'"); return null; } String cik = null; String[] strings = nodes[0].InnerText.Split(' '); if (strings.Length >= 1) cik = strings[0]; return cik; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if (null != memoryStream) memoryStream.Close(); if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ************************************ S E C F I L I N G S - F O R M 4 & F O R M 5 - F O R I N S I D E R T R A N S A C T I O N S S E C . G O V ***************************S // *************************************************************************************************************************************************************************************** public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=5) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; String[] descriptionStartsWith=new String[]{"Form 4","Form 5"}; DateTime minFilingDate=DateTime.Now; int maxFilings=80; bool continuationFlag=true; int TIMEOUT_MS_BETWEEN_REQUESTS=1000; try { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} dayCount:{1}",symbol,timePeriodDays)); DateGenerator dateGenerator=new DateGenerator(); String cik=PricingDA.GetCIKForSymbol(symbol); if(null==cik) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: No CIK for symbol {0}",symbol)); return null; } minFilingDate=dateGenerator.DaysAddActual(minFilingDate,Math.Abs(timePeriodDays)*-1); SECFilings secFilings = new SECFilings(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]"); if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count)); for (int row = 0; row < rows.Count && continuationFlag; row ++) { try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;} HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th"); if (null != headerColumns && headerColumns.Count > 0) continue; HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); if (null == dataColumns || 5 != dataColumns.Count) continue; HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl); for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++) { SECFiling innerFiling = innerCollection[index]; if(innerFiling.FilingDateinnerFiling.Description.StartsWith(x)))continue; secFilings.Add(innerFiling); } } return InsiderTransactionsParser.GetInstance().Parse(secFilings); } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if (null != memoryStream) memoryStream.Close(); if(null!=httpNetResponse)httpNetResponse.Dispose(); } } /// /// Get InsiderTransactions for years greater than or equal to specified year. /// For example. If 2023 is specified then get all for years>=2023 /// /// The symbol. /// the new y-coordinate. public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; String[] descriptionStartsWith=new String[]{"Form 4","Form 5"}; int maxFilings=120; bool continuationFlag=true; int TIMEOUT_MS_BETWEEN_REQUESTS=1000; try { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo)); DateGenerator dateGenerator=new DateGenerator(); String cik=PricingDA.GetCIKForSymbol(symbol); if(null==cik) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol)); return null; } SECFilings secFilings = new SECFilings(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]"); if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count)); for (int row = 0; row < rows.Count && continuationFlag; row ++) { try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;} HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th"); if (null != headerColumns && headerColumns.Count > 0) continue; HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); if (null == dataColumns || 5 != dataColumns.Count) continue; HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl); for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++) { SECFiling innerFiling = innerCollection[index]; if(innerFiling.FilingDate.YearinnerFiling.Description.StartsWith(x)))continue; secFilings.Add(innerFiling); } } return InsiderTransactionsParser.GetInstance().Parse(secFilings); } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if (null != memoryStream) memoryStream.Close(); if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ********************************************************************** S E C F I L I N G S - S E C . G O V ******************************************************************** // *************************************************************************************************************************************************************************************** public static SECFilings GetSECFilings(String symbol,int maxFilings=80) { try { String cik=PricingDA.GetCIKForSymbol(symbol); if(null==cik) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol)); return null; } return GetSECFilings(symbol,cik,maxFilings); } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } } public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000; SECFilings secFilings = new SECFilings(); try { StringBuilder sb = new StringBuilder(); String strRequest; sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]"); if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); for (int row = 0; row < rows.Count; row ++) { HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th"); if (null != headerColumns && headerColumns.Count > 0) continue; HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); if (null == dataColumns || 5 != dataColumns.Count) continue; HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;} SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl); for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++) { SECFiling innerFiling = innerCollection[index]; secFilings.Add(innerFiling); } } return secFilings; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if (null != memoryStream) memoryStream.Close(); if(null!=httpNetResponse)httpNetResponse.Dispose(); MDTrace.WriteLine(LogLevel.DEBUG,$"[GetSECFilings] Retrieved {secFilings.Count} SECFilings for {symbol}"); } } // Get the filing document in XML format private static SECFilings GetSECFilingDocuments(String symbol,DateTime filingDate,String secFilingDocumentUrl) { Profiler profiler = new Profiler(); MemoryStream memoryStream = null; SECFilings secFilings = new SECFilings(); String secFilingDocument = null; HttpNetResponse httpNetResponse=null; int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000; try { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl)); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments"); if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy."); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"secNum\"]"); String secAccessionNumber = secNodes[0].InnerText; secAccessionNumber = secAccessionNumber.Replace("\n",""); secAccessionNumber = secAccessionNumber.Trim(); if (secAccessionNumber.Contains("SEC Accession No.")) secAccessionNumber = secAccessionNumber.Substring(18, secAccessionNumber.Length - 18); else return null; secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"formName\"]"); String formName = secNodes[0].InnerText; formName = formName.Replace("\n", ""); formName = formName.Trim(); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]"); if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); httpNetResponse.Dispose(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. {0} Examining form {1} with {2} entries",symbol,formName,rows.Count)); // go through the rows in the table file, searching for the different submission form content int sequence = 0; for (int row = 0; row < rows.Count; row++) { HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th"); if (null != headerColumns && headerColumns.Count > 0) continue; HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); if (null == dataColumns || 5 != dataColumns.Count) continue; String document = dataColumns[2].InnerText.Trim(); String type = dataColumns[3].InnerText.Trim(); if(null!=type && type.Equals("GRAPHIC"))continue; if (document.Contains(".htm")) { HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl)); try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); continue; } secFilingDocument = httpNetResponse.ResponseString; if (null == secFilingDocument) continue; SECFiling secFiling = new SECFiling(); secFiling.SECAccessionNumber = secAccessionNumber; secFiling.Sequence = sequence++; secFiling.Symbol = symbol; secFiling.Form = type; secFiling.Description = formName; secFiling.FilingDate = filingDate; secFiling.FileNumber = dataColumns[4].InnerText.Trim(); secFiling.SecFilingUrl = secFilingDocumentXmlUrl; secFiling.FormText = secFilingDocument; secFiling.FormText = secFiling.FormText.Replace("\n",""); if (null == secFiling.FormText) continue; MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString())); secFilings.Add(secFiling); httpNetResponse.Dispose(); } } return secFilings; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if (null != memoryStream) { memoryStream.Close(); memoryStream = null; } MDTrace.WriteLine(LogLevel.DEBUG,string.Format("[GetSECFilingDocuments] Done, retrieved {0} documents for {1} in {2}(ms)",secFilings.Count,symbol,profiler.End())); } } // *************************************************************************************************************************************************************************************** // ********************************************************************** Y I E L D C U R V E - T R E A S U R Y . G O V ************************************************************** // *************************************************************************************************************************************************************************************** public static YieldCurve GetYieldCurve(int year) { YieldCurve yieldCurve = new YieldCurve(); HttpNetResponse httpNetResponse=null; try { StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/pages/xml?data=daily_treasury_yield_curve&field_tdr_date_value=").Append(year); strRequest = sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } XElement po = XElement.Parse(httpNetResponse.ResponseString); IEnumerable childElements = from el in po.Elements() where el.Name.LocalName.Equals("entry") select el; foreach (XElement element in childElements) { IEnumerable childElementsL1 = from el in element.Elements() select el; foreach (XElement elementL1 in childElementsL1) { if(elementL1.Name.LocalName.Equals("content")) { IEnumerable elementsL2=elementL1.Elements(); if(!elementsL2.FirstOrDefault().Name.LocalName.Equals("properties"))continue; IEnumerable elementsL3=elementsL2.FirstOrDefault().Elements(); YieldCurveData yieldCurveData = new YieldCurveData(); yieldCurveData.Date=Utility.Epoch; foreach (XElement elementL3 in elementsL3) { if(elementL3.Name.LocalName.EndsWith("NEW_DATE")) yieldCurveData.Date=Utility.ParseDate(Utility.RemoveAfter(elementL3.Value,'T')); else if(elementL3.Name.LocalName.EndsWith("1MONTH"))yieldCurveData.Mo1=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("2MONTH"))yieldCurveData.Mo2=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("3MONTH"))yieldCurveData.Mo3=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("6MONTH"))yieldCurveData.Mo6=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("1YEAR"))yieldCurveData.Yr1=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("2YEAR"))yieldCurveData.Yr2=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("3YEAR"))yieldCurveData.Yr3=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("5YEAR"))yieldCurveData.Yr5=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("7YEAR"))yieldCurveData.Yr7=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("10YEAR"))yieldCurveData.Yr10=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("20YEAR"))yieldCurveData.Yr20=FeedParser.ParseValue(elementL3.Value); else if(elementL3.Name.LocalName.EndsWith("30YEAR"))yieldCurveData.Yr30=FeedParser.ParseValue(elementL3.Value); } if(yieldCurveData.IsValid())yieldCurve.Add(yieldCurveData); } } } yieldCurve=new YieldCurve(yieldCurve.Distinct().ToList()); return yieldCurve; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return yieldCurve; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E ********************************************************* // *************************************************************************************************************************************************************************************** public static ETFHoldings GetETFHoldings(String etfSymbol) { MemoryStream memoryStream = null; ETFHoldings etfHoldings = new ETFHoldings(); HttpNetResponse httpNetResponse=null; DateTime modified=DateTime.Now; try { StringBuilder sb = new StringBuilder(); String strRequest; etfSymbol = etfSymbol.ToUpper(); sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings"); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } List sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table"); if(null == sections || 1!=sections.Count) { etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString); if(null==etfHoldings) { MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret the response string."); return null; } return etfHoldings; } String marker=" private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString) { ETFHoldings etfHoldings = new ETFHoldings(); try { DateTime modified = DateTime.Now; int groupBy=3; List sections = Sections.GetAllItemsInSections(responseString,"section"); if(null==sections || 0==sections.Count)return null; String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault(); if(String.IsNullOrEmpty(sectionItem))return null; List spans =Sections.GetAllItemsInSections(sectionItem,"span"); if(null==spans || 0==spans.Count || spans.Count","<"); String companyNameHeading=Utility.BetweenString(spans[index+1],">","<"); String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<"); if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets")) { MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading."); return null; } } else { ETFHolding etfHolding = new ETFHolding(); if(index+2>=spans.Count())continue; List subSectionsSymbol = Sections.GetSections(spans[index]); List subSectionsCompanyName = Sections.GetSections(spans[index+1]); List subSectionsPercentOfAssets = Sections.GetSections(spans[index+2]); etfHolding.ETFSymbol = etfSymbol; etfHolding.HoldingSymbolShareClass = null; if(null!=subSectionsSymbol && subSectionsSymbol.Count>0) { etfHolding.HoldingSymbol = subSectionsSymbol.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault(); } if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) { MDTrace.WriteLine(LogLevel.DEBUG,$"ETFHolding Symbol was not provided in the response for {etfSymbol}"); break; // we always expect to have a symbol, otherewise something is wrong with the collection. } if(null!=subSectionsCompanyName && subSectionsCompanyName.Count>0) { etfHolding.HoldingCompanyName = subSectionsCompanyName.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault(); } etfHolding.PercentOfAssets = FeedParser.ParseValue(subSectionsPercentOfAssets.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault()); if(double.IsNaN(etfHolding.PercentOfAssets)) { MDTrace.WriteLine(LogLevel.DEBUG,$"ETFHolding Unable to parse PercentOfAssets for Symbol {etfSymbol}."); } etfHolding.Modified = modified; etfHolding.HoldingSymbol=etfHolding.HoldingSymbol.Trim().ToUpper(); etfHoldings.Add(etfHolding); } } return 0==etfHoldings.Count?null:etfHoldings; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Symbol:{0} Exception:{1}",etfSymbol,exception.ToString())); return null; } } // ****************************************************************************************************************************************************************************** // **************************************************************************C O M P A N Y P R O F I L E ********************************************************************** // ***************************************************************** P R O F I L E : M O R N I N G S T A R ******************************************************************* // ***************************************************************** D E S C R I P T I O N : R E U T E R S ******************************************************************** // ****************************************************************************************************************************************************************************** public static CompanyProfile GetCompanyProfile(String symbol) { String nasdaq = "xnas"; String nyse = "xnys"; CompanyProfile companyProfile = null; companyProfile = GetCompanyProfileMorningStar(symbol,nasdaq); if (null == companyProfile)companyProfile = GetCompanyProfileMorningStar(symbol, nyse); if (null == companyProfile)companyProfile = GetCompanyProfileYahoo(symbol); return companyProfile; } // GetCompanyProfile - DataSource Yahoo Finance public static CompanyProfile GetCompanyProfileYahoo(String symbol) { HttpNetResponse httpNetResponse = null; try { StringBuilder sb = new StringBuilder(); String strRequest; symbol = symbol.ToUpper(); WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo"); sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } // Locate Industry String strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry", 4); if(strIndustry == null)strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry:??", 2); // Locate Sector String strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector", 4); if (null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector(s)", 4); if(null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector:??", 3); if (null != strIndustry && strIndustry.Contains(Constants.CONST_QUESTION)) strIndustry = strIndustry.Replace(Constants.CONST_QUESTION, " - "); if (null != strSector && strSector.Contains(Constants.CONST_QUESTION)) strSector = strSector.Replace(Constants.CONST_QUESTION, " - "); // Locate Description String strDescription = Sections.LocateItem(httpNetResponse.ResponseString, "Description", 4); if(null == strDescription) { List indices = Sections.LocateAllOccurrences(httpNetResponse.ResponseString, "Description"); if(indices.Count>0) { List sections = Sections.GetSections(httpNetResponse.ResponseString); strDescription = Sections.GetFirstNonEmptyItemInSection(sections, indices[0]+1); } } if(null!=strDescription && strDescription.Equals("Description Information Not Available")) { strDescription = null; } if(null == strIndustry || null == strSector || null == strDescription) { return null; } CompanyProfile companyProfile = new CompanyProfile(); companyProfile.Symbol = symbol; companyProfile.Sector = strSector; companyProfile.Industry = strIndustry; companyProfile.Description = strDescription; if (null != companyProfile.Sector) companyProfile.Sector = companyProfile.Sector.Trim(); if (null != companyProfile.Industry) companyProfile.Industry = companyProfile.Industry.Trim(); if (null != companyProfile.Description) companyProfile.Description = Utility.RemoveHtml(companyProfile.Description.Trim()); return companyProfile; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, exception); return null; } finally { if (null != httpNetResponse) httpNetResponse.Dispose(); } } /// /// Retrieve company profile information from MorningStar /// /// The Symbol /// XNAS or XNYS /// public static CompanyProfile GetCompanyProfileMorningStar(String symbol,String exchange) { HttpNetResponse httpNetResponse=null; try { if(null==exchange)return default; exchange=exchange.ToLower(); StringBuilder sb = new StringBuilder(); String strRequest; symbol = symbol.ToUpper(); WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyProfileMorningStar"); sb.Append(String.Format("https://www.morningstar.com/stocks/{0}/{1}/quote",exchange,symbol)); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 15000, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } String strSector=null; String strIndustry=null; String strDescription=null; int itemIndex=0; List sections=Sections.GetSections(httpNetResponse.ResponseString); bool found=Sections.FindInSections(sections,"Sector",0,ref itemIndex); if(found)strSector=sections[itemIndex+3]; found=Sections.FindInSections(sections,"Industry",itemIndex,ref itemIndex); if(found)strIndustry=sections[itemIndex+3]; found=Sections.FindInSections(sections,"Business Description",itemIndex,ref itemIndex); if(found)strDescription=sections[itemIndex+2]; else { itemIndex=0; found=Sections.FindInSections(sections,"Company Profile",itemIndex,ref itemIndex); if(found)strDescription = Sections.GetFirstNonEmptyItemInSection(sections, itemIndex+1); } if(null==strIndustry && null==strSector)return null; strSector=strSector.Trim(); strIndustry=strIndustry.Trim(); CompanyProfile companyProfile = new CompanyProfile(); companyProfile.Symbol = symbol; companyProfile.Sector = strSector; companyProfile.Industry = strIndustry; companyProfile.Description=strDescription; if(null!=companyProfile.Sector)companyProfile.Sector=companyProfile.Sector.Trim(); if(null!=companyProfile.Industry)companyProfile.Industry=companyProfile.Industry.Trim(); if(null!=strDescription)companyProfile.Description=companyProfile.Description.Trim(); if(null!=companyProfile.Industry && companyProfile.Industry.Contains(Constants.CONST_QUESTION)) { companyProfile.Industry=companyProfile.Industry.Replace(Constants.CONST_QUESTION," - "); } if(null!=companyProfile.Sector && companyProfile.Sector.Contains(Constants.CONST_QUESTION)) { companyProfile.Sector=companyProfile.Sector.Replace(Constants.CONST_QUESTION," - "); } if((null!=companyProfile.Sector && companyProfile.Sector.Equals(Constants.CONST_QUESTION)) || (null!=companyProfile.Industry && companyProfile.Industry.Equals(Constants.CONST_QUESTION)) || (null!=companyProfile.Description && companyProfile.Description.Equals(Constants.CONST_QUESTION)))return null; return companyProfile; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************** //***************************************************************** H E A D L I N E S - S E E K I N G A L P H A *********************************************************** // *************************************************************************************************************************************************************************** public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol) { Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol); return headlines; } /// /// Retrieves headlines from seeking alpha. If ignoreMarketDate is TRUE then the results will not be /// filtered by the current market date . /// /// /// /// public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); try { String domain = "seekingalpha.com"; StringBuilder sb = new StringBuilder(); String strRequest; symbol = symbol.ToUpper(); // Visit the home page WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV3"); sb.Append("https://").Append(domain); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(sb.ToString(),domain,30000,webProxy,false); if(!httpNetResponse.Success)return null; CookieCollection cookieCollection = httpNetResponse.CookieCollection; sb=new StringBuilder(); DateTime marketDate=PremarketDA.GetLatestMarketDate(); if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; #region Create Cookies StringBuilder lastVisitedPage = new StringBuilder(); lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); lastVisitedPage.Append(symbol); lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); cookieCollection.Add(new Cookie("LAST_VISITED_PAGE", lastVisitedPage.ToString()) { Domain = domain }); cookieCollection.Add(new Cookie("_ga", "GA1.1.1862228764.1739274352") { Domain = domain }); cookieCollection.Add(new Cookie("_ga_KGRFF2R2C5","GS1.1.1739274352.1.1.1739274637.60.0.0"){ Domain = domain }); cookieCollection.Add(new Cookie("_gcl_au","1.1.1611738349.1739274352"){ Domain = domain }); cookieCollection.Add(new Cookie("_pxvid","bfabea2f-e86d-11ef-832d-24c9f227f821"){ Domain = domain }); cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain }); cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain }); #endregion sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]="); sb.Append("0"); sb.Append("&filter[until]="); sb.Append("0"); sb.Append("&id=").Append(symbol); sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, strRequest); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); if(null==keyValuePairs)return null; foreach (KeyValue keyValue in keyValuePairs) { Headline headline=null; if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); headline.Entry=headline.Entry.Replace("\\"," "); headline.Entry=headline.Entry.Trim(); headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); headline.Entry=Uri.UnescapeDataString(headline.Entry); headline.Source = "Seeking Alpha"; headlines.Add(headline); } if(!ignoreMarketDate) { headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList()); } return headlines; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************** //***************************************************************** H E A D L I N E S - M A R K E T W A T C H ************************************************************* // *************************************************************************************************************************************************************************** public static Headlines GetCompanyHeadlinesMarketWatch(String symbol) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); DateTime modified=DateTime.Now; try { StringBuilder sb = new StringBuilder(); String strRequest; symbol = symbol.ToUpper(); MemoryStream memoryStream=null; CookieCollection cookieCollection=new CookieCollection(); httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.marketwatch.com",cookieCollection); Thread.Sleep(500); sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol).Append("?mod=search_symbol"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, strRequest); WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesMarketWatch"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,DEFAULT_TIMEOUT_MS,webProxy,true,cookieCollection); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream=new MemoryStream(streamBytes); HtmlDocument htmlDocument=new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection divSections=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"article__content\"]"); if(null == divSections) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for request {0} ",strRequest)); return headlines; } CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); for(int nodeIndex=0;nodeIndex sections=Sections.GetSections(htmlNodeHeadline.InnerHtml); if(null == sections || 0==sections.Count)continue; String strHeadline = Utility.RemoveHtml(sections[sections.Count-1]); String strTimestamp = htmlNodeTimestamp.InnerText; if(String.IsNullOrEmpty(strHeadline) || String.IsNullOrEmpty(strTimestamp)) { continue; } Headline headline= new Headline(); headline.Symbol=symbol; headline.Date=FeedParser.ParseValueDateTimeMonthFormatTZ(strTimestamp); headline.Entry=strHeadline.Trim(); headline.Source="MarketWatch"; headline.Modified=modified; headline.CompanyName=null!=companyProfile?companyProfile.CompanyName:null; if(Utility.IsEpoch(headline.Date)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Invalid date for symbol {0}",symbol)); continue; } headlines.Add(headline); } headlines=new Headlines(headlines.Distinct().ToList()); return headlines; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************** //************************************************************************ H E A D L I N E S - N A S D A Q ****************************************************************** // *************************************************************************************************************************************************************************** public static Headlines GetCompanyHeadlinesNASDAQ(String symbol) { HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; Headlines headlines=new Headlines(); try { StringBuilder sb=new StringBuilder(); String strRequest; symbol=symbol.ToUpper(); sb.Append("https://api.nasdaq.com/api/news/topic/articlebysymbol?q=").Append(symbol).Append("|etf&offset=0&limit=8&fallback=false"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } List> items=LocateJSONItemsTags(httpNetResponse.ResponseString); if(null==items || items.Count<7)return null; int index=0; for(;index GetHistoricalValues(String symbol) { Dictionary values = new Dictionary(); HttpNetResponse httpNetResponse=null; String nasdaq = "xnas"; String nyse = "xnyse"; String nys="xnys"; String salVersion="4.30.0"; String salVerson45100="4.51.0"; String strDefinition = default; String strColumnDefinition = default; int TIMEOUT_BETWEEN_REQUESTS_MS=500; String exchange=nasdaq; String url="https://www.morningstar.com/api/v2/stocks/"; try { StringBuilder sb = new StringBuilder(); StringBuilder strReferer = new StringBuilder(); String strRequest; sb=new StringBuilder(); sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); WebProxy webProxy=HttpNetRequest.GetProxy("GetHistoricalValues"); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { sb=new StringBuilder(); exchange=nys; sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { sb=new StringBuilder(); exchange=nyse; sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } } } // This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier. String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString); if(null==securityId) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol)); return null; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId)); // PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } List> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString); Dictionary dataSetsProfitabilityAndEfficiency=GetData("fiscalPeriodYear", profitabilityItems, httpNetResponse.ResponseString); httpNetResponse.Dispose(); // KETSTATS - FINANCIALHEALTH sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } List> items=LocateJSONItems(httpNetResponse.ResponseString); Dictionary dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString); httpNetResponse.Dispose(); // NEWFINANCIALS - ANNUAL/SUMMARY sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsAnnuals=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); // NEWFINANCIALS - INCOMESTATEMENT sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); // NEWFINANCIALS - CASHFLOW sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); //NEWFINANCIALS - BALANCESHEET sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;} httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } Dictionary dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); // RETURN ON ASSETS strDefinition="roa"; strColumnDefinition="columnDefs"; if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // RETURN ON INVESTED CAPITAL strDefinition="roic"; strColumnDefinition="columnDefs"; if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // BOOK VALUE PER SHARE strDefinition="bookValuePerShare"; strColumnDefinition="columnDefs"; if(dataSetsFinancialHealth.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.BVPS; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsFinancialHealth[strColumnDefinition],dataSetsFinancialHealth[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // INVENTORIES strDefinition="Inventories"; strColumnDefinition="columnDefs"; if(dataSetsBalanceSheet.ContainsKey(strDefinition)) { double multiplier=dataSetsBalanceSheet["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Inventory; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // ACCOUNTS RECEIVABLES strDefinition="Trade/Accounts Receivable, Current"; strColumnDefinition="columnDefs"; if(dataSetsBalanceSheet.ContainsKey(strDefinition)) { double multiplier=dataSetsBalanceSheet["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.AccountsReceivable; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // COST OF GOOD AND SERVICES strDefinition="Cost of Goods and Services"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) { double multiplier=dataSetsIncomeStatement["multiplier"].GetItem(0)*-1.00; TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.COGS; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // TOTAL OPERATING PROFIT/LOSS strDefinition="Total Operating Profit/Loss"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) { double multiplier=dataSetsIncomeStatement["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingIncome; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // INTEREST EXPENSE strDefinition="Interest Expense Net of Capitalized Interest"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) { double multiplier=dataSetsIncomeStatement["multiplier"].GetItem(0)*-1.00; TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.InterestExpense; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // REPORTED EFFECTIVE TAX RATE strDefinition="Reported Effective Tax Rate"; strColumnDefinition="columnDefs"; if(dataSetsIncomeStatement.ContainsKey(strDefinition)) { double multiplier=100.00; TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.TaxRate; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // TOTAL REVENUE strDefinition="Total Revenue"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) { double multiplier=dataSetsAnnuals["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Revenue; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // NET INCOME AVAILABLE TO COMMON STOCKHOLDERS strDefinition="Net Income Available to Common Stockholders"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) { double multiplier=dataSetsAnnuals["multiplier"].GetItem(0); multiplier/=1000000; TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.NetIncomeAvailableToCommonShareholders; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // DILUTED EPS strDefinition="Diluted EPS"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) { TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.EPS; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // WORKING CAPITAL strDefinition="Working Capital"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) { double multiplier=dataSetsAnnuals["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.WorkingCapital; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString()); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // FREE CASHFLOW strDefinition="Free Cash Flow"; strColumnDefinition="columnDefs"; if(dataSetsAnnuals.ContainsKey(strDefinition)) { double multiplier=dataSetsAnnuals["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.FreeCashflow; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } // OPERATING CASHFLOW strDefinition="Cash Generated from Operating Activities"; strColumnDefinition="columnDefs"; if(dataSetsCashflowStatement.ContainsKey(strDefinition)) { double multiplier=dataSetsCashflowStatement["multiplier"].GetItem(0); TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingCashflow; TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsCashflowStatement[strColumnDefinition],dataSetsCashflowStatement[strDefinition],elementType,multiplier); if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count) { values.Add(elementType, timeSeriesCollection); String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() ); MDTrace.WriteLine(LogLevel.DEBUG,strMessage); } } return values; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception); return null; } } internal class MStarDataSet { private String name; private List values; public MStarDataSet() { values=new List(); } public MStarDataSet(String name,List values) { this.name=name; this.values=values; } public MStarDataSet(String name,String value) { values=new List(); this.name=name; this.values.Add(value); } public String Name { get{return name;} set{name=value;} } public List Values { get{return values;} private set{;} } public T GetItem(int index=0) { T result=default(T); try {result = (T)Convert.ChangeType(values[index], typeof(T));} catch {result = default(T);} return result; } } // \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}" //\"orderOfMagnitude\":\"Million\", private static double GetMultiplierForDataSet(String responseString) { double multiplier=1.00; try { string token="\"indexName\""; int index=responseString.IndexOf(token); if(-1==index) { token="\"orderOfMagnitude\""; index=responseString.IndexOf(token); if(-1==index)return multiplier; string strSubString=responseString.Substring(index+token.Length); strSubString=Utility.BetweenString(strSubString,"\"","\""); string[] parts=strSubString.Split(' '); if(1!=parts.Length)return multiplier; if(parts[0].Equals("Million"))multiplier=1000000; else if(parts[0].Equals("Billion"))multiplier=1000000000; else if(parts[0].Equals("Trillion"))multiplier=1000000000000; else multiplier=1.00; return multiplier; } else { string strSubString=responseString.Substring(index+token.Length); strSubString=Utility.BetweenString(strSubString,"\"","\""); string[] parts=strSubString.Split(' '); if(2>parts.Length)return multiplier; if(parts[parts.Length-2].Equals("TR"))multiplier=1000000000; else { multiplier=1; } return multiplier; } } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMultiplierForDataSet encountered an exception:{0}",exception.ToString())); return multiplier; } } // \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}" //\"orderOfMagnitude\":\"Million\", // {"currency":"USD","currencySymbol":"$","orderOfMagnitude":"Billion","fiscalYearEndDate":"12-31"},"userType":"Free"} private static String GetCurrencyForDataSet(String responseString) { String currency=null; try { string token="\"indexName\""; int index=responseString.IndexOf(token); if(-1==index) { token="\"currency\""; index=responseString.IndexOf(token); if(-1==index)return currency; string strSubString=responseString.Substring(index+token.Length); strSubString=Utility.BetweenString(strSubString,"\"","\""); string[] parts=strSubString.Split(' '); if(1!=parts.Length)return currency; currency=parts[0]; return currency; } else { string strSubString=responseString.Substring(index+token.Length); strSubString=Utility.BetweenString(strSubString,"\"","\""); string[] parts=strSubString.Split(' '); if(2>parts.Length)return currency; currency=parts[parts.Length-1]; return currency; } } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCurrencyForDataSet encountered an exception:{0}",exception.ToString())); return currency; } } public static String GetMStarSecurityId(String symbol,String responseString) { String securityIdentifier=null; if(MStarSecurityDA.HasSecurityId(symbol)) { securityIdentifier=MStarSecurityDA.GetSecurityId(symbol); return securityIdentifier; } securityIdentifier=GetMStarSecurityId_1(responseString); if(null==securityIdentifier) { securityIdentifier=GetMStarSecurityId_2(symbol,responseString); if(null==securityIdentifier) { securityIdentifier=GetMStarSecurityId_3(responseString); } } if(null==securityIdentifier )return null; securityIdentifier=securityIdentifier.Trim(); if(securityIdentifier.Equals(","))return null; if (securityIdentifier.Length > 25) return null; // As a finakl check ... 25 is the maximum length of the security identifer in the database. MDTrace.WriteLine(LogLevel.DEBUG,$"Adding Morningtstar securityIdentifier. Symbol:{symbol} SecurityIdentifier:{securityIdentifier}"); MStarSecurityDA.PutSecurityId(symbol,securityIdentifier); return securityIdentifier; } //byId:{\"0P000003MU\":$} // AAPL:0P000000GY // 5","0P000003MU","MIDD",60 private static String GetMStarSecurityId_2(String symbol,String responseString) { try { String searchString="\""+symbol+"\""; int index=responseString.LastIndexOf(searchString); if(-1==index)return null; int commaCount=0; while(index>=0 && commaCount!=2) { char ch=responseString[index--]; if(ch.Equals(','))commaCount++; } if(index<0)return null; responseString=responseString.Substring(index+1); String securityIdentifier=Utility.BetweenString(responseString,"\"","\""); if(securityIdentifier.Equals("en-us")||securityIdentifier.StartsWith("blt"))return null; return securityIdentifier; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_2 encountered an exception:{0}",exception.ToString())); return null; } } // us","AAON",401,"subscription","blt8b91befd99376df6","bltc8ff21a187c6e079",2,"image\u002Fsvg+xml","blt187f65f0e94c77b5","0P00000012",60,80,"AAON In private static String GetMStarSecurityId_3(String responseString) { try { string token="0P"; int index=responseString.IndexOf(token); if(-1==index)return null; string securityIdentifier=responseString.Substring(index-token.Length); securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\""); return securityIdentifier; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_3 encountered an exception:{0}",exception.ToString())); return null; } } //","performanceID":"0P000000GY","securityID":"0P000000GY","ticker":"AAPL","exc private static String GetMStarSecurityId_1(String responseString) { try { string token="\"securityID\""; int index=responseString.IndexOf(token); if(-1==index)return null; string securityIdentifier=responseString.Substring(index+token.Length); securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\""); return securityIdentifier; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_1 encountered an exception:{0}",exception.ToString())); return null; } } private static Dictionary GetData(String responseString) { try { Dictionary uniqueDataLabels=new Dictionary(); String label="\"label\""; String datum="\"datum\""; String columnDefs="\"columnDefs\""; String strData=null; Dictionary data=new Dictionary(); if(null==responseString)return data; int index=responseString.IndexOf(columnDefs); if(-1==index)return null; strData=responseString.Substring(index+columnDefs.Length); strData=Utility.BetweenString(strData,"[","]"); List items = Utility.ToList(strData,','); for(int itemIndex=0;itemIndex(); } } private static Dictionary GetData(String mainHeading, List> items,String responseString) { try { Dictionary data=new Dictionary(); Dictionary> dataDictionary=new Dictionary>(); Dictionary uniqueElementNamesDictionary=new Dictionary(); if(null==mainHeading || null==items || null==responseString) { return data; } for(int itemIndex=0;itemIndex subItems=items[itemIndex]; if(subItems[0].Equals(mainHeading)) { String strDate=subItems[1]; dataDictionary.Add(strDate,new Dictionary()); for(++itemIndex;itemIndex subsubItems=items[itemIndex]; if(subsubItems[0].Equals(mainHeading)) { itemIndex--; break; } Dictionary datedItems=dataDictionary[strDate]; String subItemElementName=subsubItems[0]; if(!datedItems.ContainsKey(subItemElementName)) { if(!uniqueElementNamesDictionary.ContainsKey(subItemElementName))uniqueElementNamesDictionary.Add(subItemElementName,subItemElementName); datedItems.Add(subItemElementName,subsubItems[1]); } } } } List strDates=new List(dataDictionary.Keys); List uniqueElementNames=new List(uniqueElementNamesDictionary.Keys); data.Add("columnDefs",new MStarDataSet("columnDefs",strDates)); foreach(String elementName in uniqueElementNames) { List elementValues=new List(); foreach(String strDate in strDates) { Dictionary inner=dataDictionary[strDate]; if(!inner.ContainsKey(elementName)) { continue; } elementValues.Add(inner[elementName]); } data.Add(elementName,new MStarDataSet(elementName,elementValues)); } double multiplier=GetMultiplierForDataSet(responseString); if(double.IsNaN(multiplier))multiplier=1.00; // This GetData method is only used to retrieve ratios so far so the data sets do not have a multiplier section. data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString())); // If this method is expanded to retrieve data points other than non-ratios then you will have to come up with method for retrieving the multiplier. return data; // so that we know if the data is in trillions, billions, millions and what not. } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String mainHeading, List> items) encountered an exception {0}",exception.ToString())); return new Dictionary(); } } private static TimeSeriesCollection CreateTimeSeriesCollection(String symbol, MStarDataSet colDefData, MStarDataSet dataElements, TimeSeriesElement.ElementType elementType,double multiplier=1.00) { TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection(); Dictionary dates = new Dictionary(); DateTime today=DateTime.Now; try { if(null==colDefData || null==dataElements || null==colDefData.Values || null==dataElements.Values || dataElements.Values.Count()!=colDefData.Values.Count()) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot create time series for symbol {0} for element type {1}",symbol,TimeSeriesElement.StringForType(elementType))); return timeSeriesCollection; } for (int index = 0; index < colDefData.Values.Count;index++) { DateTime reportDate=DateTime.Now; String strDateElement=colDefData.Values[index]; String strDataElement=dataElements.Values[index]; if("_PO_".Equals(strDataElement)) { continue; } if("null".Equals(strDataElement)) { continue; } if(strDateElement.Length.Equals(4)) { bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate); if(!result)continue; if(reportDate.Year.Equals(today.Year)) { reportDate=new DateTime(reportDate.Year,today.Month,1); } else { reportDate=new DateTime(reportDate.Year,12,31); } } else { bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy-MM"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate); if(!result)continue; } TimeSeriesElement timeSeriesElement = new TimeSeriesElement(); timeSeriesElement.Type = elementType; timeSeriesElement.Symbol = symbol; timeSeriesElement.AsOf =reportDate; timeSeriesElement.Value = FeedParser.ParseValue(dataElements.Values[index])*multiplier; if (double.IsNaN(timeSeriesElement.Value)) continue; if (!dates.ContainsKey(timeSeriesElement.AsOf)) { dates.Add(timeSeriesElement.AsOf, timeSeriesElement.AsOf); timeSeriesCollection.Add(timeSeriesElement); } } return timeSeriesCollection; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateTimeSeriesCollection encountered an exception processing symbol '{0}' {1}",symbol,exception.ToString())); return new TimeSeriesCollection(); } } // **************************************************************************************************************************************************************************** // ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 E N D *********************************************** // **************************************************************************************************************************************************************************** // ***************************************************************************************************************************************************************************** // ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 ********************************************** // ***************************************************************************************************************************************************************************** public static List GetCashflowStatement(String symbol,CashflowStatement.PeriodType periodType) { List cashflowStatements=null; HttpNetResponse httpNetResponse=null; String nasdaq = "xnas"; String nyse = "xnyse"; String nys="xnys"; try { if(!CashflowStatement.PeriodType.Annual.Equals(periodType)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unsupported CashflowStatement.PeriodType Quarterly. Only Annual supported from M*")); return null; } StringBuilder sb = new StringBuilder(); StringBuilder strReferer = new StringBuilder(); String strRequest; sb=new StringBuilder(); sb.Append("https://www.morningstar.com/stocks/").Append(nasdaq).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement"); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { sb=new StringBuilder(); sb.Append("https://www.morningstar.com/stocks/").Append(nys).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { sb=new StringBuilder(); sb.Append("https://www.morningstar.com/stocks/").Append(nyse).Append("/").Append(symbol).Append("/performance"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy); if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } } } String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString); if(null==securityId) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol)); return null; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId)); try{Thread.Sleep(250);}finally{;} // Fetch the equity financial details. Success often requires several attempts. httpNetResponse=GetMStarEquityFinancialDetails(securityId); if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to retrieve MorningStar EquityFinancialDetails.")); } Dictionary dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); httpNetResponse.Dispose(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} cashflow statement items for {1}",dataSetsCashflowStatement.Count,symbol)); cashflowStatements=CreateCashflowStatements(symbol, dataSetsCashflowStatement); return cashflowStatements; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCashflowStatement encountered an exception {0}",exception.ToString())); return null; } } private static HttpNetResponse GetMStarEquityFinancialDetails(String securityId) { HttpNetResponse httpNetResponse = new HttpNetResponse(); int maxRetries=10; int retry=0; int waitBetweenAttemptsMS=500; StringBuilder sb=new StringBuilder(); sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=3.74.0"); String strRequest=sb.ToString(); try { MDTrace.WriteLine(LogLevel.DEBUG,strRequest); WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement"); while(retry CreateCashflowStatements(String symbol,Dictionary cashflowDataElements) { DateTime today=DateTime.Now; List cashflowStatements=new List(); String currencyCashflow=cashflowDataElements["currency"].Values[0]; double cashflowStatementMultiplier=double.Parse(cashflowDataElements["multiplier"].Values[0]); Dictionary conversionRates=new Dictionary(); String baseCurrency="USD"; try { Dictionary> cashflowDictionaryByDate=CreateDictionaryByDate(cashflowDataElements); List cashflowDateKeys=new List(cashflowDictionaryByDate.Keys); if(0==cashflowDateKeys.Count) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: No dates in cashflowDataElement.")); return null; } foreach(DateTime cashflowDate in cashflowDateKeys) { Dictionary cashflowDatedElements=cashflowDictionaryByDate[cashflowDate]; CashflowStatement cashflowStatement=new CashflowStatement(); cashflowStatement.Symbol=symbol; cashflowStatement.AsOf=cashflowDate; cashflowStatement.Modified=DateTime.Now; cashflowStatement.Period=CashflowStatement.PeriodType.Annual; if(!baseCurrency.Equals(currencyCashflow)) { if(null==CurrencyConversionDA.GetMaxDateForCurrency(currencyCashflow,baseCurrency,cashflowStatement.AsOf)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currencyCashflow,cashflowStatement.AsOf.ToShortDateString())); CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currencyCashflow,cashflowStatement.AsOf); if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString())); return null; } } CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currencyCashflow,baseCurrency,cashflowStatement.AsOf); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6))); conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement); } double currencyMultiplier=1.00; currencyMultiplier=conversionRates.ContainsKey(cashflowStatement.AsOf)?conversionRates[cashflowStatement.AsOf].UnitsPerSource:currencyMultiplier; if(cashflowDatedElements.ContainsKey("Depreciation and Amortization, Non-Cash Adjustment")) { String value = cashflowDatedElements["Depreciation and Amortization, Non-Cash Adjustment"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.DepreciationAndAmortization=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.DepreciationAndAmortization=double.NaN; } if(cashflowDatedElements.ContainsKey("Deferred Taxes, Non-Cash Adjustment")) { String value = cashflowDatedElements["Deferred Taxes, Non-Cash Adjustment"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.DeferredIncomeTaxes=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.DeferredIncomeTaxes=double.NaN; } if(cashflowDatedElements.ContainsKey("Stock-Based Compensation, Non-Cash Adjustment")) { String value = cashflowDatedElements["Stock-Based Compensation, Non-Cash Adjustment"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.StockBasedCompensation=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.StockBasedCompensation=double.NaN; } if(cashflowDatedElements.ContainsKey("Change in Trade/Accounts Receivable")) { String value = cashflowDatedElements["Change in Trade/Accounts Receivable"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.AccountsReceivable=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.AccountsReceivable=double.NaN; } if(cashflowDatedElements.ContainsKey("Change in Accrued Expenses")) { String value = cashflowDatedElements["Change in Accrued Expenses"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.AccruedLiabilities=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.AccruedLiabilities=double.NaN; } if(cashflowDatedElements.ContainsKey("Cash Generated from Operating Activities")) { String value = cashflowDatedElements["Cash Generated from Operating Activities"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)) { cashflowStatement.OperatingCashflow=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier; } else cashflowStatement.OperatingCashflow=double.NaN; } if(cashflowDatedElements.ContainsKey("Purchase/Sale and Disposal of Property, Plant and Equipment, Net")) { String value = cashflowDatedElements["Purchase/Sale and Disposal of Property, Plant and Equipment, Net"]; if(!"_PO_".Equals(value)&&!"null".Equals(value)&& !double.IsNaN(cashflowStatement.OperatingCashflow)) { cashflowStatement.FreeCashflow=(cashflowStatement.OperatingCashflow+(double.Parse(value)*cashflowStatementMultiplier))*currencyMultiplier; } else cashflowStatement.FreeCashflow=double.NaN; } if(cashflowStatement.IsValid()) { cashflowStatements.Add(cashflowStatement); } } return cashflowStatements; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: Exception encountered {0}",exception.ToString())); return null; } } private static Dictionary> CreateDictionaryByDate(Dictionary dataSets) { try { DateTime today=DateTime.Now; Dictionary> dictionaryByDate=new Dictionary>(); MStarDataSet columnDefs=dataSets["columnDefs"]; List itemKeys=new List(dataSets.Keys); itemKeys.RemoveAt(itemKeys.IndexOf("columnDefs")); itemKeys.RemoveAt(itemKeys.IndexOf("multiplier")); itemKeys.RemoveAt(itemKeys.IndexOf("currency")); for(int dateIndex=0;dateIndex()); } Dictionary datedCollection=dictionaryByDate[reportDate]; if(!datedCollection.ContainsKey(itemKey)) { datedCollection.Add(itemKey,value); } else datedCollection[itemKey]=value; } } return dictionaryByDate; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateDictionaryByDate: Exception encountered {0}",exception.ToString())); return null; } } // ***************************************************************************************************************************************************************************** // ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 E N D ************************************ // ***************************************************************************************************************************************************************************** // **************************************************************************************************************************************************************************** // ************************************************************ I N C O M E S T A T E M E N T - N A S D A Q / F I N V I Z ************************************************** // **************************************************************************************************************************************************************************** public static List GetIncomeStatement(String symbol,IncomeStatement.PeriodType periodType) { List incomeStatements=GetIncomeStatementNASDAQ(symbol,periodType); if(null!=incomeStatements && incomeStatements.Count>0)return incomeStatements; incomeStatements=GetIncomeStatementFinViz(symbol,periodType); return incomeStatements; } //{"currency":"USD","data":{ //"Period End Date":["TTM","1/2/2021","12/28/2019","12/29/2018","12/30/2017","12/31/2016","1/2/2016","1/3/2015"], //"Total Revenue":["2,930.65","2,513.26","2,959.45","2,722.93","2,335.54","2,267.85","1,826.60","1,636.54"], //"Cost of Revenue":["1,872.32","1,631.21","1,855.95","1,718.79","1,422.80","1,366.67","1,120.09","995.95"], //"Selling, General and Administrative":["596.80","531.90","583.81","538.84","468.22","471.64","344.55","320.37"], //"Gross Profit":["1,058.33","882.05","1,103.50","1,004.14","912.74","901.18","706.50","640.59"], //"Total Operating Expense":["2,492.86","2,188.83","2,445.40","2,276.97","1,956.93","1,848.83","1,523.99","1,336.11"], //"Research and Development":["","35.30","41.20","35.30","29.10","26.30","22.40","22.60"], //"Net Income":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"], //"Income Avail. to Common Incl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"], //"Interest Income Net":["-71.44","-81.52","-83.51","-56.21","-23.64","-21.98","-10.17","-11.99"], //"Period Length":["12 Months","53 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","53 Weeks"], //"Unusual Expense/Income":["23.74","25.72","5.64","19.33","65.91","10.52","59.35","19.78"], //"Operating Income":["437.79","324.43","514.04","445.97","378.61","419.02","302.60","300.43"], //"Other Income Net":["31.16","25.14","32.09","33.76","28.55","24.27","-11.27","-7.65"], //"Net Income Before Taxes":["397.51","268.06","462.62","423.51","383.53","421.31","281.17","280.79"], //"Provision for Income Taxes":["75.31","60.76","110.38","106.36","85.40","137.09","89.56","87.48"], //"Income Avail. to Common Excl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"], //"Diluted Average Shares":["55.71","55.14","55.66","55.60","56.72","57.09","56.97","56.78"], //"Diluted EPS Excl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"], //"Diluted EPS Incl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"]}} public static List GetIncomeStatementFinViz(String symbol,IncomeStatement.PeriodType periodType) { HttpNetResponse httpNetResponse=null; String baseCurrency="USD"; try { List incomeStatements=new List(); Dictionary conversionRates=new Dictionary(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower()); if(IncomeStatement.PeriodType.Annual.Equals(periodType))sb.Append("&s=IA"); else sb.Append("&s=IQ"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz {0}",strRequest)); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage)); return null; } String currency="USD"; JObject json=JObject.Parse(httpNetResponse.ResponseString); JToken jsonToken=null; if(null!=(jsonToken=json.SelectToken("currency"))) { currency=jsonToken.ToObject(); } if(null==(jsonToken=json.SelectToken("data"))) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("")); return null; } List strDateItems=null; List strTotalRevenueItems=null; List strCostOfRevenueItems=null; List strSGAItems=null; List strGrossProfitItems=null; List strTotalOperatingExpense=null; List strResearchAndDevelopmentItems=null; List strIncomeAvailableToCommonShareHolders=null; JToken token=jsonToken["Period End Date"]; if(null==token) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No IncomeStatemnt dates returned for {0}",symbol)); return null; } strDateItems=token.ToObject>(); token=jsonToken["Total Revenue"]; if(null!=token)strTotalRevenueItems=token.ToObject>(); token=jsonToken["Cost of Revenue"]; if(null!=token)strCostOfRevenueItems=token.ToObject>(); token=jsonToken["Selling, General and Administrative"]; if(null!=token)strSGAItems=token.ToObject>(); token=jsonToken["Gross Profit"]; if(null!=token)strGrossProfitItems=token.ToObject>(); token=jsonToken["Total Operating Expense"]; if(null!=token)strTotalOperatingExpense=token.ToObject>(); token=jsonToken["Research and Development"]; if(null!=token)strResearchAndDevelopmentItems=token.ToObject>(); token=jsonToken["Income Avail. to Common Incl. Extraord."]; if(null!=token)strIncomeAvailableToCommonShareHolders=token.ToObject>(); for(int index=0;indexindex)incomeStatement.TotalRevenue=FeedParser.ParseValue(strTotalRevenueItems[index])*1000000.00*multiplier; if(null!=strCostOfRevenueItems && strCostOfRevenueItems.Count>index)incomeStatement.CostOfRevenue=FeedParser.ParseValue(strCostOfRevenueItems[index])*1000000.00*multiplier; if(null!=strSGAItems && strSGAItems.Count>index)incomeStatement.SGA=FeedParser.ParseValue(strSGAItems[index])*1000000.00*multiplier; if(null!=strGrossProfitItems && strGrossProfitItems.Count>index)incomeStatement.GrossProfit=FeedParser.ParseValue(strGrossProfitItems[index])*1000000.00*multiplier; if(null!=strTotalOperatingExpense && strTotalOperatingExpense.Count>index)incomeStatement.OperatingExpenses=FeedParser.ParseValue(strTotalOperatingExpense[index])*1000000.00*multiplier; if(null!=strResearchAndDevelopmentItems && strResearchAndDevelopmentItems.Count>index)incomeStatement.ResearchAndDevelopment=FeedParser.ParseValue(strResearchAndDevelopmentItems[index])*1000000.00*multiplier; if(null!=strIncomeAvailableToCommonShareHolders && strIncomeAvailableToCommonShareHolders.Count>index)incomeStatement.NetIncomeApplicableToCommonShares=FeedParser.ParseValue(strIncomeAvailableToCommonShareHolders[index])*1000000.00*multiplier; incomeStatements.Add(incomeStatement); } return incomeStatements; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString())); return null; } } public static List GetIncomeStatementNASDAQ(String symbol,IncomeStatement.PeriodType periodType) { List incomeStatements=new List(); HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; double multiplier=1000.00; try { StringBuilder sb=new StringBuilder(); String strRequest; sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials"); if(periodType.Equals(IncomeStatement.PeriodType.Annual))sb.Append(""); else sb.Append("?frequency=2"); strRequest=sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest); if(!httpNetResponse.Success) return incomeStatements; byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); JObject json=JObject.Parse(httpNetResponse.ResponseString); JToken jsonToken=null; if(null==(jsonToken=json.SelectToken("data.incomeStatementTable.headers"))) { String reason="unknown"; jsonToken=json.SelectToken("message"); if(null!=jsonToken)reason=jsonToken.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason)); return incomeStatements; } JObject headerObject=(JObject)jsonToken; int elements=headerObject.Count-1; String headerElement=headerObject["value1"].ToString(); if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:"))) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest)); return incomeStatements; } for(int headerIndex=2;headerIndex GetBalanceSheet(String symbol,BalanceSheet.PeriodType periodType) { List balanceSheet=null; balanceSheet=GetBalanceSheetNASDAQ(symbol,periodType); if(null!=balanceSheet && balanceSheet.Count>0)return balanceSheet; balanceSheet=GetBalanceSheetFinViz(symbol,periodType); return balanceSheet; } //https://finviz.com/api/statement.ashx?t=aapl&s=BA //https://finviz.com/api/statement.ashx?t=aapl&s=BQ //{"currency":"USD", //"data":{ //"Period End Date":["9/26/2020","9/28/2019","9/29/2018","9/30/2017","9/24/2016","9/26/2015","9/27/2014","9/28/2013"], //"Cash and Equivalents":["20,243.00","36,640.00","14,338.00","12,307.00","11,883.00","9,731.00","3,612.00","5,554.00"], //"Short Term Investments":["52,927.00","51,713.00","40,388.00","53,892.00","46,671.00","20,481.00","11,233.00","26,287.00"], //"Cash and Short Term Investments":["90,943.00","100,557.00","66,301.00","74,181.00","67,155.00","41,601.00","25,077.00","40,546.00"], //"Accounts Receivable - Trade, Net":["16,120.00","22,926.00","23,186.00","17,874.00","15,754.00","16,849.00","17,460.00","13,102.00"], //"Total Receivables, Net":["37,445.00","45,804.00","48,995.00","35,673.00","29,299.00","30,343.00","27,219.00","20,641.00"], //"Total Inventory":["4,061.00","4,106.00","3,956.00","4,855.00","2,132.00","2,349.00","2,111.00","1,764.00"], //"Total Current Assets":["143,713.00","162,819.00","131,339.00","128,645.00","106,869.00","89,378.00","68,531.00","73,286.00"], //"Property, Plant And Equipment - Gross":["112,096.00","95,957.00","90,403.00","75,076.00","61,245.00","49,257.00","39,015.00","28,519.00"], //"Property, Plant And Equipment - Net":["45,336.00","37,378.00","41,304.00","33,783.00","27,010.00","22,471.00","20,624.00","16,597.00"], //"Goodwill, Net":["","","","","5,414.00","5,116.00","4,616.00","1,577.00"], //"Intangibles, Net":["","","","","3,206.00","3,893.00","4,142.00","4,179.00"], //"Long Term Investments":["100,887.00","105,341.00","170,799.00","194,714.00","170,430.00","164,065.00","130,162.00","106,215.00"], //"Other Long Term Assets":["33,952.00","32,978.00","22,283.00","18,177.00","8,757.00","5,422.00","3,764.00","5,146.00"], //"Total Assets":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"], //"Accounts Payable":["42,296.00","46,236.00","55,888.00","44,242.00","37,294.00","35,490.00","30,196.00","22,367.00"], //"Accrued Expenses":["1,436.00","","","","20,951.00","24,169.00","7,689.00","4,782.00"], //"Notes Payable/Short Term Debt":["4,996.00","5,980.00","11,964.00","11,977.00","8,105.00","8,499.00","6,308.00","0.00"], //"Current Port. of LT Debt/Capital Leases":["8,797.00","10,260.00","8,784.00","6,496.00","3,500.00","2,500.00","",""], //"Other Current liabilities":["47,867.00","43,242.00","39,293.00","38,099.00","9,156.00","9,952.00","19,255.00","16,509.00"], //"Total Current Liabilities":["105,392.00","105,718.00","115,929.00","100,814.00","79,006.00","80,610.00","63,448.00","43,658.00"], //"Long Term Debt":["98,667.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"], //"Capital Lease Obligations":["637.00","","","","","","",""], //"Total Long Term Debt":["99,304.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"], //"Total Debt":["113,097.00","108,047.00","114,483.00","115,680.00","87,032.00","64,328.00","35,295.00","16,960.00"], //"Other Liabilities":["53,853.00","50,503.00","48,914.00","11,747.00","12,985.00","12,989.00","7,598.00","6,344.00"], //"Total Liabilities":["258,549.00","248,028.00","258,578.00","241,272.00","193,437.00","170,990.00","120,292.00","83,451.00"], //"Common Stock":["50,779.00","45,174.00","40,201.00","35,867.00","31,251.00","27,416.00","23,313.00","19,764.00"], //"Additional Paid-In Capital":["","","","","","","",""], //"Retained Earnings":["14,966.00","45,898.00","70,400.00","98,330.00","96,364.00","92,284.00","87,152.00","104,256.00"], //"Unrealized Gain (Loss)":["1,846.00","707.00","-3,209.00","328.00","1,174.00","-653.00","-242.00","-105.00"], //"Other Equity":["-2,252.00","-1,291.00","-245.00","-478.00","-540.00","308.00","1,324.00","-366.00"], //"Total Equity":["65,339.00","90,488.00","107,147.00","134,047.00","128,249.00","119,355.00","111,547.00","123,549.00"], //"Total Liabilities and Equity":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"], //"Total Common Shares Outstanding":["16,976.76","17,772.94","19,019.94","20,504.80","21,344.66","22,315.01","23,464.64","25,177.46"], //"Full-Time Employees":["147,000","137,000","132,000","123,000","116,000","110,000","92,600","80,300"], //"Number of Common Shareholders":["22,797","23,233","23,712","25,333","25,641","25,924","26,112","24,710"]}} public static List GetBalanceSheetFinViz(String symbol,BalanceSheet.PeriodType periodType) { HttpNetResponse httpNetResponse=null; String baseCurrency="USD"; try { List balanceSheets=new List(); Dictionary conversionRates=new Dictionary(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower()); if(BalanceSheet.PeriodType.Annual.Equals(periodType))sb.Append("&s=BA"); else sb.Append("&s=BQ"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz {0}",strRequest)); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage)); return null; } String currency="USD"; JObject json=JObject.Parse(httpNetResponse.ResponseString); JToken jsonToken=null; if(null!=(jsonToken=json.SelectToken("currency"))) { currency=jsonToken.ToObject(); } if(null==(jsonToken=json.SelectToken("data"))) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("")); return null; } List strDateItems=null; List strLongTermDebtItems=null; List strTotalCurrentAssetItems=null; List strTotalLiabilityItems=null; List strTotalAssetItems=null; List strGoodwillItems=null; List strPPEItems=null; List strCashAndEquivalentsItems=null; List strTotalInventoryItems=null; List strIntangiblesItems=null; List strTotalCurrentLiabilityItems=null; List strOtherLiabilityItems=null; List strTotalEquityItems=null; JToken token=jsonToken["Period End Date"]; if(null==token) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No BalanceSheet dates returned for {0}",symbol)); return null; } strDateItems=token.ToObject>(); token=jsonToken["Long Term Debt"]; if(null!=token)strLongTermDebtItems=token.ToObject>(); token=jsonToken["Total Current Assets"]; if(null!=token)strTotalCurrentAssetItems=token.ToObject>(); token=jsonToken["Total Liabilities"]; if(null!=token)strTotalLiabilityItems=token.ToObject>(); token=jsonToken["Total Assets"]; if(null!=token)strTotalAssetItems=token.ToObject>(); token=jsonToken["Goodwill, Net"]; if(null!=token)strGoodwillItems=token.ToObject>(); token=jsonToken["Property, Plant And Equipment - Net"]; if(null!=token)strPPEItems=token.ToObject>(); token=jsonToken["Cash and Equivalents"]; if(null!=token)strCashAndEquivalentsItems=token.ToObject>(); token=jsonToken["Total Inventory"]; if(null!=token)strTotalInventoryItems=token.ToObject>(); token=jsonToken["Intangibles, Net"]; if(null!=token)strIntangiblesItems=token.ToObject>(); token=jsonToken["Total Current Liabilities"]; if(null!=token)strTotalCurrentLiabilityItems=token.ToObject>(); token=jsonToken["Other Liabilities"]; if(null!=token)strOtherLiabilityItems=token.ToObject>(); token=jsonToken["Total Equity"]; if(null!=token)strTotalEquityItems=token.ToObject>(); for(int index=0;indexindex)balanceSheet.LongTermDebt=FeedParser.ParseValue(strLongTermDebtItems[index])*1000000.00*multiplier; if(null!=strTotalCurrentAssetItems && strTotalCurrentAssetItems.Count>index)balanceSheet.TotalCurrentAssets=FeedParser.ParseValue(strTotalCurrentAssetItems[index])*1000000.00*multiplier; if(null!=strTotalLiabilityItems && strTotalLiabilityItems.Count>index)balanceSheet.TotalLiabilities=FeedParser.ParseValue(strTotalLiabilityItems[index])*1000000.00*multiplier; if(null!=strTotalAssetItems && strTotalAssetItems.Count>index)balanceSheet.TotalAssets=FeedParser.ParseValue(strTotalAssetItems[index])*1000000.00*multiplier; if(null!=strGoodwillItems && strGoodwillItems.Count>index)balanceSheet.Goodwill=FeedParser.ParseValue(strGoodwillItems[index])*1000000.00*multiplier; if(null!=strPPEItems && strPPEItems.Count>index)balanceSheet.PropertyPlantAndEquipment=FeedParser.ParseValue(strPPEItems[index])*1000000.00*multiplier; if(null!=strCashAndEquivalentsItems && strCashAndEquivalentsItems.Count>index)balanceSheet.CashAndCashEquivalents=FeedParser.ParseValue(strCashAndEquivalentsItems[index])*1000000.00*multiplier; if(null!=strTotalInventoryItems && strTotalInventoryItems.Count>index)balanceSheet.Inventory=FeedParser.ParseValue(strTotalInventoryItems[index])*1000000.00*multiplier; if(null!=strIntangiblesItems && strIntangiblesItems.Count>index)balanceSheet.IntangibleAssets=FeedParser.ParseValue(strIntangiblesItems[index])*1000000.00*multiplier; if(null!=strTotalCurrentLiabilityItems && strTotalCurrentLiabilityItems.Count>index)balanceSheet.TotalCurrentLiabilities=FeedParser.ParseValue(strTotalCurrentLiabilityItems[index])*1000000.00*multiplier; if(null!=strOtherLiabilityItems && strOtherLiabilityItems.Count>index)balanceSheet.OtherLiabilities=FeedParser.ParseValue(strOtherLiabilityItems[index])*1000000.00*multiplier; if(null!=strTotalEquityItems && strTotalEquityItems.Count>index)balanceSheet.TotalStockHolderEquity=FeedParser.ParseValue(strTotalEquityItems[index])*1000000.00*multiplier; balanceSheets.Add(balanceSheet); } return balanceSheets; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString())); return null; } } public static List GetBalanceSheetNASDAQ(String symbol,BalanceSheet.PeriodType periodType) { List balanceSheets=new List(); HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; double multiplier=1000.00; try { StringBuilder sb=new StringBuilder(); String strRequest; sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials"); if(periodType.Equals(BalanceSheet.PeriodType.Annual)) sb.Append("?frequency=1"); else sb.Append("?frequency=2"); strRequest=sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest); if(!httpNetResponse.Success) return balanceSheets; byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); JObject json=JObject.Parse(httpNetResponse.ResponseString); JToken jsonToken=null; if(null==(jsonToken=json.SelectToken("data.balanceSheetTable.headers"))) { String reason="unknown"; jsonToken=json.SelectToken("message"); if(null!=jsonToken) reason=jsonToken.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason)); return balanceSheets; } JObject headerObject=(JObject)jsonToken; int elements=headerObject.Count-1; String headerElement=headerObject["value1"].ToString(); if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:"))) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest)); return balanceSheets; } for(int headerIndex=2;headerIndex sections=Sections.GetSections(httpNetResponse.ResponseString); if(!Sections.FindInSections(sections,"Currency Table:",0,ref itemIndex,false))return null; if(!Sections.FindInSections(sections,"Currency",itemIndex,ref itemIndex))return null; itemIndex+=20; while(true) { CurrencyConversionElement currencyConversionElement=new CurrencyConversionElement(); currencyConversionElement.AsOf=conversionDate; currencyConversionElement.SourceCurrency=sourceCurrency; currencyConversionElement.DestinationCurrency=sections[itemIndex]; itemIndex+=3; currencyConversionElement.DestinationCurrencyName=sections[itemIndex]; itemIndex+=2; currencyConversionElement.UnitsPerSource=FeedParser.ParseValue(sections[itemIndex]); itemIndex+=2; currencyConversionElement.SourcePerUnit=FeedParser.ParseValue(sections[itemIndex]); currencyConversionElements.Add(currencyConversionElement); itemIndex+=5; if(itemIndex>=sections.Count || null==sections[itemIndex] || "".Equals(sections[itemIndex]) || sections[itemIndex].Length<3)break; } return currencyConversionElements; } catch(Exception) { return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // *************************************************************************************************************************************************************************************** // ************************************************************************** F U N D A M E N T A L - F I N V I S ********************************************************* // *************************************************************************************************************************************************************************************** public static Fundamental GetFundamentalFinViz(String symbol) { HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; try { Fundamental fundamental = new Fundamental(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower()); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest)); WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz"); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Get Fundamental for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"js-snapshot-table snapshot-table2 screener_snapshot-table-body\"]"); if(null==nodeCollection || nodeCollection.Count<1)return null; fundamental.AsOf=DateTime.Now; fundamental.Symbol=symbol; fundamental.Source = "FINVIZ"; for(int index=0;index incomeStatements=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual); if(null!=incomeStatements&&0!=incomeStatements.Count) incomeStatement=incomeStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault(); CashflowStatement cashflowStatement=null; List cashflowStatements=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual); if(null != cashflowStatements && 0!=cashflowStatements.Count) { cashflowStatement = cashflowStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault(); } BalanceSheet balanceSheet=null; List balanceSheets=GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual); if(null != balanceSheets && 0!=balanceSheets.Count) { balanceSheet = balanceSheets.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault(); } if(null!=cashflowStatement) { fundamental.OperatingCashflow=cashflowStatement.OperatingCashflow; fundamental.LeveragedFreeCashflow=cashflowStatement.FreeCashflow; fundamental.EBITDA=fundamental.EBIT+cashflowStatement.DepreciationAndAmortization; } else { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO CASHFLOW STATEMENT *************",symbol)); fundamental.OperatingCashflow=double.NaN; fundamental.LeveragedFreeCashflow=double.NaN; fundamental.EBITDA=double.NaN; } if(null!=incomeStatement) { fundamental.GrossProfit=incomeStatement.GrossProfit; fundamental.Revenue=incomeStatement.TotalRevenue; fundamental.EBIT=incomeStatement.EBIT; } else { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO INCOME STATEMENT *************",symbol)); fundamental.GrossProfit=double.NaN; fundamental.Revenue=double.NaN; fundamental.EBIT=double.NaN; } if(null!=balanceSheet) { fundamental.TotalCash=balanceSheet.CashAndCashEquivalents; fundamental.TotalDebt=balanceSheet.TotalLiabilities; } else { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO BALANCESHEET *************",symbol)); fundamental.TotalCash=double.NaN; fundamental.TotalDebt=double.NaN; } if(!double.IsNaN(fundamental.MarketCap)&&!double.IsNaN(fundamental.TotalDebt)&&!double.IsNaN(fundamental.TotalCash)) fundamental.EnterpriseValue=fundamental.MarketCap+fundamental.TotalDebt-fundamental.TotalCash; else fundamental.EnterpriseValue=double.NaN; if(!Double.IsNaN(fundamental.SharesOutstanding)&&0!=fundamental.SharesOutstanding)fundamental.RevenuePerShare=fundamental.Revenue/fundamental.SharesOutstanding; else fundamental.RevenuePerShare=double.NaN; if(null!=balanceSheet&&!double.IsNaN(balanceSheet.TotalStockHolderEquity)&&!double.IsNaN(fundamental.TotalDebt)&&0!=fundamental.TotalDebt)fundamental.DebtToEquity=fundamental.TotalDebt/balanceSheet.TotalStockHolderEquity; else fundamental.DebtToEquity=double.NaN; if(double.IsNaN(fundamental.DebtToEquity)&&!double.IsNaN(fundamental.Equity)&&0.00!=fundamental.Equity)fundamental.DebtToEquity=fundamental.TotalDebt/fundamental.Equity; return fundamental; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}.",exception.ToString())); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();} } } // *************************************************************************************************************************************************************************************** // ************************************************************************** F U N D A M E N T A L - Y A H O O F I N A N C E ********************************************************* // *************************************************************************************************************************************************************************************** public static Fundamental GetFundamental(String symbol) { Fundamental fundamental=GetFundamentalEx(symbol); if(null==fundamental)return fundamental; if(fundamental.GetLoad()<80) { Fundamental fundamental2 = GetFundamentalEx(symbol); if(null!=fundamental2)fundamental.MergeFrom(fundamental2); } return fundamental; } private static Fundamental GetFundamentalEx(String symbol) { HttpNetResponse httpNetResponse=null; int TIMEOUT_MS=1000*30; try { Fundamental fundamental = new Fundamental(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("?p=").Append(symbol); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest)); WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS, webProxy); // user agents refreshed in this version if(!httpNetResponse.Success) { sb=new StringBuilder(); sb.Append("http://finance.yahoo.com/q?s=").Append(symbol).Append("&ql=1"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FAILED!, TRYING {1}",strRequest,sb.ToString())); strRequest = sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FINAL FAILURE.",strRequest)); return null; } } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} SUCCESS.",strRequest)); fundamental.Symbol=symbol; fundamental.AsOf=DateTime.Now; fundamental.Source = "YAHOO"; fundamental.Beta=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Beta<")); fundamental.NextEarningsDate=FeedParser.ParseValueDateTimeMonthFormat(Sections.LocateItemAcceptAnyText(httpNetResponse.ResponseString,">Earnings Date<")); String strText=Sections.LocateFirstItem(httpNetResponse.ResponseString,">52 Week Range<"); if(null!=strText) { String[] strArray = strText.Split('-'); fundamental.Low52 = FeedParser.ParseValue(strArray[0].Trim()); fundamental.High52 = FeedParser.ParseValue(strArray[1].Trim()); } fundamental.Volume=FeedParser.ParseValueLong(Sections.LocateItem(httpNetResponse.ResponseString,">Volume<")); fundamental.MarketCap=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Market Cap<")); fundamental.PE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PE Ratio (TTM)<")); fundamental.EPS=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EPS (TTM)<")); try { Thread.Sleep(250);PopulateKeyStatistics(fundamental,webProxy); } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate key statistics."); MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString()); } try { Thread.Sleep(250);PopulateFinancials(fundamental); } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate financials."); MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString()); } if(fundamental.IsZero()) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Fundamental for {0} is Invalid *****",symbol)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",fundamental.ToString())); return null; } return fundamental; } catch (Exception) { return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // ************************************************************************************************************************************************************************************* // ***************************************************************** K E Y S T A T I S T I C S - Y A H O O F I N A N C E ********************************************************** // ************************************************************************************************************************************************************************************* private static bool PopulateKeyStatistics(Fundamental fundamental,WebProxy webProxy=null) { HttpNetResponse httpNetResponse=null; try { StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("http://finance.yahoo.com/q/ks?s=").Append(fundamental.Symbol); strRequest = sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); if(!httpNetResponse.Success) { try{Thread.Sleep(250);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return false; } } fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<")); fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<")); if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<")); fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<")); if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets (ttm)<")); fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<")); if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity (ttm)<")); fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<")); if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share (ttm)<")); fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<")); if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash (mrq)<")); fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<")); if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt (mrq)<")); fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7)); if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding 6 ")); fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<")); if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue (ttm)<")); fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<")); if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth (yoy)<")); fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<")); if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit (ttm)<")); fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<")); if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA <")); fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<")); if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common (ttm)<")); fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<")); if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share (mrq)<")); fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<")); if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow (ttm)<")); fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<")); if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow (ttm)<")); fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7)); return true; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString())); return false; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } // ************************************************************************************************************************************************************************************* // ***************************************************************** F I N A N C I A L S - Y A H O O F I N A N C E ***************************************************************** // ************************************************************************************************************************************************************************************* private static bool PopulateFinancials(Fundamental fundamental) { HttpNetResponse httpNetResponse=null; try { StringBuilder sb = new StringBuilder(); String strRequest; sb.Append("https://finance.yahoo.com/quote/").Append(fundamental.Symbol).Append("/financials?p=").Append(fundamental.Symbol); strRequest = sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); if(!httpNetResponse.Success) { try{Thread.Sleep(250);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return false; } } fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00; if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00; return true; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString())); return false; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } //************************************************************************************************************************************************************************************** //*************************************************************************** I N T R A D A Y P R I C I N G ************************************************************************** // ************************************************************** M A I N : Y A H O O ************************************************************************************************** // ************************************************************** B A C K U P : B I G C H A R T S ************************************************************************************** //************************************************************************************************************************************************************************************** private delegate Price LatestPriceDelegate(String symbol); public static Price GetLatestPrice(String symbol) { if(null==symbol)return null; CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); if(null!=companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); return null; } DateTime currentMarketDate=PremarketDA.GetLatestMarketDate(); LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceBigCharts,GetLatestPriceGoogle}; Price latestPrice=null; foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates) { latestPrice=latestPriceDelegate.Invoke(symbol); if(null!=latestPrice && latestPrice.IsValid) { if(latestPrice.Date.Date.Equals(currentMarketDate.Date.Date)) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Price request from {0} succeeded for {1}",latestPriceDelegate.GetMethodInfo().Name,symbol)); return latestPrice; } else { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date. Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString())); } } } latestPrice=GetPriceAsOf(symbol,currentMarketDate); // try to get the price from MarketWatch historical feed using todays date if(null!=latestPrice && latestPrice.Date.Date.Equals(currentMarketDate.Date)) return latestPrice; if(null!=companyProfile && companyProfile.CanRollPrevious) { latestPrice=PricingMarketDataHelper.RollPriceForward(symbol); if(null!=latestPrice)return latestPrice; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol)); return null; } // *************************************************************************************************************** // ******************** L A T E S T P R I C E R E T R I E V A L F O R I N T R A - D A Y F E E D ******** // *************************************************************************************************************** public static Price GetLatestPriceYahoo(String symbol) { HttpNetResponse httpNetResponse=null; try { StringBuilder sb=new StringBuilder(); String strRequest; if(null==symbol)return null; sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol); strRequest=sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo"); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } Price price=new Price(); price.Symbol=symbol.ToUpper(); price.Date=DateTime.Now; price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN; price.Source=Price.PriceSource.Yahoo; return BuildPriceFromResponse(httpNetResponse.ResponseString,price); } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally { if(null!=httpNetResponse) httpNetResponse.Dispose(); } } private static Price BuildPriceFromResponse(String httpNetResponse,Price price) { if(httpNetResponse.Contains("HTML_TAG_END")) { price = BuildPriceFromSections(httpNetResponse, price); } else { price = BuildPriceFromTable(httpNetResponse, price); } return price; } private static Price BuildPriceFromSections(String httpNetResponse,Price price) { try { String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END"); List sections = null; sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer"); String marker ="data-symbol=\""+price.Symbol+"\""; List dataSections = sections.Where(x => x.Contains(marker)).ToList(); dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList(); if(dataSections.Count>0) { dataSections = Sections.GetSections(dataSections[0]); if(null!=dataSections && dataSections.Count==1) { price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[0]); } else if(null!=dataSections && dataSections.Count>0) { price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]); } } marker = " x.StartsWith(marker)).ToList(); for(int index=0;index {1}",price.Symbol,exception.ToString())); return null; } } private static Price BuildPriceFromTable(String httpNetResponse,Price price) { MemoryStream memoryStream = null; try { List sections = null; double bid = double.NaN; double bidContracts = double.NaN; double ask = double.NaN; double askContracts = double.NaN; String symbol=price.Symbol.ToUpper(); byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]"); if(null == table || 0 == table.Count)return null; HtmlNodeCollection rows = table[0].SelectNodes(".//tr"); if (null == rows || 0 == rows.Count) return null; for(int index=0;index< rows.Count; index++) { int itemIndex=0; String itemValue=null; String innerHtml = rows[index].InnerHtml; sections=Sections.GetSections(innerHtml); if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; } else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; price.Open=FeedParser.ParseValue(itemValue); } else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; if(String.IsNullOrEmpty(itemValue))continue; String[] bidContractSplit = itemValue.Split('x'); bid=FeedParser.ParseValue(bidContractSplit[0]); bidContracts=FeedParser.ParseValue(bidContractSplit[1]); } else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; if(String.IsNullOrEmpty(itemValue))continue; String[] askContractSplit = itemValue.Split('x'); ask = FeedParser.ParseValue(askContractSplit[0]); askContracts = FeedParser.ParseValue(askContractSplit[1]); } else if(Sections.FindInSections(sections, "Day's Range",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; String[] subItems=itemValue.Split('-'); price.Low=FeedParser.ParseValue(subItems[0]); price.High=FeedParser.ParseValue(subItems[1]); } else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; } else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true)) { itemValue = sections[itemIndex+4]; price.Volume=FeedParser.ParseValueLong(itemValue); } else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true)) { itemValue = sections[itemIndex+3]; } } if (double.IsNaN(price.Close)) { sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer"); String marker = "data-symbol=\"" + price.Symbol + "\""; sections = sections.Where(x => x.Contains(marker)).ToList(); sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList(); if (sections.Count > 0) { sections = Sections.GetSections(sections[0]); if (null != sections && sections.Count > 0) { price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]); } } } if(double.IsNaN(price.Close)) { double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts); if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice; if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00; } CheckPrice(price); return price; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: for {0} -> {1}",price.Symbol,exception.ToString())); return null; } finally { if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); memoryStream=null; } } } private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts) { if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN; double totalContracts = bidContracts + askContracts; double bidWeight = bidContracts/totalContracts; double askWeight = askContracts/totalContracts; double weightedBid = bid*bidWeight; double weightedAsk = ask*askWeight; MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2))); return weightedBid + weightedAsk; } public static void CheckPrice(Price price) { if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low)) { price.Close=price.AdjClose=(price.High+price.Low)/2.00; } if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low) { // if the close price is garbage set it to the midpoint of the high and the low if(price.Closeprice.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00; } if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close; if(double.IsNaN(price.Open))price.Open=0.00; if(double.IsNaN(price.Low))price.Low=0.00; if(double.IsNaN(price.High))price.High=0.00; if(0==price.High && !double.IsNaN(price.Close) && 0!=price.Close) { price.High=price.Close; } if(0==price.Low && !double.IsNaN(price.Close) && 0!=price.Close) { price.Low=price.Close; } if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low) { price.Open=price.High=price.Low=price.Close; } if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low)) { price.Open=(price.High+price.Low)/2.00; } } // This is used in the intra-day price feed. public static Price GetLatestPriceBigCharts(String symbol) { HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; try { StringBuilder sb = new StringBuilder(); String strRequest; if(null==symbol) return null; sb.Append("http://bigcharts.marketwatch.com/quickchart/quickchart.asp?symb=").Append(symbol).Append("&insttype=Stock").Append(symbol); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceBigCharts"); httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } Price price = new Price(); price.Symbol = symbol.ToUpper(); price.Date = DateTime.Now; price.Source=Price.PriceSource.BigCharts; byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quickchart\"]"); if(null==table||0==table.Count)return null; HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"maincontent\"]"); if(null==divSection)return null; HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr"); if(rows.Count<3)return null; HtmlNode softTimeNode = htmlDocument.DocumentNode.SelectSingleNode("//*[@class=\"soft time\"]"); if(null!=softTimeNode) { DateTime givenDate=ConvertBigChartsDate(softTimeNode.InnerHtml); if(!Utility.IsEpoch(givenDate))price.Date=givenDate; } HtmlNodeCollection dataColumns = rows[2].SelectNodes(".//td"); for(int index=0;index divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div"); if(null==divSections || 0==divSections.Count)return null; List searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList(); if(null!=searchPattern && searchPattern.Count>0) { httpNetResponse.Dispose(); sb = new StringBuilder(); sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNYSE); strRequest = sb.ToString(); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div"); searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList(); if(null!=searchPattern && searchPattern.Count>0) { return null; } } List sections = Sections.GetSections(httpNetResponse.ResponseString); sections = Sections.FlattenSection(sections); int index=0; Price price = new Price(); price.Symbol = symbol.ToUpper(); price.Source=Price.PriceSource.Google; price.Close=price.Open=price.High=price.Low=price.AdjClose=double.NaN; price.Open=0.00; price.Date = pricingDate; if(Sections.FindInSections(sections, "After Hours:", 0,ref index, true)) { price.Close=price.AdjClose=FeedParser.ParseValue(sections[index-1]); } else if(Sections.FindInSections(sections, "Share", 0,ref index, true)) { price.Close=price.AdjClose=FeedParser.ParseValue(sections[index+1]); } if(Sections.FindInSections(sections, "Previous close", 0,ref index, true)) { price.Open=FeedParser.ParseValue(sections[index+2]); } if(Sections.FindInSections(sections, "Day range", 0,ref index, true)) { String itemValue = sections[index+2]; String[] highLow = itemValue.Split('-'); price.Low=FeedParser.ParseValue(highLow[0]); price.High=FeedParser.ParseValue(highLow[1]); } if(!price.IsValid)return null; return price; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } /// /// GetLatestPriceBarChart - This is a new feed I am implementing because I am having difficulty retrieving prices from MarketWatch(BigCharts) /// I am planning to use this field initially as a sweep to run after the BigCharts feed and prior to the Yahoo Sweep /// /// /// public static Price GetLatestPriceBarChart(String symbol) { HttpNetResponse httpNetResponse = null; try { DateGenerator dateGenerator = new DateGenerator(); DateTime currentPricingDate = dateGenerator.GetPrevBusinessDay(DateTime.Now); StringBuilder sb = new StringBuilder(); String strRequest; if (null == symbol) return null; sb = new StringBuilder(); sb.Append("https://www.barchart.com/stocks/quotes/").Append(symbol).Append("/overview"); strRequest = sb.ToString(); WebProxy webProxy = HttpNetRequest.GetProxy("GetLatestPriceBarChart"); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetLatestPriceBarChart: Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } String beginMarker = "data-ng-init='init(\"" + symbol + "\""; String endMarker = ">"; String item = Utility.BetweenString(httpNetResponse.ResponseString, beginMarker, endMarker); if (null == item) return null; item = item.Replace(""", "\""); item = item.Replace("{", null); item = item.Replace("}", null); String[] pairs = item.Split(","); Dictionary values = CreateBarChartValues(pairs); Price price = new Price(); price.Symbol = symbol; price.Date = currentPricingDate; price.Source = Price.PriceSource.BarChart; if (values.ContainsKey("lowPrice")) price.Low = values["lowPrice"]; if (values.ContainsKey("highPrice")) price.High = values["highPrice"]; if (values.ContainsKey("openPrice")) price.Open = values["openPrice"]; if (values.ContainsKey("lastPrice")) price.Close = values["lastPrice"]; if (values.ContainsKey("volume")) price.Volume = (long)values["volume"]; if (values.ContainsKey("prevClose")) price.PrevClose = values["prevClose"]; else price.PrevClose = price.Close; price.AdjClose = price.Close; MarketDataHelper.CheckPrice(price); if (double.IsNaN(price.Open) && double.IsNaN(price.High) && double.IsNaN(price.Low) && double.IsNaN(price.Close)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("*** No closing price for {0}", price.Symbol)); return null; } if (double.IsNaN(price.Open)) price.Open = 0.00; if (double.IsNaN(price.Low)) price.Low = 0.00; if (double.IsNaN(price.High)) price.High = 0.00; if (0 != price.Close && 0 == price.Open && 0 == price.High && 0 == price.Low) { price.Open = price.High = price.Low = price.Close; } return price; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString()); return null; } finally { if (null != httpNetResponse) { httpNetResponse.Dispose(); } } } private static Dictionary CreateBarChartValues(String[] pairs) { Dictionary barChartValues = new Dictionary(); if (null == pairs || 0 == pairs.Length) return barChartValues; foreach (String item in pairs) { String[] nameValue = item.Split(":"); if (null == nameValue || 2 != nameValue.Length) continue; String name = nameValue[0].Replace("\"", null); String value = nameValue[1].Replace("\"",null); double result = 0.00; if (double.TryParse(value, out result)) { if (barChartValues.ContainsKey(name)) { barChartValues[name] = result; } else { barChartValues.Add(name, result); } } } return barChartValues; } // ******************************************************************************************************************************************************************************* // ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ******************************************************** // ******************************************************************************************************************************************************************************* /// /// Gets historical pricing from BigCharts /// /// /// Most recent date /// Most historical date /// Prices public static Prices GetPricesAsOf(String symbol, DateTime startDate, DateTime endDate) { Prices prices = new Prices(); if (null == symbol) return null; CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); if (null != companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } DateGenerator dateGenerator = new DateGenerator(); List dates = dateGenerator.GenerateHistoricalDates(startDate, endDate); for (int index = 0; index < dates.Count; index++) { DateTime currentDate = dates[index]; Price price = GetPriceAsOfV2(symbol, currentDate); if (null == price) { price = GetPriceAsOf(symbol, currentDate); if (null == price) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, Utility.DateTimeToStringMMHDDHYYYY(currentDate))); continue; } } prices.Add(price); } return prices; } /// /// The main BigChart Feed which is no longer working /// /// /// /// public static Price GetPriceAsOf(String symbol, DateTime asOf) { HttpNetResponse httpNetResponse = null; try { String strRequest; StringBuilder sb = null; if (null == symbol) return null; CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); if (null != companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } String requestSymbol = symbol; if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); sb.Append(requestSymbol); sb.Append("&closeDate="); sb.Append(asOf.Month.ToString()); sb.Append("%2F"); sb.Append(asOf.Day.ToString()); sb.Append("%2F"); sb.Append((asOf.Year - 2000).ToString()); sb.Append("&x=38&y=25"); strRequest = sb.ToString(); httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); MemoryStream memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]"); if (null == tables || 0 == tables.Count) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); if (rows.Count < 7) return null; Price price = new Price(); price.Source = Price.PriceSource.BigCharts; price.Symbol = symbol.ToUpper(); price.Date = FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText); price.Close = FeedParser.ParseValueFromMarketWatch("Closing Price:", rows[2].InnerText); price.AdjClose = price.Close; price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText); price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText); price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText); price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText); if (!(price.Date.Date.Equals(asOf.Date.Date))) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}", symbol, price.Date.ToShortDateString(), asOf.ToShortDateString())); return null; } return price; } catch (Exception) { return null; } finally { if (null != httpNetResponse) httpNetResponse.Dispose(); } } /// /// This is a modified version of the above query. It uses a cookie collection that contains the datadome cookie. /// If this query starts to fail then you should try the query in developer tools in the chromium browser and capture the /// datadome value and update it here in the GetCookieCollection() below. /// /// /// /// public static Price GetPriceAsOfV2(String symbol, DateTime asOf) { HttpNetResponse httpNetResponse = null; try { String strRequest; StringBuilder sb = null; if (null == symbol) return null; CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); if (null != companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } String requestSymbol = symbol; if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1); sb = new StringBuilder(); sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb="); sb.Append(requestSymbol); sb.Append("&closeDate="); sb.Append(asOf.Month.ToString()); sb.Append("%2F"); sb.Append(asOf.Day.ToString()); sb.Append("%2F"); sb.Append((asOf.Year - 2000).ToString()); sb.Append("&x=38&y=25"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,$"{strRequest}"); CookieCollection cookieCollection = GetCookieCollection("www.marketwatch.com"); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV2(strRequest,cookieCollection,new Uri("https://www.marketwatch.com/investing")); if (!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); MemoryStream memoryStream = new MemoryStream(streamBytes); HtmlDocument htmlDocument = new HtmlDocument(); htmlDocument.Load(memoryStream); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table table--overflow align--center\"]"); if (null == tables || 0 == tables.Count) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); if (rows.Count < 2) return null; Prices prices = new Prices(); for (int rowIndex = 1; rowIndex < rows.Count; rowIndex++) { HtmlNodeCollection data = rows[rowIndex].SelectNodes(".//td"); if (data.Count != 6) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("The price retrieved for {0} does not contain the correct number of data elements (expected 6 data elements).", symbol)); return null; } Price price = new Price(); price.Source = Price.PriceSource.BigCharts; price.Symbol = symbol.ToUpper(); String[] dateElements = data[0].InnerText.Trim().Split(" "); price.Date = Utility.ParseDate(dateElements[0].Replace("\n", null)); price.Open = Utility.ParseCurrency(data[1].InnerText.Trim().Replace("\n", null)); price.High = Utility.ParseCurrency(data[2].InnerText.Trim().Replace("\n", null)); price.Low = Utility.ParseCurrency(data[3].InnerText.Trim().Replace("\n", null)); price.Close = price.AdjClose = Utility.ParseCurrency(data[4].InnerText.Trim().Replace("\n", null)); price.Volume = FeedParser.ParseValueLong(data[5].InnerText.Trim()); prices.Add(price); } Price selectedPrice = prices.Where(x => x.Date.Date.Equals(asOf.Date.Date)).FirstOrDefault(); if (null == selectedPrice) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("No price for {0} on {1}", symbol, asOf.ToShortDateString())); return null; } return selectedPrice; } catch (Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString())); return null; } finally { if (null != httpNetResponse) httpNetResponse.Dispose(); } } private static CookieCollection GetCookieCollection(String cookieDomain) { String[] cookies = { "datadome=VLSWQ1PWL4SV2rljkJEV7GUOK3T11Sm73RP17IwDEtRzfWHykYwy7ZoeSPrjwmbwvGRAenazmLVTpyCB6Yqlw6vUL6Wbqq4zML5DGuHLxNU4LgAQ7Ko9tztglqjIGLZE", }; CookieCollection cookieCollection = new CookieCollection(); for (int index = 0; index < cookies.Count(); index++) { String strCookie = cookies[index]; if (strCookie.EndsWith(";")) strCookie = strCookie.Substring(0, strCookie.Length - 1); String[] pairs = strCookie.Split('='); cookieCollection.Add(new Cookie() { Name = pairs[0], Value = pairs[1], Domain = cookieDomain }); } return cookieCollection; } //******************************************************************************************************************************************************************************************************** // ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O ********************************************************************* //******************************************************************************************************************************************************************************************************** public static Price GetDailyPrice(String symbol, DateTime pricingDate) { if (null == symbol) return null; CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol); if (null != companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol)); return null; } Prices prices = GetDailyPrices(symbol, pricingDate, pricingDate); if (null == prices || 0 == prices.Count) return null; Price price = prices.FirstOrDefault(); if (!price.Date.Date.Equals(pricingDate.Date)) { MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'", symbol, pricingDate.ToShortDateString(), price.Date.Date.ToShortDateString())); return null; } return price; } public static Prices GetDailyPrices(String symbol, DateTime startDate, DateTime endDate) { HttpNetResponse httpNetResponse=null; try { if(symbol==null)return null; CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol); if(null!=companyProfile && companyProfile.FreezePricing) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol)); return null; } startDate=startDate.Date; endDate=endDate.Date; if(startDate>endDate) { DateTime tempDate=startDate; startDate=endDate; endDate=tempDate; } MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[MarketDataHelper:GetDailyPrices]{0} start:{1} end:{2}",symbol,Utility.DateTimeToStringMMHDDHYYYY(startDate),Utility.DateTimeToStringMMHDDHYYYY(endDate))); StringBuilder sb = new StringBuilder(); String strRequest; Prices prices=null; sb.Append("https://query1.finance.yahoo.com/v7/finance/chart/").Append(symbol).Append("?period1=").Append(Utility.DateToUnixDate(startDate)).Append("&period2=").Append(Utility.DateToUnixDate(endDate)).Append("&interval=1d&events=history"); strRequest=sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Query:{0}",strRequest)); WebProxy webProxy=HttpNetRequest.GetProxy("GetDailyPrices"); CookieCollection cookieCollection=new CookieCollection(); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3B(strRequest,cookieCollection,webProxy); if(!httpNetResponse.Success) return null; JObject json=JObject.Parse(httpNetResponse.ResponseString); prices=GetPricesFromJSONString(json,symbol); return prices; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString())); return null; } finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); } } private static Prices GetPricesFromJSONString(JObject json,String symbol) { Prices prices=new Prices(); try { JToken token=null; int items=0; try{token=json["chart"]["result"][0]["timestamp"];}catch(Exception){;} if(null==token) { string instrumentType=null; try{instrumentType=json["chart"]["result"][0]["meta"]["instrumentType"].ToString();}catch(Exception){;} if(null==instrumentType||!instrumentType.Equals("MUTUALFUND"))return prices; String strPrice=null; String strDate=null; try{strPrice=json["chart"]["result"][0]["meta"]["regularMarketPrice"].ToString();}catch(Exception){;} try{strDate=json["chart"]["result"][0]["meta"]["currentTradingPeriod"]["regular"]["start"].ToString();}catch(Exception){;} if(null==strPrice||null==strDate)return prices; Price price=new Price(); price.Symbol=symbol; price.Source=Price.PriceSource.Yahoo; price.High=price.Low=price.Close=price.Open=price.AdjClose=FeedParser.ParseValue(strPrice); DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strDate)); price.Date=dateTimeOffset.DateTime; price.Volume=0L; prices.Add(price); return prices; } try{items=token.Children().Count();}catch(Exception){;} if(0==items)return prices; for(int index=0;indexprice.High)price.Close=(price.High+price.Low)/2.00; prices.Add(price); } return prices; } catch(Exception exception) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString())); return prices; } } // ****************************************************************************************************************************************************************************************************************** // *************************************************************************** E N D Y A H O O H I S T O R I C A L P R I C I N G ******************************************************************************* // ****************************************************************************************************************************************************************************************************************** // **************************************************************************************************************************************************************************************** // **************************************************************************** E N D D A T A F E E D S ********************************************************************************* // **************************************************************************************************************************************************************************************** // *************************************************************************************************************************************************************************************** // *************************************************************************************************************************************************************************************** // *************************************************************************************************************************************************************************************** // 2017 Yahoo Finance Modified Format of all feeds. Most of the HTMLDocument parser code is broken because the response strings are no longer in tabular format. // It's difficult to make sense of the new response string outside of noting that it remains in aa tag format. The following code is an attempt to make sense of it all. public static List LocateJSONItems(String strInput,String item,bool isHeading=false) { try { if(null==item||null==strInput)return null; int startPos=strInput.IndexOf(item); if(-1==startPos)return null; strInput=strInput.Substring(startPos-1); if(isHeading) { strInput=strInput.Substring(item.Length); startPos=strInput.IndexOf(item); if(-1==startPos)return null; strInput=strInput.Substring(startPos-1); } strInput=Utility.BetweenString(strInput,"[","]"); if(null==strInput)return null; String[] itemsArray=strInput.Split(','); return itemsArray.ToList(); } catch(Exception) { return null; } } public static List LocateJSONKeyValuePairs(String strInput, String item1,String item2) { try { List items=new List(); if (null == item1 || null == strInput) return null; int startPos = strInput.IndexOf(item1); while(true) { KeyValue keyValue=new KeyValue(); if (-1 == startPos||null==strInput) break; String strItem = strInput.Substring(startPos); strItem = Utility.BetweenString(strItem, "\"", "\""); String strValue = strInput.Substring((startPos+strItem.Length+4)-1); strValue = Utility.BetweenString(strValue, "\"", "\""); keyValue.Key=strValue; try{strInput=strInput.Substring((startPos+strItem.Length+4+strValue.Length+4)-1);} catch(Exception){break;} if(null==strInput)break; startPos = strInput.IndexOf(item2); if(-1==startPos)break; strItem = strInput.Substring(startPos); strItem = Utility.BetweenString(strItem, "\"", "\""); strValue = strInput.Substring((startPos + strItem.Length + 4) - 1); strValue = Utility.BetweenString(strValue, "\"", "\""); keyValue.Value=strValue; try { strInput = strInput.Substring((startPos + strValue.Length + 4) - 1); } catch (Exception) { break; } if(null==strInput)break; startPos = strInput.IndexOf(item1); items.Add(keyValue); } return items; } catch (Exception) { return null; } } public static List> LocateJSONItems(String strInput) { List> items=new List>(); try { if(null==strInput)return null; strInput=Utility.BetweenString(strInput,"[","]"); if(null==strInput)return null; String[] itemsArray=strInput.Split(','); if(null==itemsArray||0==itemsArray.Length)return null; for(int index=0;index> LocateJSONItemsTags(String strInput) { List> items=new List>(); try { if(null==strInput)return null; String[] itemsArray=strInput.Split(','); if(null==itemsArray||0==itemsArray.Length)return null; for(int index=0;indexx.Equals(items[0]))) { DateTime now=DateTime.Now; StringBuilder sb=new StringBuilder(); sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year); referenceDate=Utility.ParseDate(sb.ToString()); if(referenceDate>now) { sb=new StringBuilder(); sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year-1); referenceDate=Utility.ParseDate(sb.ToString()); } } else referenceDate=Utility.Epoch; if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch; return referenceDate; } // **************************************************************************************************************************************************************************************** // **************************************************************************** H E L P E R M E T H O D S ******************************************************************************* // **************************************************************************************************************************************************************************************** private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName) { StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt"); for (int row = 0; row < rows.Count; row++) { streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml); } streamWriter.Flush(); streamWriter.Close(); } private static void WriteToDisk(String responseString, String strPathFileName) { StreamWriter streamWriter = File.CreateText(strPathFileName); streamWriter.WriteLine(responseString); streamWriter.Flush(); streamWriter.Close(); } private static void WriteToDisk(List sections, String strPathFileName) { StreamWriter streamWriter = File.CreateText(strPathFileName); for(int index=0;index