diff --git a/App.config b/App.config index 666005c..121871a 100644 --- a/App.config +++ b/App.config @@ -18,5 +18,7 @@ + + diff --git a/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs b/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs index dd5dd4c..63a5560 100644 --- a/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs +++ b/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using MarketData.MarketDataModel; using MarketData.DataAccess; @@ -11,8 +10,8 @@ namespace MarketData.Helper { public class HeadlinesMarketDataHelper { - private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads; - private static int WAIT_BETWEEN_REQUESTS_MS = 1000; // wait 1000 ms between requests + private static int MaxThreads = 5; // (int)ThreadHelperEnum.MaxThreads; + private static int WAIT_BETWEEN_REQUESTS_MS = 2000; // wait ms between requests private List symbols; private int currentIndex = 0; @@ -134,23 +133,6 @@ namespace MarketData.Helper marketDate=new DateTime(marketDate.Year,marketDate.Month,marketDate.Day,23,59,59); Headlines headlines=null; -// SEEKING ALPHA - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol)); - headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol); - if(headlines.IsNullOrEmpty()) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol)); - } - else - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol)); - headlines=new Headlines(headlines.Where(x=>x.Date {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry)); - } - } // NASDAQ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesNASDAQ {0}",symbol)); headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol); @@ -185,6 +167,23 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("MARKETWATCH: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry)); } } +// SEEKING ALPHA + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol)); + headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol); + if(headlines.IsNullOrEmpty()) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol)); + } + else + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol)); + headlines=new Headlines(headlines.Where(x=>x.Date {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry)); + } + } return; } } diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index f2861b9..4d46c20 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1655,6 +1655,7 @@ namespace MarketData.Helper if(null!=httpNetResponse)httpNetResponse.Dispose(); } } + // *************************************************************************************************************************************************************************** //***************************************************************** H E A D L I N E S - S E E K I N G A L P H A *********************************************************** // *************************************************************************************************************************************************************************** @@ -1665,7 +1666,7 @@ namespace MarketData.Helper return headlines; } - private static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol) + public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); @@ -1675,14 +1676,25 @@ namespace MarketData.Helper String strRequest; symbol = symbol.ToUpper(); - CookieCollection cookieCollection=new CookieCollection(); - httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.seekingalpha.com",cookieCollection); - Thread.Sleep(250); + #region Create Cookies + + StringBuilder lastVisitedPage = new StringBuilder(); + lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); + lastVisitedPage.Append(symbol); + lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); + + String domain = "seekingalpha.com"; + CookieCollection cookieCollection = new CookieCollection(); + cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); + cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); + + #endregion + sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, strRequest); - WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,cookieCollection); + WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1"); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1707,7 +1719,7 @@ namespace MarketData.Helper } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,exception); + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally @@ -1716,7 +1728,7 @@ namespace MarketData.Helper } } - private static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol) + public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); @@ -1730,16 +1742,29 @@ namespace MarketData.Helper DateTime marketDate=PremarketDA.GetLatestMarketDate(); if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; + #region Create Cookies - sb.Append("https://seekingalpha.com/symbol/").Append(symbol).Append("/news?from="); + StringBuilder lastVisitedPage = new StringBuilder(); + lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); + lastVisitedPage.Append(symbol); + lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); + + String domain = "seekingalpha.com"; + CookieCollection cookieCollection = new CookieCollection(); + cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); + cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); + + #endregion + + sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from="); sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); sb.Append("T04%3A00%3A00.000Z&to="); sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); sb.Append("T14%3A20%3A34.999Z"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, strRequest); - WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,null); + WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2"); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1764,7 +1789,7 @@ namespace MarketData.Helper } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,exception); + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally @@ -1861,7 +1886,7 @@ namespace MarketData.Helper } catch (Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,exception); + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally @@ -1913,12 +1938,11 @@ namespace MarketData.Helper headline.Entry=Utility.RemoveHtml(headline.Entry); headlines.Add(headline); } -// headlines=new Headlines(headlines.GroupBy(x=>x.Entry).Select(y=>y.First()).ToList()); return headlines; } catch(Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,exception); + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); return null; } finally @@ -4547,9 +4571,15 @@ namespace MarketData.Helper public static Fundamental GetFundamental(String symbol) { Fundamental fundamental=GetFundamentalEx(symbol); + if(null==fundamental)return fundamental; + if(fundamental.GetLoad()<80) + { + Fundamental fundamental2 = GetFundamentalEx(symbol); + if(null!=fundamental2)fundamental.MergeFrom(fundamental2); + } return fundamental; } - public static Fundamental GetFundamentalEx(String symbol) + private static Fundamental GetFundamentalEx(String symbol) { HttpNetResponse httpNetResponse=null; int TIMEOUT_MS=1000*30; @@ -4641,26 +4671,63 @@ namespace MarketData.Helper httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - return false; + try{Thread.Sleep(250);}catch(Exception){;} + httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return false; + } } fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<")); + fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<")); + if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<")); + fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<")); + if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets (ttm)<")); + fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<")); - fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<")); - fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<")); - fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7)); - fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<")); + if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity (ttm)<")); + fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<")); + if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share (ttm)<")); + + fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<")); + if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash (mrq)<")); + + fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<")); + if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt (mrq)<")); + + fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7)); + if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding 6 ")); + + fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<")); + if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue (ttm)<")); + fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<")); + if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth (yoy)<")); + fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<")); + if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit (ttm)<")); + fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<")); + if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA <")); + fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<")); + if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common (ttm)<")); + fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<")); + if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share (mrq)<")); + fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<")); + if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow (ttm)<")); + fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<")); + if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow (ttm)<")); + fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7)); + return true; } catch(Exception exception) @@ -4688,10 +4755,17 @@ namespace MarketData.Helper httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - return false; + try{Thread.Sleep(250);}catch(Exception){;} + httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest); + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return false; + } } fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00; + if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00; + return true; } catch(Exception exception) @@ -5032,6 +5106,10 @@ namespace MarketData.Helper { price.Open=price.High=price.Low=price.Close; } + if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low)) + { + price.Open=(price.High+price.Low)/2.00; + } } // This is used in the intra-day price feed. diff --git a/MarketDataLib/Integration/HttpNetRequest.cs b/MarketDataLib/Integration/HttpNetRequest.cs index c9b6acf..3c4d0c7 100644 --- a/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketDataLib/Integration/HttpNetRequest.cs @@ -958,7 +958,7 @@ namespace MarketData.Integration "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0" }; - MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5A[ENTER]{0}", strRequest)); int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); @@ -1006,7 +1006,7 @@ namespace MarketData.Integration } finally { - MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5A[LEAVE]"); } } @@ -1015,7 +1015,7 @@ namespace MarketData.Integration HttpWebResponse webResponse = null; try { - MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5B[ENTER]{0}", strRequest)); int charCount = 0; byte[] buffer = new byte[8192]; StringBuilder sb = new StringBuilder(); @@ -1057,7 +1057,102 @@ namespace MarketData.Integration } finally { - MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5B[LEAVE]"); + } + } + +// This accepts gzip and deflate which seems to work for the seeking alpha news retrieval + public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest)); + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Host = host; + webRequest.Headers.Add("Sec-GPC: 1"); + webRequest.Headers.Add("Sec-Fetch-Dest: document"); + webRequest.Headers.Add("Sec-Fetch-Mode: navigate"); + webRequest.Headers.Add("Sec-Fetch-Site: none"); + webRequest.Headers.Add("Sec-Fetch-User: ?1"); + webRequest.Headers.Add("DNT: 1"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.Headers.Add("TE: trailers"); +// webRequest.Headers.Add("If-None-Match: W/\"135dac681d6a78233fc3539ece5ea75a\""); + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"; + if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent(); + else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse); + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return httpNetResponse; + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]"); + } + } + + private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse) + { + try + { + Stream responseStream = webResponse.GetResponseStream(); + StringBuilder sb = new StringBuilder(); + int charCount = 0; + byte[] buffer = new byte[8192]; + + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if(webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + } + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); } } diff --git a/MarketDataLib/MarketDataModel/Fundamentals.cs b/MarketDataLib/MarketDataModel/Fundamentals.cs index a6a4856..fb1e140 100644 --- a/MarketDataLib/MarketDataModel/Fundamentals.cs +++ b/MarketDataLib/MarketDataModel/Fundamentals.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net.Http.Headers; using System.Text; using MarketData.Utils; @@ -203,7 +204,7 @@ namespace MarketData.MarketDataModel get{return debtToEquity;} set{debtToEquity=value;} } -// if columns are added to this object must ensure that the columns are also added to the MergeFrom and IsZero methods so that the consistency checks can still function properly +// If columns are added to this object must ensure that the columns are also added to the MergeFrom, IsZero, and GetLoad() methods so that the consistency checks can still function properly // Do not include a comparison of the AsOf date. The prior fundamental will always have an AsOf date that precedes the one that we fetch. // In contrast, the AsOf date in the financial statements is the statement date and we use the AsOf date as a comparison there. // The idea here is to ensure that we do not lose any fidelity of the fundamental data if elements are missing in the current fetch but were present in a previous fetch. @@ -279,6 +280,45 @@ namespace MarketData.MarketDataModel if(percentMissing>85.00)return true; return false; } + + public double GetLoad() + { + double missingItemCount=0.00; + double totalItems=29.00; + double percentMissing=0.00; + if(Utility.IsZeroOrNaN(Beta))missingItemCount++; + if(Utility.IsZeroOrNaN(Low52))missingItemCount++; + if(Utility.IsZeroOrNaN(High52))missingItemCount++; + if(Utility.IsZeroOrNaN((double)Volume))missingItemCount++; + if(Utility.IsZeroOrNaN(MarketCap))missingItemCount++; + if(Utility.IsZeroOrNaN(PE))missingItemCount++; + if(Utility.IsZeroOrNaN(EPS))missingItemCount++; + if(Utility.IsZeroOrNaN(PEG))missingItemCount++; + if(Utility.IsZeroOrNaN(ReturnOnAssets))missingItemCount++; + if(Utility.IsZeroOrNaN(ReturnOnEquity))missingItemCount++; + if(Utility.IsZeroOrNaN(TotalCash))missingItemCount++; + if(Utility.IsZeroOrNaN(TotalDebt))missingItemCount++; + if(Utility.IsZeroOrNaN(SharesOutstanding))missingItemCount++; + if(Utility.IsZeroOrNaN(Revenue))missingItemCount++; + if(Utility.IsZeroOrNaN(SharesOutstanding))missingItemCount++; + if(Utility.IsZeroOrNaN(Revenue))missingItemCount++; + if(Utility.IsZeroOrNaN(RevenuePerShare))missingItemCount++; + if(Utility.IsZeroOrNaN(QtrlyRevenueGrowth))missingItemCount++; + if(Utility.IsZeroOrNaN(GrossProfit))missingItemCount++; + if(Utility.IsZeroOrNaN(EBITDA))missingItemCount++; + if(Utility.IsZeroOrNaN(NetIncomeAvailableToCommon))missingItemCount++; + if(Utility.IsZeroOrNaN(BookValuePerShare))missingItemCount++; + if(Utility.IsZeroOrNaN(OperatingCashflow))missingItemCount++; + if(Utility.IsZeroOrNaN(LeveragedFreeCashflow))missingItemCount++; + if(Utility.IsZeroOrNaN(Equity))missingItemCount++; + if(Utility.IsZeroOrNaN(TrailingPE))missingItemCount++; + if(Utility.IsZeroOrNaN(EnterpriseValue))missingItemCount++; + if(Utility.IsZeroOrNaN(EBIT))missingItemCount++; + if(Utility.IsZeroOrNaN(DebtToEquity))missingItemCount++; + percentMissing=(missingItemCount/totalItems)*100.00; + return 100.00-percentMissing; + } + public static String Header { get diff --git a/MarketDataLib/Utility/FeedParser.cs b/MarketDataLib/Utility/FeedParser.cs index c0bee2f..66a6fc9 100644 --- a/MarketDataLib/Utility/FeedParser.cs +++ b/MarketDataLib/Utility/FeedParser.cs @@ -158,7 +158,45 @@ namespace MarketData.Utils return DateTime.Parse("01-01-0001"); } } -// Sep. 25, 2022 at 4:31 p.m. ET + // "Apr. 04" + // "1:00pm" + public static DateTime ParseValueDateTimeMonth(String strText) + { + try + { + if(null==strText)return DateTime.Parse("01-01-0001"); + + if(strText.EndsWith("am") || strText.EndsWith("pm")) + { + return DateTime.Now; + } + else if(strText.Contains(".")) + { + strText=strText.Replace(".",null); + String[] subItems=strText.Split(' '); + DateTime currentDate = DateTime.Now; + StringBuilder sb=new StringBuilder(); + sb.Append(subItems[0]).Append(" "); + sb.Append(subItems[1]).Append(",").Append(" "); + sb.Append(currentDate.Year.ToString()); + DateTime resultingDate = ParseValueDateTimeMonthFormat(sb.ToString()); + if(resultingDate>currentDate)resultingDate = new DateTime(resultingDate.Year-1,resultingDate.Month, resultingDate.Day); + return resultingDate; + } + else + { + return Utility.ParseDate(strText); + } + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTimeMonth] Error parsing date '" + strText + "', " + exception.ToString()); + return DateTime.Parse("01-01-0001"); + } + } + + + // Sep. 25, 2022 at 4:31 p.m. ET public static DateTime ParseValueDateTimeMonthFormatTZ(String strText) { try @@ -177,7 +215,6 @@ namespace MarketData.Utils } } - public static DateTime? ParseRelativeDate(String strDate) { try diff --git a/MarketDataLib/Utility/Sections.cs b/MarketDataLib/Utility/Sections.cs index 8555ade..2e0aa2e 100644 --- a/MarketDataLib/Utility/Sections.cs +++ b/MarketDataLib/Utility/Sections.cs @@ -265,7 +265,7 @@ namespace MarketDataLib.Utility { String sectionItem=sections[startIndex]; if(item.Contains(sectionItem))continue; - if("".Equals(sectionItem)||"-".Equals(sectionItem)||sectionItem.StartsWith("("))continue; + if("".Equals(sectionItem)||"-".Equals(sectionItem)||"--".Equals(sectionItem)||sectionItem.StartsWith("("))continue; if((sectionItem.All(Char.IsLetter)||sectionItem.Contains(" ")))break; strItem=sectionItem; break; diff --git a/MarketDataLib/Utility/Utility.cs b/MarketDataLib/Utility/Utility.cs index 160983c..5d9d039 100644 --- a/MarketDataLib/Utility/Utility.cs +++ b/MarketDataLib/Utility/Utility.cs @@ -48,6 +48,19 @@ namespace MarketData.Utils return "Unknown"; } } + /// + /// poses a question to the console and receives a confirmation response + /// + /// The message to ask the user. + public static bool GetVerificationToProceed(String message) + { + Console.Write(String.Format("{0} Y/N? ",message)); + String answer = Console.ReadLine(); + answer=answer.ToUpper(); + if(!"Y".Equals(answer))return false; + return true; + } + public static long DateToUnixDate(DateTime dateTime) { DateTime javascriptEpoch=DateTime.Parse("01-01-1970 00:00:00"); @@ -432,7 +445,7 @@ namespace MarketData.Utils public static DateTime ParseDate(String strDate) { System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US"); - String[] formats=new[] { "yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray(); + String[] formats=new[] { "yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray(); strDate = strDate.Trim(); DateTime dateTime=DateTime.ParseExact(strDate, formats, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal); return dateTime; diff --git a/MarketDataUnitTests/App.config b/MarketDataUnitTests/App.config index 992ee85..837353a 100644 --- a/MarketDataUnitTests/App.config +++ b/MarketDataUnitTests/App.config @@ -18,5 +18,7 @@ + + diff --git a/MarketDataUnitTests/MarketDataFeedTests.cs b/MarketDataUnitTests/MarketDataFeedTests.cs index 506222e..e891db7 100644 --- a/MarketDataUnitTests/MarketDataFeedTests.cs +++ b/MarketDataUnitTests/MarketDataFeedTests.cs @@ -104,7 +104,6 @@ namespace MarketDataUnitTests Assert.IsTrue(null!=priceIndices && priceIndices.Count>0); } - // Yahoo Fundamental feed is very poor quality and lots of misses. It's a last resort. [TestMethod] public void FundamentalYahooRetrieval() @@ -120,14 +119,14 @@ namespace MarketDataUnitTests // Assert.IsTrue(!double.IsNaN(fundamental.MarketCap),"MarketCap"); Assert.IsTrue(!double.IsNaN(fundamental.PE),"PE"); Assert.IsTrue(!double.IsNaN(fundamental.EPS),"EPS"); -// Assert.IsTrue(!double.IsNaN(fundamental.PEG),"PEG"); -// Assert.IsTrue(!double.IsNaN(fundamental.ReturnOnAssets),"ReturnOnAssets"); -// Assert.IsTrue(!double.IsNaN(fundamental.ReturnOnEquity),"ReturnOnEquity"); -// Assert.IsTrue(!double.IsNaN(fundamental.TotalCash),"TotalCash"); -// Assert.IsTrue(!double.IsNaN(fundamental.TotalDebt),"TotalDebt"); -// Assert.IsTrue(!double.IsNaN(fundamental.SharesOutstanding),"SharesOutstanding"); -// Assert.IsTrue(!double.IsNaN(fundamental.Revenue),"Revenue"); -// Assert.IsTrue(!double.IsNaN(fundamental.RevenuePerShare),"RevenuePerShare"); + Assert.IsTrue(!double.IsNaN(fundamental.PEG),"PEG"); + Assert.IsTrue(!double.IsNaN(fundamental.ReturnOnAssets),"ReturnOnAssets"); + Assert.IsTrue(!double.IsNaN(fundamental.ReturnOnEquity),"ReturnOnEquity"); + Assert.IsTrue(!double.IsNaN(fundamental.TotalCash),"TotalCash"); + Assert.IsTrue(!double.IsNaN(fundamental.TotalDebt),"TotalDebt"); + Assert.IsTrue(!double.IsNaN(fundamental.SharesOutstanding),"SharesOutstanding"); + Assert.IsTrue(!double.IsNaN(fundamental.Revenue),"Revenue"); + Assert.IsTrue(!double.IsNaN(fundamental.RevenuePerShare),"RevenuePerShare"); Assert.IsTrue(!double.IsNaN(fundamental.QtrlyRevenueGrowth),"QtrlyRevenueGrowth"); // Assert.IsTrue(!double.IsNaN(fundamental.GrossProfit),"GrossProfit"); Assert.IsTrue(!double.IsNaN(fundamental.EBITDA),"EBITDA"); @@ -138,7 +137,7 @@ namespace MarketDataUnitTests Assert.IsTrue(!double.IsNaN(fundamental.Equity),"Equity"); Assert.IsTrue(!double.IsNaN(fundamental.TrailingPE),"TrailingPE"); Assert.IsTrue(!double.IsNaN(fundamental.EnterpriseValue),"EnterpriseValue"); -// Assert.IsTrue(!double.IsNaN(fundamental.EBIT),"EBIT"); + Assert.IsTrue(!double.IsNaN(fundamental.EBIT),"EBIT"); Assert.IsTrue(!double.IsNaN(fundamental.DebtToEquity),"DebtToEquity"); } @@ -292,11 +291,27 @@ namespace MarketDataUnitTests [TestMethod] public void HeadlinesSeekingAlphaRetrieval() { - String symbol="AAPL"; + String symbol="ALPN"; Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol); Assert.IsTrue(null!=companyHeadlines && companyHeadlines.Count>0); } + [TestMethod] + public void HeadlinesSeekingAlphaV1Retrieval() + { + String symbol="AAPL"; + Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV1(symbol); + Assert.IsTrue(null!=companyHeadlines && companyHeadlines.Count>0); + } + + [TestMethod] + public void HeadlinesSeekingAlphaV2Retrieval() + { + String symbol="AAPL"; + Headlines companyHeadlines = MarketDataHelper.GetCompanyHeadlinesSeekingAlphaV2(symbol); + Assert.IsTrue(null!=companyHeadlines && companyHeadlines.Count>0); + } + [TestMethod] public void AnalystPriceTargetMarketBeatRetrieval() { diff --git a/Program.cs b/Program.cs index 19f8a81..a2f1f02 100644 --- a/Program.cs +++ b/Program.cs @@ -2414,8 +2414,14 @@ namespace MarketData DateTime startDate =Constants.MIN_PRICING_DATE; DateTime endDate = DateTime.Now; Prices prices=MarketDataHelper.GetDailyPrices(symbol,startDate,endDate); // use the Yahoo JSON bulk feed - //if(null==prices||0==prices.Count)prices=MarketDataHelper.GetPricesAsOf(symbol,startDate,endDate); - if (null == prices || 0==prices.Count) // if less than 252 days of prices then forget it + if(null==prices||0==prices.Count) + { + if(!Utility.GetVerificationToProceed(String.Format("No prices from Yahoo for symbol{0}. Would you like to try BigCharts?",symbol))) + { + return; + } + } + if (null == prices || 0==prices.Count) { MDTrace.WriteLine(LogLevel.DEBUG,"No prices for '" + symbol + "'"); return;