Fix Yahoo Fundamental retrieval.

Fix SeekingAlpha news retrieval.
2024-04-12 12:40:41 -04:00
parent 799356533a
commit c0c1d37bf0
11 changed files with 353 additions and 66 deletions
--- a/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs
+++ b/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs
@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using MarketData.MarketDataModel;
 using MarketData.DataAccess;
@@ -11,8 +10,8 @@ namespace MarketData.Helper
 {
  public class HeadlinesMarketDataHelper
  {
-    private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
-    private static int WAIT_BETWEEN_REQUESTS_MS = 1000;   // wait 1000 ms between requests
+    private static int MaxThreads = 5; // (int)ThreadHelperEnum.MaxThreads;
+    private static int WAIT_BETWEEN_REQUESTS_MS = 2000;   // wait ms between requests
    private List<String> symbols;
    private int currentIndex = 0;

@@ -134,23 +133,6 @@ namespace MarketData.Helper
      marketDate=new DateTime(marketDate.Year,marketDate.Month,marketDate.Day,23,59,59);
     
      Headlines headlines=null;
-// SEEKING ALPHA
-      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol));
-      headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
-      if(headlines.IsNullOrEmpty())
-      {
-        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
-      }
-      else
-      {
-        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
-        headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
-        HeadlinesDA.InsertHeadlines(headlines);
-        foreach(Headline headline in headlines)
-        {
-          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
-        }
-      }
 // NASDAQ
      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesNASDAQ {0}",symbol));
      headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
@@ -185,6 +167,23 @@ namespace MarketData.Helper
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("MARKETWATCH: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
        }
      }
+// SEEKING ALPHA
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol));
+      headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
+      if(headlines.IsNullOrEmpty())
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
+      }
+      else
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
+        headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
+        HeadlinesDA.InsertHeadlines(headlines);
+        foreach(Headline headline in headlines)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
+        }
+      }
      return;
    }
  }
--- a/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketDataLib/Helper/MarketDataHelper.cs
@@ -1655,6 +1655,7 @@ namespace MarketData.Helper
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
+
 // ***************************************************************************************************************************************************************************
 //***************************************************************** H E A D L I N E S  - S E E K I N G   A L P H A ***********************************************************
 // ***************************************************************************************************************************************************************************
@@ -1665,7 +1666,7 @@ namespace MarketData.Helper
      return headlines;
    }

-    private static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
+    public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      Headlines headlines=new Headlines();
@@ -1675,14 +1676,25 @@ namespace MarketData.Helper
        String strRequest;
        symbol = symbol.ToUpper();

-        CookieCollection cookieCollection=new CookieCollection();
-        httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.seekingalpha.com",cookieCollection);
-        Thread.Sleep(250);
+        #region Create Cookies
+
+        StringBuilder lastVisitedPage = new StringBuilder();
+        lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
+        lastVisitedPage.Append(symbol);
+        lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
+
+        String domain = "seekingalpha.com";
+        CookieCollection cookieCollection = new CookieCollection();
+        cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
+        cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
+
+        #endregion
+
        sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
-        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,cookieCollection);
+        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
+        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1707,7 +1719,7 @@ namespace MarketData.Helper
      }
      catch (Exception exception)
      {
-        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
@@ -1716,7 +1728,7 @@ namespace MarketData.Helper
      }
    }

-    private static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
+    public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      Headlines headlines=new Headlines();
@@ -1730,16 +1742,29 @@ namespace MarketData.Helper
        DateTime marketDate=PremarketDA.GetLatestMarketDate();
        if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;

+        #region Create Cookies

-        sb.Append("https://seekingalpha.com/symbol/").Append(symbol).Append("/news?from=");
+        StringBuilder lastVisitedPage = new StringBuilder();
+        lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
+        lastVisitedPage.Append(symbol);
+        lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
+
+        String domain = "seekingalpha.com";
+        CookieCollection cookieCollection = new CookieCollection();
+        cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
+        cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
+
+        #endregion
+
+        sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
        sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
        sb.Append("T04%3A00%3A00.000Z&to=");
        sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
        sb.Append("T14%3A20%3A34.999Z");
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
-        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
-        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,null);
+        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
+        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1764,7 +1789,7 @@ namespace MarketData.Helper
      }
      catch (Exception exception)
      {
-        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
@@ -1861,7 +1886,7 @@ namespace MarketData.Helper
      }
      catch (Exception exception)
      {
-        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
@@ -1913,12 +1938,11 @@ namespace MarketData.Helper
          headline.Entry=Utility.RemoveHtml(headline.Entry);
 					headlines.Add(headline);
 				}
-//        headlines=new Headlines(headlines.GroupBy(x=>x.Entry).Select(y=>y.First()).ToList());
        return headlines;
      }
      catch(Exception exception)
      {
-        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
@@ -4547,9 +4571,15 @@ namespace MarketData.Helper
    public static Fundamental GetFundamental(String symbol)
    {
      Fundamental fundamental=GetFundamentalEx(symbol);
+      if(null==fundamental)return fundamental;
+      if(fundamental.GetLoad()<80)
+      {
+        Fundamental fundamental2 = GetFundamentalEx(symbol);
+        if(null!=fundamental2)fundamental.MergeFrom(fundamental2);
+      }
      return fundamental;
    }
-    public static Fundamental GetFundamentalEx(String symbol)
+    private static Fundamental GetFundamentalEx(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      int TIMEOUT_MS=1000*30;
@@ -4641,26 +4671,63 @@ namespace MarketData.Helper
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
        if(!httpNetResponse.Success)
        {
-          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
-          return false;
+          try{Thread.Sleep(250);}catch(Exception){;}
+          httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
+          if(!httpNetResponse.Success)
+          {
+            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+            return false;
+          }
        }
        fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));
+
        fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
+        if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<"));
+
        fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
+        if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets  (ttm)<"));
+
        fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
-        fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
-        fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
-        fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
-        fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
+        if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity  (ttm)<"));
+
        fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
+        if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share  (ttm)<"));
+        
+        fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
+        if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash  (mrq)<"));
+
+        fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
+        if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt  (mrq)<"));
+
+        fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
+        if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding <sup>6</sup> "));
+
+        fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
+        if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue  (ttm)<"));
+
        fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
+        if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth  (yoy)<"));
+
        fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
+        if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit  (ttm)<"));
+
        fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
+        if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA  <"));
+
        fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
+        if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common  (ttm)<"));
+
        fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
+        if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share  (mrq)<"));
+
        fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
+        if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow  (ttm)<"));
+
        fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
+        if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow  (ttm)<"));
+
        fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));
+
        return true;
      }
      catch(Exception exception)
@@ -4688,10 +4755,17 @@ namespace MarketData.Helper
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
        if(!httpNetResponse.Success)
        {
-          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
-          return false;
+          try{Thread.Sleep(250);}catch(Exception){;}
+          httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
+          if(!httpNetResponse.Success)
+          {
+            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+            return false;
+          }
        }
        fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
+        if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00;
+
        return true;
      }
      catch(Exception exception)
@@ -5032,6 +5106,10 @@ namespace MarketData.Helper
      {
        price.Open=price.High=price.Low=price.Close;
      }
+      if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low))
+      {
+        price.Open=(price.High+price.Low)/2.00;
+      }
    }

 // This is used in the intra-day price feed.
--- a/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketDataLib/Integration/HttpNetRequest.cs
@@ -958,7 +958,7 @@ namespace MarketData.Integration
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
        };
        
-        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
+        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5A[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
@@ -1006,7 +1006,7 @@ namespace MarketData.Integration
      }
      finally
      {
-        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
+        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5A[LEAVE]");
      }
    }

@@ -1015,7 +1015,7 @@ namespace MarketData.Integration
      HttpWebResponse webResponse = null;
      try
      {
-        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
+        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5B[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
@@ -1057,7 +1057,102 @@ namespace MarketData.Integration
      }
      finally
      {
-        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
+        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5B[LEAVE]");
+      }
+    }
+
+// This accepts gzip and deflate which seems to work for the seeking alpha news retrieval
+    public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
+    {
+      HttpWebResponse webResponse = null;
+      try
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
+        byte[] buffer = new byte[8192];
+        StringBuilder sb = new StringBuilder();
+        bool expect100Condition = ServicePointManager.Expect100Continue;
+        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
+        ServicePointManager.Expect100Continue = true;
+        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
+        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
+        if(null!=webProxy)webRequest.Proxy=webProxy;
+        webRequest.Timeout = webRequestTimeoutMS;
+        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
+        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
+        webRequest.Host = host;
+        webRequest.Headers.Add("Sec-GPC: 1");
+        webRequest.Headers.Add("Sec-Fetch-Dest: document");
+        webRequest.Headers.Add("Sec-Fetch-Mode: navigate");
+        webRequest.Headers.Add("Sec-Fetch-Site: none");
+        webRequest.Headers.Add("Sec-Fetch-User:	?1");
+        webRequest.Headers.Add("DNT:	1");
+        webRequest.Headers.Add("Upgrade-Insecure-Requests:	1");
+        webRequest.Headers.Add("TE:	trailers");
+//        webRequest.Headers.Add("If-None-Match: W/\"135dac681d6a78233fc3539ece5ea75a\"");
+        webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
+        if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
+        else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0";
+        webRequest.KeepAlive = true;
+        webRequest.CookieContainer = new CookieContainer();
+        if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
+        webResponse = (HttpWebResponse)webRequest.GetResponse();
+        HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse);
+        webResponse.Close();
+        ServicePointManager.Expect100Continue = expect100Condition;
+        ServicePointManager.SecurityProtocol = securityProtocolType;
+        return httpNetResponse;
+      }
+      catch (WebException webException)
+      {
+        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
+      }
+      catch (Exception exception)
+      {
+        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
+      }
+      finally
+      {
+        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
+      }
+    }
+
+    private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
+    {
+      try
+      {
+        Stream responseStream = webResponse.GetResponseStream();
+        StringBuilder sb = new StringBuilder();
+        int charCount = 0;
+        byte[] buffer = new byte[8192];
+
+        if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
+        {
+          responseStream = new GZipStream(responseStream, CompressionMode.Decompress);          
+          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
+          sb.Append(reader.ReadToEnd());
+          reader.Close();
+        }
+        else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
+        {
+          responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
+          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
+          sb.Append(reader.ReadToEnd());
+          reader.Close();
+        }
+        else
+        {
+          while (true)
+          {
+            charCount = responseStream.Read(buffer, 0, buffer.Length);
+            if (0 == charCount) break;
+            sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
+          }
+        }
+        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
+      }
+      catch(Exception exception)
+      {
+        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
    }

--- a/MarketDataLib/MarketDataModel/Fundamentals.cs
+++ b/MarketDataLib/MarketDataModel/Fundamentals.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Http.Headers;
 using System.Text;
 using MarketData.Utils;

@@ -203,7 +204,7 @@ namespace MarketData.MarketDataModel
      get{return debtToEquity;}
      set{debtToEquity=value;}
    }
-// if columns are added to this object must ensure that the columns are also added to the MergeFrom and IsZero methods so that the consistency checks can still function properly
+// If columns are added to this object must ensure that the columns are also added to the MergeFrom, IsZero, and GetLoad() methods so that the consistency checks can still function properly
 // Do not include a comparison of the AsOf date.  The prior fundamental will always have an AsOf date that precedes the one that we fetch.
 // In contrast, the AsOf date in the financial statements is the statement date and we use the AsOf date as a comparison there.
 // The idea here is to ensure that we do not lose any fidelity of the fundamental data if elements are missing in the current fetch but were present in a previous fetch.
@@ -279,6 +280,45 @@ namespace MarketData.MarketDataModel
      if(percentMissing>85.00)return true;
      return false;
    }
+
+    public double GetLoad()
+    {
+      double missingItemCount=0.00;
+      double totalItems=29.00;
+      double percentMissing=0.00;
+      if(Utility.IsZeroOrNaN(Beta))missingItemCount++;
+      if(Utility.IsZeroOrNaN(Low52))missingItemCount++;
+      if(Utility.IsZeroOrNaN(High52))missingItemCount++;
+      if(Utility.IsZeroOrNaN((double)Volume))missingItemCount++;
+      if(Utility.IsZeroOrNaN(MarketCap))missingItemCount++;
+      if(Utility.IsZeroOrNaN(PE))missingItemCount++;
+      if(Utility.IsZeroOrNaN(EPS))missingItemCount++;
+      if(Utility.IsZeroOrNaN(PEG))missingItemCount++;
+      if(Utility.IsZeroOrNaN(ReturnOnAssets))missingItemCount++;
+      if(Utility.IsZeroOrNaN(ReturnOnEquity))missingItemCount++;
+      if(Utility.IsZeroOrNaN(TotalCash))missingItemCount++;
+      if(Utility.IsZeroOrNaN(TotalDebt))missingItemCount++;
+      if(Utility.IsZeroOrNaN(SharesOutstanding))missingItemCount++;
+      if(Utility.IsZeroOrNaN(Revenue))missingItemCount++;
+      if(Utility.IsZeroOrNaN(SharesOutstanding))missingItemCount++;
+      if(Utility.IsZeroOrNaN(Revenue))missingItemCount++;
+      if(Utility.IsZeroOrNaN(RevenuePerShare))missingItemCount++;
+      if(Utility.IsZeroOrNaN(QtrlyRevenueGrowth))missingItemCount++;
+      if(Utility.IsZeroOrNaN(GrossProfit))missingItemCount++;
+      if(Utility.IsZeroOrNaN(EBITDA))missingItemCount++;
+      if(Utility.IsZeroOrNaN(NetIncomeAvailableToCommon))missingItemCount++;
+      if(Utility.IsZeroOrNaN(BookValuePerShare))missingItemCount++;
+      if(Utility.IsZeroOrNaN(OperatingCashflow))missingItemCount++;
+      if(Utility.IsZeroOrNaN(LeveragedFreeCashflow))missingItemCount++;
+      if(Utility.IsZeroOrNaN(Equity))missingItemCount++;
+      if(Utility.IsZeroOrNaN(TrailingPE))missingItemCount++;
+      if(Utility.IsZeroOrNaN(EnterpriseValue))missingItemCount++;
+      if(Utility.IsZeroOrNaN(EBIT))missingItemCount++;
+      if(Utility.IsZeroOrNaN(DebtToEquity))missingItemCount++;
+      percentMissing=(missingItemCount/totalItems)*100.00;
+      return 100.00-percentMissing;
+    }
+
    public static String Header
    {
      get
--- a/MarketDataLib/Utility/FeedParser.cs
+++ b/MarketDataLib/Utility/FeedParser.cs
@@ -158,7 +158,45 @@ namespace MarketData.Utils
        return DateTime.Parse("01-01-0001");
      }
    }
-// Sep. 25, 2022 at 4:31 p.m. ET
+    // "Apr. 04"
+    // "1:00pm"
+    public static DateTime ParseValueDateTimeMonth(String strText)
+    {
+      try
+      {
+        if(null==strText)return DateTime.Parse("01-01-0001");
+
+        if(strText.EndsWith("am") || strText.EndsWith("pm"))
+        {
+          return DateTime.Now;
+        }
+        else if(strText.Contains("."))
+        {
+          strText=strText.Replace(".",null);            
+          String[] subItems=strText.Split(' ');
+          DateTime currentDate = DateTime.Now;
+          StringBuilder sb=new StringBuilder();
+          sb.Append(subItems[0]).Append(" ");
+          sb.Append(subItems[1]).Append(",").Append(" ");
+          sb.Append(currentDate.Year.ToString());
+          DateTime resultingDate = ParseValueDateTimeMonthFormat(sb.ToString());
+          if(resultingDate>currentDate)resultingDate = new DateTime(resultingDate.Year-1,resultingDate.Month, resultingDate.Day);
+          return resultingDate;
+        }
+        else
+        {
+          return Utility.ParseDate(strText);
+        }
+      }
+      catch (Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG, "[ParseValueDateTimeMonth] Error parsing date '" + strText + "', " + exception.ToString());
+        return DateTime.Parse("01-01-0001");
+      }
+    }
+
+
+    // Sep. 25, 2022 at 4:31 p.m. ET
    public static DateTime ParseValueDateTimeMonthFormatTZ(String strText)
    {
      try
@@ -177,7 +215,6 @@ namespace MarketData.Utils
      }
    }

-
    public static DateTime? ParseRelativeDate(String strDate)
    {
      try
--- a/MarketDataLib/Utility/Sections.cs
+++ b/MarketDataLib/Utility/Sections.cs
@@ -265,7 +265,7 @@ namespace MarketDataLib.Utility
        {
          String sectionItem=sections[startIndex];
          if(item.Contains(sectionItem))continue;
-          if("".Equals(sectionItem)||"-".Equals(sectionItem)||sectionItem.StartsWith("("))continue;
+          if("".Equals(sectionItem)||"-".Equals(sectionItem)||"--".Equals(sectionItem)||sectionItem.StartsWith("("))continue;
          if((sectionItem.All(Char.IsLetter)||sectionItem.Contains(" ")))break;
          strItem=sectionItem;
          break;
--- a/MarketDataLib/Utility/Utility.cs
+++ b/MarketDataLib/Utility/Utility.cs
@@ -48,6 +48,19 @@ namespace MarketData.Utils
 				  return "Unknown";
 			}
 		}
+    /// <summary>
+    /// poses a question to the console and receives a confirmation response
+    /// </summary>
+    /// <param name="message">The message to ask the user.</param>
+    public static bool GetVerificationToProceed(String message)
+    {
+        Console.Write(String.Format("{0} Y/N? ",message));
+        String answer = Console.ReadLine();
+        answer=answer.ToUpper();
+        if(!"Y".Equals(answer))return false;
+        return true;
+    }
+
    public static long DateToUnixDate(DateTime dateTime)
    {
      DateTime javascriptEpoch=DateTime.Parse("01-01-1970 00:00:00");
@@ -432,7 +445,7 @@ namespace MarketData.Utils
    public static DateTime ParseDate(String strDate)
    {
      System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US");
-      String[] formats=new[] { "yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray();
+      String[] formats=new[] { "yy-MM-dd","yyyy-MM-dd hh:mm:ss tt","dddd, MMMM dd","MMM dd yyyy","yyyy-MM","ddd, MMM. d","ddd, MMM. dd","yyyy/MM/dd","M-d-yyyy","dd-MM-yyyy","MM-dd-yyyy","M.d.yyyy","dd.MM.yyyy","MM.dd.yyyy","yyyyMMdd" }.Union(cultureInfo.DateTimeFormat.GetAllDateTimePatterns()).ToArray();
      strDate = strDate.Trim();
      DateTime dateTime=DateTime.ParseExact(strDate, formats, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal);
      return dateTime;