Fix rejection statistics

MarketDataLib/Generator/Momentum/MomentumGenerator.cs

@@ -1,14 +1,11 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
 using MarketData.MarketDataModel;
 using MarketData.DataAccess;
 using MarketData.Utils;
 using System.Linq;
-using MarketData.Helper;
 using MarketData.Numerical;
 using MarketData.Cache;
-using MarketData.Generator.Indicators;
 
 // Filename: MomentumGenerator.cs
 // Author:Sean Kessler
@@ -72,57 +69,101 @@ namespace MarketData.Generator.Momentum
       DateTime startDateOfReturns=dateGenerator.GetPrevMonthEnd(tradeDate,2); 
       List<String> noTradeSymbols=Utility.ToList(config.NoTradeSymbols);
       List<String> noTradeFinancialSymbols=Utility.ToList(config.NoTradeFinancialSymbols);
-      MomentumRejectionStatistics momentumRejectionStatistics=new MomentumRejectionStatistics();
+      CandidateViolations candidateViolations = new CandidateViolations();
 
       MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Generate momentum.. examining candidates"));
-      momentumRejectionStatistics.TotalItems=symbols.Count;                                                        
 // Go through the universe of stocks 
       for(int index=0;index<symbols.Count;index++)
       {
         String symbol=symbols[index];
         if(0==(index%500))Console.WriteLine("Processing item {0} of {1}",index+1,symbols.Count);
 // Check if the symbol is held in any open positions
-        if(symbolsHeld.Any(x=>x.Equals(symbol))){momentumRejectionStatistics.SymbolHeldList.Add(symbol);continue;}
+        if(symbolsHeld.Any(x=>x.Equals(symbol)))
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate already held."));
+          continue;
+        }
 
 // Check if the symbol is in the no trade list (i.e.) Bitcoin etc.,
-        if(noTradeSymbols.Any(x=>x.Equals(symbol))){momentumRejectionStatistics.NoTradeListList.Add(symbol);continue;}
+        if(noTradeSymbols.Any(x=>x.Equals(symbol)))
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate in NoTradeSymbol."));
+          continue;
+        }
 
 // Check MarketCap, EBITDA, PE, and Revenue Per Share
         Fundamental fundamental=FundamentalDA.GetFundamentalMaxDate(symbol,tradeDate);
-        if(null==fundamental){momentumRejectionStatistics.NoFundamentalList.Add(symbol);continue;}
+        if(null==fundamental)
-        if(!(fundamental.MarketCap>=config.MarketCapLowerLimit)){momentumRejectionStatistics.MarketCapLimitList.Add(symbol);continue;} 
+        {
-        if(config.UseEBITDAScreen && (double.IsNaN(fundamental.EBITDA)||fundamental.EBITDA<=0)){momentumRejectionStatistics.EBITDAScreenList.Add(symbol);continue;}
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate no fundamental."));
-        if(config.UseRevenuePerShareScreen && (double.IsNaN(fundamental.RevenuePerShare)||fundamental.RevenuePerShare<0.00)){momentumRejectionStatistics.RevenuePerShareScreenList.Add(symbol);continue;}
+          continue;
+        }
+
+        if(!(fundamental.MarketCap>=config.MarketCapLowerLimit))
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate MarketCapLimit."));
+          continue;
+        } 
+
+        if(config.UseEBITDAScreen && (double.IsNaN(fundamental.EBITDA)||fundamental.EBITDA<=0))
+        {  
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate EBITDA violation."));
+          continue;
+        }
+
+        if(config.UseRevenuePerShareScreen && (double.IsNaN(fundamental.RevenuePerShare)||fundamental.RevenuePerShare<0.00))
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate RevenuePerShare violation."));
+          continue;
+        }
 
 // Initial PE screening.  This screen checks for existance of PE and if it is availabe it must be >0.00 .  There is another PE based on limits further below
-//        if(config.UsePEScreen && (double.IsNaN(fundamental.PE))||fundamental.PE<=0.00)
         if(config.UsePEScreen && (double.IsNaN(fundamental.PE)||fundamental.PE<=0.00))
         {
-          momentumRejectionStatistics.PEScreenList.Add(symbol);
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate PE violation."));
           continue;
         }
 
 // Exclude any company in the "Financial" sector
         CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
-        if(null!=companyProfile&&null!=companyProfile.Sector&&noTradeFinancialSymbols.Any(x=>x.Equals(companyProfile.Sector))){momentumRejectionStatistics.FinancialSectorScreenList.Add(symbol);continue;}
+        if(null!=companyProfile&&null!=companyProfile.Sector&&noTradeFinancialSymbols.Any(x=>x.Equals(companyProfile.Sector)))
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate Financial Sector violation."));
+          continue;
+        }
 
 // Fetch single day price
         Price price=GBPriceCache.GetInstance().GetPrice(symbol,tradeDate);
-        if(null==price){momentumRejectionStatistics.OneDayPriceScreenList.Add(symbol);continue;}
+        if(null==price)
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate missing price on trade date."));
+          continue;
+        }
 // Filter penny stocks - don't trade anything less than $1.00
-        if(price.Close<1.00||price.Open<1.00){momentumRejectionStatistics.PennyStockScreenList.Add(symbol);continue;}
+        if(price.Close<1.00||price.Open<1.00)
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate penny stock violation."));
+          continue;
+        }
 
 // Retrieve prices                    
         Prices prices=null;
         prices=GBPriceCache.GetInstance().GetPrices(symbol,tradeDate,(int)MomentumGeneratorConstants.DayCount);
-        if(null==prices||prices.Count!=(int)MomentumGeneratorConstants.DayCount){momentumRejectionStatistics.TradeDateHistoricalPricingScreenList.Add(symbol);continue;}
+        if(null==prices||prices.Count!=(int)MomentumGeneratorConstants.DayCount)
-        float[] returns=null;
+        {
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate missing price history."));
+          continue;
+        }
 
 // calculate the one day return
         double return1D=prices.GetReturn1D();
 
 // Liquidity check - if any day has volume < 10,000 then we reject it
-       if(((from Price xPrice in prices where xPrice.Volume<10000 select xPrice).Count())>1){momentumRejectionStatistics.LiquidityScreenList.Add(symbol);continue;}
+       if(((from Price xPrice in prices where xPrice.Volume<10000 select xPrice).Count())>1)
+       {
+          candidateViolations.Add(new CandidateViolation(symbol,"Liquidity violation."));
+         continue;
+       }
 
 // Calculate velocity as a percentage range of the open price within the 252+20 day range of prices - This is used for display purposes 
         double velocity;
@@ -145,7 +186,11 @@ namespace MarketData.Generator.Momentum
           pricesArray=Numerics.ToDouble(benchmarkPrices.GetPricesLow());
           leastSquaresResult=Numerics.LeastSquares(pricesArray);
           double slopeBmk=leastSquaresResult.Slope;
-          if(slopeBmk<0){momentumRejectionStatistics.HighBetaBenchmarkSlopeScreenList.Add(symbol);continue;}
+          if(slopeBmk<0)
+          {
+            candidateViolations.Add(new CandidateViolation(symbol,"Beta threshhold violation."));
+            continue;
+          }
         }
 
 // *** MACDSignal detection
@@ -157,8 +202,16 @@ namespace MarketData.Generator.Momentum
           signalsMACD=new Signals(signalsMACD.Take(config.MACDSignalDays).ToList());
           int weakSellSignals=(from Signal signal in signalsMACD where signal.IsWeakSell() select signal).Count();
           int strongSellSignals=(from Signal signal in signalsMACD where signal.IsStrongSell() select signal).Count();
-          if(config.MACDRejectWeakSellSignals && weakSellSignals>0){momentumRejectionStatistics.MACDWeakSellScreenList.Add(symbol);continue;}
+          if(config.MACDRejectWeakSellSignals && weakSellSignals>0)
-          if(config.MACDRejectStrongSellSignals && strongSellSignals>0){momentumRejectionStatistics.MACDStrongSellScreenList.Add(symbol);continue;}
+          {
+            candidateViolations.Add(new CandidateViolation(symbol,"MACD Reject Weak Sell violation."));
+            continue;
+          }
+          if(config.MACDRejectStrongSellSignals && strongSellSignals>0)
+          {
+            candidateViolations.Add(new CandidateViolation(symbol,"MACD Reject Strong Sell violation."));
+            continue;
+          }
         }
 
 // *** Stochastics oscillator 
@@ -169,8 +222,16 @@ namespace MarketData.Generator.Momentum
           signalsStochastics=new Signals(signalsStochastics.Take(config.StochasticsSignalDays).ToList());
           int weakSellCount=(from Signal signal in signalsStochastics where signal.IsWeakSell() select signal).Count();
           int strongSellCount=(from Signal signal in signalsStochastics where signal.IsStrongSell() select signal).Count();
-          if(config.StochasticsRejectStrongSells&&strongSellCount>0) { momentumRejectionStatistics.StochasticsStrongSellScreenList.Add(symbol); continue; }
+          if(config.StochasticsRejectStrongSells&&strongSellCount>0)
-          if(config.StochasticsRejectWeakSells&&weakSellCount>0) { momentumRejectionStatistics.StochasticsWeakSellScreenList.Add(symbol); continue; }
+          { 
+            candidateViolations.Add(new CandidateViolation(symbol,"Stochastics Oscillator Reject Strong Sell violation."));
+            continue;
+          }
+          if(config.StochasticsRejectWeakSells&&weakSellCount>0)
+          { 
+            candidateViolations.Add(new CandidateViolation(symbol,"Stochastics Oscillator Reject Weak Sell violation."));
+            continue;
+          }
         }
 
 // Analyst Ratings - "Downgrades" that are more than a year old (252 days) are not considered.  Mean reversion.... bad companies improve, good companies decline.
@@ -181,7 +242,7 @@ namespace MarketData.Generator.Momentum
         if(null!=analystRatings)rating=(from AnalystRating analystRating in analystRatings where analystRating.Type.Equals("Downgrades") select analystRating).FirstOrDefault();
         if(null!=rating)
         {
-          momentumRejectionStatistics.AnalystRatingsScreenList.Add(symbol);
+          candidateViolations.Add(new CandidateViolation(symbol,"AnalystRating Downgrade violation within set period."));
           continue;
         }
 
@@ -189,31 +250,25 @@ namespace MarketData.Generator.Momentum
         prices=GBPriceCache.GetInstance().GetPrices(symbol,startDateOfReturns,(int)MomentumGeneratorConstants.DayCount);
         if(null==prices||(int)MomentumGeneratorConstants.DayCount!=prices.Count)
         {
-          momentumRejectionStatistics.StartDateOfReturnsHistoricalPricingScreenList.Add(symbol);
+          candidateViolations.Add(new CandidateViolation(symbol,"Insufficient pricing, cannot determine rank."));
           continue;
         }
 
 // check for outliers in the return stream
+        float[] returns = default;
         returns=prices.GetReturns();
         if((from float value in returns where Math.Abs(value)>.50 select value).Count()>0)
         {
-          momentumRejectionStatistics.OutliersInReturnsScreenList.Add(symbol);
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate pricing contains outliers in the returns."));
           continue;
         }
 // Cumulative return
         double cumulativeReturn=prices.GetCumulativeReturn();
-        if(cumulativeReturn<.10){momentumRejectionStatistics.NegativeReturnsScreenList.Add(symbol);continue;}
+        if(cumulativeReturn<.10)
-
+        {
-// OverExtended check
+          candidateViolations.Add(new CandidateViolation(symbol,"Candidate cumulative returns below threshhold."));
-        //if(config.UseOverExtendedIndicator)
+          continue;
-        //{
+        }
-        //  bool? result=OverExtendedIndicator.IsOverextended(symbol,tradeDate,config.UseOverExtendedIndicatorDays,config.UseOverExtendedIndicatorViolationThreshhold,config.UseOverExtendedIndicatorMarginPercent);
-        //  if(null!=result && true==result.Value)
-        //  {
-        //    momentumRejectionStatistics.OverExtendedScreenList.Add(symbol);
-        //    continue;
-        //  }
-        //}
 
 // Zacks Rank.  This is for informational purposes for now but may further it's use in the future.
         ZacksRank zacksRank=ZacksRankDA.GetZacksRankOnOrBefore(symbol,tradeDate);
@@ -222,7 +277,7 @@ namespace MarketData.Generator.Momentum
 // The idea is to try to avoid high PE stocks as they are more likey to introduce drawdowns as backtests have shown.
         if(config.UseMaxPEScreen && !double.IsNaN(fundamental.PE) && fundamental.PE>config.MaxPE)
         {
-            momentumRejectionStatistics.MaxPEScreenList.Add(symbol);
+            candidateViolations.Add(new CandidateViolation(symbol,"PE violation."));
             MomentumCandidate highPECandidate=new MomentumCandidate();
             highPECandidate.AnalysisDate=tradeDate;
             highPECandidate.Symbol=symbol;
@@ -259,9 +314,26 @@ namespace MarketData.Generator.Momentum
         momentumCandidate.Return1D=return1D;
         if(null!=zacksRank)momentumCandidate.ZacksRank=zacksRank.Rank;
         momentumCandidates.Add(momentumCandidate);
+
       }  // for all symbols
+
+      if(0!=candidateViolations.Count)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,"**************** C A N D I D A T E   S U M M A R Y  ************************");
+        IEnumerable<Tuple<string, int>> groups = candidateViolations.GroupBy(x => x.ReasonCategory).OrderByDescending(group => group.Count()).Select(group => Tuple.Create(group.Key, group.Count()));
+        foreach(Tuple<string, int> group in groups)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Group: {0} Count:{1}",group.Item1, group.Item2));
+        }
+      }
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format($"Total Considered : {momentumCandidates.Count+candidateViolations.Count}"));
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format($"Total Disqualified : {candidateViolations.Count}"));
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format($"Total Eligible : {momentumCandidates.Count}"));
+      MDTrace.WriteLine(LogLevel.DEBUG,"******************************************************************************************************");
+
 // ********************************************************* E N D  C A N D I D A T E  S E L E C T I O N   C R I T E R I A  ****************************************
-//  If we wind up with less than the number of required candidates then check the StrictMaxPE flag and, if allowed, add the highPECandidate (that we've accumulated but skipped) to the momentumCandidates ordering them by the Lowest PE
+//  If we wind up with less than the number of required candidates then check the StrictMaxPE
+//  flag and, if allowed, add the highPECandidate (that we've accumulated but skipped) to the momentumCandidates ordering them by the Lowest PE
       if(!config.StrictMaxPE && momentumCandidates.Count<config.MaxPositions && highPECandidates.Count>0)
       {
         int takeCandidates=config.MaxPositions-momentumCandidates.Count;