InsiderTransactions refactorings.

MarketDataLib/DataAccess/InsiderTransactionDA.cs

@@ -173,7 +173,7 @@ namespace MarketData.DataAccess
 				if(null!=sqlTransaction)sqlTransaction.Dispose();
 			}
     }
-    private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) //, MySqlTransaction sqlTransaction)
+    private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) 
     {
       StringBuilder sb = null;
       String strQuery = null;
@@ -210,5 +210,45 @@ namespace MarketData.DataAccess
 				if(null!=sqlTransaction)sqlTransaction.Dispose();
       }
     }
+    /// <summary>
+    /// Delete InsiderTransaction for given symbol for all years>=given year.
+    /// </summary>
+    /// <param name="symbol">The symbol.</param>
+    /// <param name="yearsGreaterEqual">All years greater then or equal to the year.</param>
+    public static bool DeleteInsiderTransactionsYearsGreaterEqual(String symbol, int yearsGreaterEqual) 
+    {
+      MySqlConnection sqlConnection = null;
+      StringBuilder sb = null;
+      String strQuery = null;
+      MySqlTransaction sqlTransaction = null;
+
+      try
+      {
+        if (null == symbol) return false;
+        sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
+        sqlTransaction = sqlConnection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
+        sb = new StringBuilder();
+        sb.Append("DELETE FROM InsiderTransaction WHERE symbol=");
+        sb.Append("'").Append(symbol).Append("'");
+        sb.Append(" AND YEAR(filing_date)>=").Append(yearsGreaterEqual);
+        strQuery = sb.ToString();
+        MySqlCommand sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction);
+        sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
+        sqlCommand.ExecuteNonQuery();
+        sqlTransaction.Commit();
+        return true;
+      }
+      catch (Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        if (null != strQuery) MDTrace.WriteLine(LogLevel.DEBUG,"Query was " + strQuery);
+        return false;
+      }
+      finally
+      {
+				if(null!=sqlTransaction)sqlTransaction.Dispose();
+        if(null!=sqlConnection)sqlConnection.Close();
+      }
+    }
   }
 }

MarketDataLib/Helper/FundamentalMarketDataHelper.cs

@@ -11,7 +11,6 @@ namespace MarketData.Helper
 {
   public class FundamentalMarketDataHelper : MarketDataHelperBase<String>
   {
-//    private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
     private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
     private UpdateManager updateManager=new UpdateManager();
 

MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using MarketData.MarketDataModel;
 using MarketData.DataAccess;
@@ -9,12 +8,23 @@ using MarketData.Utils;
 
 namespace MarketData.Helper
 {
+
+  public class InsiderTransactionThreadHelper : ThreadHelper
+  {
+    public InsiderTransactionThreadHelper(String symbol, ManualResetEvent resetEvent)
+    : base(symbol, resetEvent)
+    {
+    }
+    public int Year { get; set; }
+  }
+
   public class InsiderTransactionMarketDataHelper
   {
-    private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
+    private static int MaxThreads = 10; //  10 threads avoids receiving HTTP Response 429 (Too many requests)
     private static int WAIT_TIME_MS=500;  // wait between request
     private List<String> symbols;
     private int currentIndex = 0;
+    private UpdateManager UpdateManager = new UpdateManager();
 
     public InsiderTransactionMarketDataHelper()
     {
@@ -83,6 +93,45 @@ namespace MarketData.Helper
 				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
 			}
     }
+    public bool LoadInsiderTransactionsYear(List<String> symbols,int year)
+    {
+			Profiler profiler=new Profiler();
+			try
+			{
+				this.symbols = symbols;
+				currentIndex = 0;
+
+        UpdateManager.Prepare("load_insider_transactions_year.txt", 7);  // use max age 7 days
+        this.symbols=this.symbols.Except(new List<String>(UpdateManager.Entries)).ToList();
+
+				while (true)
+				{
+					List<String> queueSymbols = GetQueueSymbols();
+					if (null == queueSymbols || 0 == queueSymbols.Count) break;
+					ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
+					for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
+					{
+						resetEvents[eventIndex] = new ManualResetEvent(false);
+					}
+					for (int index = 0; index < queueSymbols.Count; index++)
+					{
+						InsiderTransactionThreadHelper threadHelper = new InsiderTransactionThreadHelper(queueSymbols[index], resetEvents[index]);
+            threadHelper.UpdateManager=UpdateManager;
+            threadHelper.Year=year;
+						ThreadPool.QueueUserWorkItem(ThreadPoolCallbackYearGreaterEqual, threadHelper);
+            try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;}     // wait
+					}
+					MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
+					WaitHandle.WaitAll(resetEvents);
+				}
+				return true;
+			}
+			finally
+			{
+				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
+			}
+    }
+
     private List<String> GetQueueSymbols()
     {
       List<String> queueSymbols = new List<String>();
@@ -94,6 +143,7 @@ namespace MarketData.Helper
       currentIndex = index;
       return queueSymbols;
     }
+
     public void ThreadPoolCallback(Object threadHelperContext)
     {
       ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
@@ -102,6 +152,7 @@ namespace MarketData.Helper
       MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
       threadHelper.ResetEvent.Set();
     }
+
     public static void LoadInsiderTransactionsSymbolEx(String symbol)
     {
       symbol = symbol.ToUpper();
@@ -116,5 +167,31 @@ namespace MarketData.Helper
       InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
       MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
     }
+
+    public void ThreadPoolCallbackYearGreaterEqual(Object threadHelperContext)
+    {
+      InsiderTransactionThreadHelper threadHelper = (InsiderTransactionThreadHelper)threadHelperContext;
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
+      LoadInsiderTransactionsYearGreaterEqualEx(threadHelper.Symbol,threadHelper.Year);
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
+      threadHelper.UpdateManager.Add(threadHelper.Symbol);
+      threadHelper.ResetEvent.Set();
+    }
+
+    public static void LoadInsiderTransactionsYearGreaterEqualEx(String symbol, int year)
+    {
+      symbol = symbol.ToUpper();
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0} years>={1}", symbol,year));
+      InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactionsYear(symbol, year);
+      if (null == insiderTransactions || 0 == insiderTransactions.Count)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0} years>={1}", symbol,year));
+        return;
+      }
+      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1} years>={2}", insiderTransactions.Count, symbol, year));
+      InsiderTransactionDA.DeleteInsiderTransactionsYearsGreaterEqual(symbol, year);
+      InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
+      MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
+    }
   }
 }

MarketDataLib/Helper/InsiderTransactionsParser.cs

@@ -6,12 +6,13 @@ using HtmlAgilityPack;
 using MarketData.Utils;
 using MarketData.MarketDataModel;
 using MarketDataLib.Utility;
+using System.Collections.Concurrent;
 
 namespace MarketData.Helper
 {
   public class InsiderTransactionsParser
   {
-    private static Dictionary<String,String> transactionCodes=new Dictionary<String,String>();
+    private static ConcurrentDictionary<String,String> transactionCodes=new ConcurrentDictionary<String,String>();
     private static InsiderTransactionsParser instance=null;
     private InsiderTransactionsParser()
     {                               
@@ -26,6 +27,140 @@ namespace MarketData.Helper
         return instance; 
       }
     }
+
+    /// <summary>
+    /// Parse the SECFilings.  Each SECFiling may contain a list of InsiderTransaction
+    /// </summary>
+    /// <param name="secFilings">The SECFilings.</param>
+    public InsiderTransactions Parse(SECFilings secFilings)
+    {
+      try
+      {
+        InsiderTransactions insiderTransactions=new InsiderTransactions();
+
+        if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
+        foreach(SECFiling secFiling in secFilings)
+        {
+          InsiderTransactions secFilinginsiderTransactions =
+            Parse(secFiling.FormText, secFiling.Symbol,
+              secFiling.SECAccessionNumber,
+                secFiling.Form, secFiling.FilingDate);
+          if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0)
+           insiderTransactions.AddRange(secFilinginsiderTransactions);
+        }
+        return insiderTransactions;
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
+        return new InsiderTransactions();
+      }
+    }
+
+    public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate)
+    {
+      try
+      {
+        InsiderTransactions insiderTransactions = new InsiderTransactions();
+        if(null==strHtml)return null;
+
+        List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
+        String insiderName=null;
+
+        byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml);
+        MemoryStream memoryStream = new MemoryStream(streamBytes);
+        HtmlDocument htmlDocument = new HtmlDocument();
+        htmlDocument.Load(memoryStream);
+        HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
+
+        if(null==tables || tables.Count<5)return null;
+
+        HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
+        if(null==nameAndAddressTable)return null;
+        HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
+        if(nameAndAddressRows.Count<2)return null;
+        insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
+
+        for(int index=0;index<tables.Count;index++)
+        {
+          HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
+          foreach(HtmlNode node in nodeCollection)
+          {
+            HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
+            if(null==dataCollection || 0==dataCollection.Count)continue;
+            if(11==dataCollection.Count)
+            {
+              Dictionary<String,String> dictionary=new Dictionary<String,String>();
+              dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
+              dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
+              dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
+              dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
+              dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
+              dictionary.Add("Price",dataCollection[7].InnerHtml);
+              dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
+              dictionaryList.Add(dictionary);
+            }
+          }
+        }
+
+        foreach(Dictionary<String,String> dictionary in dictionaryList)
+        {
+          String strItem=null;
+          InsiderTransaction insiderTransaction=new InsiderTransaction();
+          insiderTransaction.Symbol=symbol;
+          insiderTransaction.SECAccessionNumber=secAccessionNumber;
+          insiderTransaction.Form=form;
+          insiderTransaction.FilingDate=filingDate;
+          insiderTransaction.InsiderName=insiderName;
+          insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
+          insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
+          if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
+          strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
+          if(String.IsNullOrEmpty(strItem))continue;
+          insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
+          insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
+          insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
+          insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString();
+          insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
+          String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary);
+          if(null==acquiredOrDisposed)
+          {
+            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber));
+            continue;
+          }
+          if(acquiredOrDisposed.Equals("D"))
+          {
+            insiderTransaction.NumberOrValueAcquiredDisposed*=-1;
+          }
+          insiderTransactions.Add(insiderTransaction);
+        }
+        return insiderTransactions;
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString()));
+        return null;
+      }
+    }
+
+    private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
+    {
+      if(null==tables)return null;
+      for(int index=0;index<tables.Count;index++)
+      {
+        HtmlNodeCollection collection = tables[index].SelectNodes(".//tr");
+        for(int itemIndex=0;itemIndex<collection.Count;itemIndex++)
+        {
+          String marker=collection[itemIndex].InnerText;
+          if(marker.StartsWith(startsWith))
+          {
+            return tables[index];
+          }
+        }
+      }
+      return null;
+    }
+
     private String ApplyNameCase(String name)
     {
       StringBuilder sb=new StringBuilder();
@@ -40,18 +175,33 @@ namespace MarketData.Helper
       }
       return sb.ToString();
     }
+
     private String GetOwnershipForm(List<String> sections)
     {
       String item=GetFirstSection(sections);
+      if(null==item)item="";
+      item=item.ToUpper();
       if(item.Equals("D"))return "Direct Ownership";
       else if(item.Equals("I"))return "Indirect Ownership";
       return "";
     }
+
+    private String GetAcquiredOrDisposed(Dictionary<String,String> dictionary)
+    {
+      if(!dictionary.ContainsKey("Acquired or Disposed"))return null;
+      List<String> section=Sections.GetSections(dictionary["Acquired or Disposed"]);
+      if(null==section||0==section.Count)return null;
+      String acquiredOrDisposed=section[0];
+      if(null==acquiredOrDisposed)return null;
+      return acquiredOrDisposed.Trim().ToUpper();
+    }
+
     private String GetFirstSection(List<String> sections)
     {
       if(null==sections || 0==sections.Count)return "";
       return sections[0];
     }
+
     private String CombineSections(List<String> sections)
     {
       StringBuilder sb=new StringBuilder();
@@ -62,135 +212,29 @@ namespace MarketData.Helper
       }
       return sb.ToString();
     }
-    public InsiderTransactions Parse(SECFilings secFilings)
-    {
-      lock(instance)
-      {
-        try
-        {
-          InsiderTransactions insiderTransactions=new InsiderTransactions();
-
-          if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
-          foreach(SECFiling secFiling in secFilings)
-          {
-            List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
-            String insiderName=null;
-            String relationshipOfReportingPerson=null;
-
-            byte[] streamBytes = Encoding.ASCII.GetBytes(secFiling.FormText);
-            MemoryStream memoryStream = new MemoryStream(streamBytes);
-            HtmlDocument htmlDocument = new HtmlDocument();
-            htmlDocument.Load(memoryStream);
-            HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
-
-            if(null==tables || tables.Count<5)continue;
-
-            HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
-            if(null==nameAndAddressTable)continue;
-            HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
-            if(nameAndAddressRows.Count<2)continue;
-            insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
-
-
-            //HtmlNodeCollection nameAndAddressRows = tables[4].SelectNodes(".//tr");
-            //if(nameAndAddressRows.Count<10)continue;
-            //insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
-            //relationshipOfReportingPerson=Utility.RemoveHtml(nameAndAddressRows[9].InnerText);
-
-            for(int index=0;index<tables.Count;index++)
-            {
-              HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
-              foreach(HtmlNode node in nodeCollection)
-              {
-                HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
-                if(null==dataCollection || 0==dataCollection.Count)continue;
-                if(11==dataCollection.Count)
-                {
-                  Dictionary<String,String> dictionary=new Dictionary<String,String>();
-                  dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
-                  dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
-                  dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
-                  dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
-                  dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
-                  dictionary.Add("Price",dataCollection[7].InnerHtml);
-                  dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
-                  dictionaryList.Add(dictionary);
-                }
-              }
-            }
-
-            foreach(Dictionary<String,String> dictionary in dictionaryList)
-            {
-              String strItem=null;
-              InsiderTransaction insiderTransaction = new InsiderTransaction();
-              insiderTransaction.Symbol=secFiling.Symbol;
-              insiderTransaction.SECAccessionNumber=secFiling.SECAccessionNumber;
-              insiderTransaction.FormRowNumber=(secFiling.Sequence+1).ToString();
-              insiderTransaction.Form=secFiling.Form;
-              insiderTransaction.FilingDate=secFiling.FilingDate;
-              insiderTransaction.InsiderName=insiderName;
-              insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
-              insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
-              if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
-              strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
-              if(String.IsNullOrEmpty(strItem))continue;
-              insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
-              insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
-              insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
-              insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
-              insiderTransactions.Add(insiderTransaction); 
-            }
-          }
-          return insiderTransactions;
-        }
-        catch(Exception exception)
-        {
-          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
-          return new InsiderTransactions();
-        }
-      }
-    }
-
-    private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
-    {
-      if(null==tables)return null;
-      for(int index=0;index<tables.Count;index++)
-      {
-        HtmlNodeCollection nameAndAddressRows = tables[index].SelectNodes(".//tr");
-        for(int itemIndex=0;itemIndex<nameAndAddressRows.Count;itemIndex++)
-        {
-          String marker=nameAndAddressRows[itemIndex].InnerText;
-          if(marker.StartsWith(startsWith))
-          {
-            return tables[index];
-          }
-        }
-      }
-      return null;
-    }
 
     private void BuildTransactionCodes()
     {
-      transactionCodes.Add("P","Open market or private purchase of non-derivative or derivative security");
+      transactionCodes.TryAdd("P","Open market or private purchase of non-derivative or derivative security");
-      transactionCodes.Add("S","Open market or private sale of non-derivative or derivative security");
+      transactionCodes.TryAdd("S","Open market or private sale of non-derivative or derivative security");
-      transactionCodes.Add("V","Transaction voluntarily reported earlier than required");
+      transactionCodes.TryAdd("V","Transaction voluntarily reported earlier than required");
-      transactionCodes.Add("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
+      transactionCodes.TryAdd("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
-      transactionCodes.Add("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
+      transactionCodes.TryAdd("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
-      transactionCodes.Add("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
+      transactionCodes.TryAdd("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
-      transactionCodes.Add("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
+      transactionCodes.TryAdd("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
-      transactionCodes.Add("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
+      transactionCodes.TryAdd("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
-      transactionCodes.Add("C","Conversion of derivative security");
+      transactionCodes.TryAdd("C","Conversion of derivative security");
-      transactionCodes.Add("E","Expiration of short derivative position");
+      transactionCodes.TryAdd("E","Expiration of short derivative position");
-      transactionCodes.Add("H","Expiration (or cancellation) of long derivative position with value received");
+      transactionCodes.TryAdd("H","Expiration (or cancellation) of long derivative position with value received");
-      transactionCodes.Add("O","Exercise of out-of-the-money derivative security");
+      transactionCodes.TryAdd("O","Exercise of out-of-the-money derivative security");
-      transactionCodes.Add("X","Exercise of in-the-money or at-the-money derivative security");
+      transactionCodes.TryAdd("X","Exercise of in-the-money or at-the-money derivative security");
-      transactionCodes.Add("G","Bona fide gift");
+      transactionCodes.TryAdd("G","Bona fide gift");
-      transactionCodes.Add("L","Small acquisition under Rule 16a-6");
+      transactionCodes.TryAdd("L","Small acquisition under Rule 16a-6");
-      transactionCodes.Add("W","Acquisition or disposition by will or the laws of descent and distribution");
+      transactionCodes.TryAdd("W","Acquisition or disposition by will or the laws of descent and distribution");
-      transactionCodes.Add("Z","Deposit into or withdrawal from voting trust");
+      transactionCodes.TryAdd("Z","Deposit into or withdrawal from voting trust");
-      transactionCodes.Add("J","Other acquisition or disposition (describe transaction)");
+      transactionCodes.TryAdd("J","Other acquisition or disposition (describe transaction)");
-      transactionCodes.Add("K","Transaction in equity swap or instrument with similar characteristics");
+      transactionCodes.TryAdd("K","Transaction in equity swap or instrument with similar characteristics");
-      transactionCodes.Add("U","Disposition pursuant to a tender of shares in a change of control transaction");
+      transactionCodes.TryAdd("U","Disposition pursuant to a tender of shares in a change of control transaction");
     }
   }
 }

MarketDataLib/Helper/MarketDataHelper.cs

@@ -3,19 +3,15 @@ using System.IO;
 using System.Collections.Generic;
 using System.Text;
 using System.Linq;
-using System.Text.RegularExpressions;
 using System.Data;
 using System.Threading;
 using System.Net;
-using System.Collections;
 using System.Reflection;
-using System.Web;
 using System.Xml.Linq;
 using Newtonsoft.Json.Linq;
 using HtmlAgilityPack;
 using MarketData.MarketDataModel;
 using MarketData.Utils;                                                                                                         
-using MarketData.Numerical;
 using MarketData.CSVHelper;
 using MarketData.DataAccess;
 using MarketData.Integration;
@@ -1245,7 +1241,6 @@ namespace MarketData.Helper
 // ***************************************************************************************************************************************************************************************
 // ************************************ S E C   F I L I N G S  -  F O R M  4  &  F O R M 5  - F O R  I N S I D E R  T R A N S A C T I O N S  S E C . G O V     ***************************S
 // ***************************************************************************************************************************************************************************************
-//    public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=7)
     public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=30)
     {
       MemoryStream memoryStream = null;
@@ -1254,6 +1249,7 @@ namespace MarketData.Helper
       DateTime minFilingDate=DateTime.Now;
       int maxFilings=80;
       bool continuationFlag=true;
+      int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
 
       try
       {
@@ -1285,8 +1281,10 @@ namespace MarketData.Helper
         HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
         if (null == tables || tables.Count < 1) return null;
         HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count));
         for (int row = 0; row < rows.Count && continuationFlag; row ++)  
         {
+          try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
           HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
           if (null != headerColumns && headerColumns.Count > 0) continue;
           HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
@@ -1320,13 +1318,114 @@ namespace MarketData.Helper
         if(null!=httpNetResponse)httpNetResponse.Dispose();
       }
     }
+    /// <summary>
+    /// Get InsiderTransactions for years greater than or equal to specified year.
+    /// For example.  If 2023 is specified then get all for years>=2023
+    /// </summary>
+    /// <param name="symbol">The symbol.</param>
+    /// <param name="yearGreaterThanEqualTo">the new y-coordinate.</param>
+    public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
+    {
+      MemoryStream memoryStream = null;
+      HttpNetResponse httpNetResponse=null;
+      String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
+      int maxFilings=120;
+      bool continuationFlag=true;
+      int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
+
+      try
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
+        DateGenerator dateGenerator=new DateGenerator();
+        String cik=PricingDA.GetCIKForSymbol(symbol);
+        if(null==cik)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
+          return null;
+        }
+        SECFilings secFilings = new SECFilings();
+        StringBuilder sb = new StringBuilder();
+        String strRequest;
+        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
+        strRequest = sb.ToString();
+        WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
+        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+        if(!httpNetResponse.Success)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+          return null;
+        }
+        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
+        memoryStream = new MemoryStream(streamBytes);
+        HtmlDocument htmlDocument = new HtmlDocument();
+        htmlDocument.Load(memoryStream);
+        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
+        if (null == tables || tables.Count < 1) return null;
+        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}.  Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
+        for (int row = 0; row < rows.Count && continuationFlag; row ++)  
+        {
+          try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
+          HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
+          if (null != headerColumns && headerColumns.Count > 0) continue;
+          HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
+          if (null == dataColumns || 5 != dataColumns.Count) continue;
+          HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
+          if (null == nodes || 0 == nodes.Count) continue;
+          String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
+          SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
+          for (int index = 0; null!=innerCollection &&  index < innerCollection.Count; index++)
+          {
+            SECFiling innerFiling = innerCollection[index];
+            if(innerFiling.FilingDate.Year<yearGreaterThanEqualTo)
+            {
+              continuationFlag=false;
+              break;
+            }
+            if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
+            secFilings.Add(innerFiling);
+          }
+        }
+        return InsiderTransactionsParser.GetInstance().Parse(secFilings);
+      }
+      catch (Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        return null;
+      }
+      finally
+      {
+        if (null != memoryStream) memoryStream.Close();
+        if(null!=httpNetResponse)httpNetResponse.Dispose();
+      }
+    }
 // ***************************************************************************************************************************************************************************************
 // ********************************************************************** S E C   F I L I N G S  -  S E C . G O V     ********************************************************************
 // ***************************************************************************************************************************************************************************************
+    public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
+    {
+      try
+      {
+        String cik=PricingDA.GetCIKForSymbol(symbol);
+        if(null==cik)
+        {
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
+          return null;
+        }
+        return GetSECFilings(symbol,cik,maxFilings);
+      }
+      catch(Exception exception)
+      {
+        MDTrace.WriteLine(LogLevel.DEBUG,exception);
+        return null;      }
+    }
+
     public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
     {
       MemoryStream memoryStream = null;
       HttpNetResponse httpNetResponse=null;
+      int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
+
       try
       {
         SECFilings secFilings = new SECFilings();
@@ -1357,6 +1456,7 @@ namespace MarketData.Helper
           HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
           if (null == nodes || 0 == nodes.Count) continue;
           String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
+          try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;} 
           SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
           for (int index = 0; null!=innerCollection &&  index < innerCollection.Count; index++)
           {
@@ -1384,11 +1484,12 @@ namespace MarketData.Helper
       SECFilings secFilings = new SECFilings();
       String secFilingDocument = null;
       HttpNetResponse httpNetResponse=null;
+      int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
+
       try
       {
-        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl));
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl));
         WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
-        try { Thread.Sleep(1000); }catch(Exception) { ;}
         httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
         if(!httpNetResponse.Success)
         {
@@ -1409,11 +1510,11 @@ namespace MarketData.Helper
         String formName = secNodes[0].InnerText;
         formName = formName.Replace("\n", "");
         formName = formName.Trim();
-        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName));
         HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
         if (null == tables || tables.Count < 1) return null;
         HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
         httpNetResponse.Dispose();
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments.  Examining form {0} with {1} entries",formName,rows.Count));
         // go through the rows in the table file, searching for the different submission form content
         int sequence = 0;
         for (int row = 0; row < rows.Count; row++)
@@ -1425,21 +1526,22 @@ namespace MarketData.Helper
           String document = dataColumns[2].InnerText.Trim();
           String type = dataColumns[3].InnerText.Trim();
           if(null!=type && type.Equals("GRAPHIC"))continue;
-//          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
+//          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
           if (document.Contains(".htm"))
           {
+            
             HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
             if (null == nodes || 0 == nodes.Count) continue;
             String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
-//            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl));
+            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments.  Requesting {0}",secFilingDocumentXmlUrl));
-            try{Thread.Sleep(1000);}catch(Exception){;}
+            try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
             httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
             if(!httpNetResponse.Success)
             {
               MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
               continue;
             }
-            secFilingDocument =httpNetResponse.ResponseString;
+            secFilingDocument = httpNetResponse.ResponseString;
             if (null == secFilingDocument) continue;
             SECFiling secFiling = new SECFiling();
             secFiling.SECAccessionNumber = secAccessionNumber;

MarketDataLib/Helper/ThreadHelper.cs

@@ -1,8 +1,4 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using System.Threading;
 using MarketData.Utils;
 

MarketDataLib/Utility/UpdateManager.cs

@@ -16,28 +16,28 @@ namespace MarketData.Utils
     public UpdateManager()
     {
     }
-    public bool Prepare(String strPathFileName)
+    public bool Prepare(String strPathFileName, int maxAgeDays=5)
     {
       try
       {
         String currentWorkingDirectory=Directory.GetCurrentDirectory();
-        if(!File.Exists(strPathFileName)||IsExpired(strPathFileName))
+        if(!File.Exists(strPathFileName)||IsExpired(strPathFileName, maxAgeDays))
         {
           if(File.Exists(strPathFileName))File.Delete(strPathFileName);
           fileStream=new FileStream(strPathFileName,FileMode.Create);
           streamWriter=new StreamWriter(fileStream);
-          Console.WriteLine(String.Format("Creating session file:{0}",strPathFileName));
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Creating session file:{0}",strPathFileName));
         }
         else
         {
-          FileStream fileStream=new FileStream(strPathFileName,FileMode.Open);
+          FileStream fileStream=new FileStream(strPathFileName,FileMode.Open,FileAccess.ReadWrite,FileShare.Read);
           StreamReader streamReader=new StreamReader(fileStream);
           String strLine=null;
           while(null!=(strLine=streamReader.ReadLine()))
           {
             if(!entries.ContainsKey(strLine))entries.Add(strLine,strLine);
           }
-          Console.WriteLine(String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
           streamReader.Close();
           streamReader.Dispose();
           fileStream.Close();
@@ -49,27 +49,28 @@ namespace MarketData.Utils
       }
       catch(Exception exception)
       {
-        Console.WriteLine(String.Format("Exception:{0}",exception));
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
         return false;
       }
     }
-    private bool IsExpired(String strPathFileName)
+    private bool IsExpired(String strPathFileName, int maxAgeDays)
     {
       try
       {
         DateGenerator dateGenerator=new DateGenerator();
         DateTime creationTime=File.GetCreationTime(strPathFileName);
         int daysElapsed=Math.Abs(dateGenerator.DaysBetweenActual(creationTime,DateTime.Now));
-        if(daysElapsed>5)
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is {1} days(s) old.",strPathFileName,daysElapsed));
+        if(daysElapsed>maxAgeDays)
         {
-          Console.WriteLine(String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
+          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
           return true;
         }
         return false;
       }
       catch(Exception exception)
       {
-        Console.WriteLine(String.Format("Exception:{0}",exception));
+        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
         return true;
       }
     }
@@ -79,9 +80,12 @@ namespace MarketData.Utils
     }
     public void Add(String entry)
     {
-      if(null==streamWriter)return;
+      lock(this)
-      streamWriter.WriteLine(entry);
+      {
-      streamWriter.Flush();
+        if(null==streamWriter)return;
+        streamWriter.WriteLine(entry);
+        streamWriter.Flush();
+      }
     }
     public void Dispose()
     {