marketdata/MarketDataLib/Helper/InsiderTransactionsParser.cs

using System;
using System.Text;
using System.Collections.Generic;
using System.IO;
using HtmlAgilityPack;
using MarketData.Utils;
using MarketData.MarketDataModel;
using MarketDataLib.Utility;
using System.Collections.Concurrent;

namespace MarketData.Helper
{
  public class InsiderTransactionsParser
  {
    private static ConcurrentDictionary<String,String> transactionCodes=new ConcurrentDictionary<String,String>();
    private static InsiderTransactionsParser instance=null;
    private InsiderTransactionsParser()
    {                               
      BuildTransactionCodes();
    }

    public static InsiderTransactionsParser GetInstance()
    {
      lock(typeof(InsiderTransactionsParser))
      {
        if(null==instance)instance=new InsiderTransactionsParser();
        return instance; 
      }
    }

    /// <summary>
    /// Parse the SECFilings.  Each SECFiling may contain a list of InsiderTransaction
    /// </summary>
    /// <param name="secFilings">The SECFilings.</param>
    public InsiderTransactions Parse(SECFilings secFilings)
    {
      try
      {
        InsiderTransactions insiderTransactions=new InsiderTransactions();

        if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
        foreach(SECFiling secFiling in secFilings)
        {
          InsiderTransactions secFilinginsiderTransactions =
            Parse(secFiling.FormText, secFiling.Symbol,
              secFiling.SECAccessionNumber,
                secFiling.Form, secFiling.FilingDate);
          if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0)
           insiderTransactions.AddRange(secFilinginsiderTransactions);
        }
        return insiderTransactions;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
        return new InsiderTransactions();
      }
    }

    public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate)
    {
      try
      {
        InsiderTransactions insiderTransactions = new InsiderTransactions();
        if(null==strHtml)return null;

        List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
        String insiderName=null;

        byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml);
        MemoryStream memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");

        if(null==tables || tables.Count<5)return null;

        HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
        if(null==nameAndAddressTable)return null;
        HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
        if(nameAndAddressRows.Count<2)return null;
        insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);

        for(int index=0;index<tables.Count;index++)
        {
          HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
          foreach(HtmlNode node in nodeCollection)
          {
            HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
            if(null==dataCollection || 0==dataCollection.Count)continue;
            if(11==dataCollection.Count)
            {
              Dictionary<String,String> dictionary=new Dictionary<String,String>();
              dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
              dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
              dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
              dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
              dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
              dictionary.Add("Price",dataCollection[7].InnerHtml);
              dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
              dictionaryList.Add(dictionary);
            }
          }
        }

        int sequenceNumber=0;
        foreach(Dictionary<String,String> dictionary in dictionaryList)
        {
          String strItem=null;
          InsiderTransaction insiderTransaction=new InsiderTransaction();
          insiderTransaction.Symbol=symbol;
          insiderTransaction.SECAccessionNumber=secAccessionNumber;
          insiderTransaction.Form=form;
          insiderTransaction.FilingDate=filingDate;
          insiderTransaction.InsiderName=insiderName;
          insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
          insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
          if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
          strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
          if(String.IsNullOrEmpty(strItem))continue;
          insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
          insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
          insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
//          insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString();
          insiderTransaction.FormRowNumber=(++sequenceNumber).ToString();
          insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
          String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary);
          if(null==acquiredOrDisposed)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber));
            continue;
          }
          if(acquiredOrDisposed.Equals("D"))
          {
            insiderTransaction.NumberOrValueAcquiredDisposed*=-1;
          }
          insiderTransactions.Add(insiderTransaction);
        }
        return insiderTransactions;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString()));
        return null;
      }
    }

    private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
    {
      if(null==tables)return null;
      for(int index=0;index<tables.Count;index++)
      {
        HtmlNodeCollection collection = tables[index].SelectNodes(".//tr");
        if(null==collection)continue;
        for(int itemIndex=0;itemIndex<collection.Count;itemIndex++)
        {
          String marker=collection[itemIndex].InnerText;
          if(null!=marker && marker.StartsWith(startsWith))
          {
            return tables[index];
          }
        }
      }
      return null;
    }

    private String ApplyNameCase(String name)
    {
      StringBuilder sb=new StringBuilder();
      name=name.ToLower();
      String[] parts=name.Split(' ');
      for(int index=0;index<parts.Length;index++)
      {
        String item=parts[index];
        if(item.Length>1)sb.Append(item.Substring(0,1).ToUpper()+item.Substring(1));
        else sb.Append(item.ToUpper());
        if(index<parts.Length-1)sb.Append(" ");
      }
      return sb.ToString();
    }

    private String GetOwnershipForm(List<String> sections)
    {
      String item=GetFirstSection(sections);
      if(null==item)item="";
      item=item.ToUpper();
      if(item.Equals("D"))return "Direct Ownership";
      else if(item.Equals("I"))return "Indirect Ownership";
      return "";
    }

    private String GetAcquiredOrDisposed(Dictionary<String,String> dictionary)
    {
      if(!dictionary.ContainsKey("Acquired or Disposed"))return null;
      List<String> section=Sections.GetSections(dictionary["Acquired or Disposed"]);
      if(null==section||0==section.Count)return null;
      String acquiredOrDisposed=section[0];
      if(null==acquiredOrDisposed)return null;
      return acquiredOrDisposed.Trim().ToUpper();
    }

    private String GetFirstSection(List<String> sections)
    {
      if(null==sections || 0==sections.Count)return "";
      return sections[0];
    }

    private String CombineSections(List<String> sections)
    {
      StringBuilder sb=new StringBuilder();
      if(null==sections || 0==sections.Count)return "";
      foreach(String section in sections)
      {
        sb.Append(section);
      }
      return sb.ToString();
    }

    private void BuildTransactionCodes()
    {
      transactionCodes.TryAdd("P","Open market or private purchase of non-derivative or derivative security");
      transactionCodes.TryAdd("S","Open market or private sale of non-derivative or derivative security");
      transactionCodes.TryAdd("V","Transaction voluntarily reported earlier than required");
      transactionCodes.TryAdd("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
      transactionCodes.TryAdd("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
      transactionCodes.TryAdd("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
      transactionCodes.TryAdd("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
      transactionCodes.TryAdd("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
      transactionCodes.TryAdd("C","Conversion of derivative security");
      transactionCodes.TryAdd("E","Expiration of short derivative position");
      transactionCodes.TryAdd("H","Expiration (or cancellation) of long derivative position with value received");
      transactionCodes.TryAdd("O","Exercise of out-of-the-money derivative security");
      transactionCodes.TryAdd("X","Exercise of in-the-money or at-the-money derivative security");
      transactionCodes.TryAdd("G","Bona fide gift");
      transactionCodes.TryAdd("L","Small acquisition under Rule 16a-6");
      transactionCodes.TryAdd("W","Acquisition or disposition by will or the laws of descent and distribution");
      transactionCodes.TryAdd("Z","Deposit into or withdrawal from voting trust");
      transactionCodes.TryAdd("J","Other acquisition or disposition (describe transaction)");
      transactionCodes.TryAdd("K","Transaction in equity swap or instrument with similar characteristics");
      transactionCodes.TryAdd("U","Disposition pursuant to a tender of shares in a change of control transaction");
    }
  }
}