marketdata/MarketDataLib/Helper/InsiderTransactionsParser.cs

using System;
using System.Text;
using System.Collections.Generic;
using System.IO;
using HtmlAgilityPack;
using MarketData.Utils;
using MarketData.MarketDataModel;
using MarketDataLib.Utility;

namespace MarketData.Helper
{
  public class InsiderTransactionsParser
  {
    private static Dictionary<String,String> transactionCodes=new Dictionary<String,String>();
    private static InsiderTransactionsParser instance=null;
    private InsiderTransactionsParser()
    {                               
      BuildTransactionCodes();
    }

    public static InsiderTransactionsParser GetInstance()
    {
      lock(typeof(InsiderTransactionsParser))
      {
        if(null==instance)instance=new InsiderTransactionsParser();
        return instance; 
      }
    }
    private String ApplyNameCase(String name)
    {
      StringBuilder sb=new StringBuilder();
      name=name.ToLower();
      String[] parts=name.Split(' ');
      for(int index=0;index<parts.Length;index++)
      {
        String item=parts[index];
        if(item.Length>1)sb.Append(item.Substring(0,1).ToUpper()+item.Substring(1));
        else sb.Append(item.ToUpper());
        if(index<parts.Length-1)sb.Append(" ");
      }
      return sb.ToString();
    }
    private String GetOwnershipForm(List<String> sections)
    {
      String item=GetFirstSection(sections);
      if(item.Equals("D"))return "Direct Ownership";
      else if(item.Equals("I"))return "Indirect Ownership";
      return "";
    }
    private String GetFirstSection(List<String> sections)
    {
      if(null==sections || 0==sections.Count)return "";
      return sections[0];
    }
    private String CombineSections(List<String> sections)
    {
      StringBuilder sb=new StringBuilder();
      if(null==sections || 0==sections.Count)return "";
      foreach(String section in sections)
      {
        sb.Append(section);
      }
      return sb.ToString();
    }
    public InsiderTransactions Parse(SECFilings secFilings)
    {
      lock(instance)
      {
        try
        {
          InsiderTransactions insiderTransactions=new InsiderTransactions();

          if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
          foreach(SECFiling secFiling in secFilings)
          {
            List<Dictionary<String,String>> dictionaryList=new List<Dictionary<String,String>>();
            String insiderName=null;
            String relationshipOfReportingPerson=null;

            byte[] streamBytes = Encoding.ASCII.GetBytes(secFiling.FormText);
            MemoryStream memoryStream = new MemoryStream(streamBytes);
            HtmlDocument htmlDocument = new HtmlDocument();
            htmlDocument.Load(memoryStream);
            HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");

            if(null==tables || tables.Count<5)continue;

            HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
            if(null==nameAndAddressTable)continue;
            HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
            if(nameAndAddressRows.Count<2)continue;
            insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);


            //HtmlNodeCollection nameAndAddressRows = tables[4].SelectNodes(".//tr");
            //if(nameAndAddressRows.Count<10)continue;
            //insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
            //relationshipOfReportingPerson=Utility.RemoveHtml(nameAndAddressRows[9].InnerText);

            for(int index=0;index<tables.Count;index++)
            {
              HtmlNodeCollection nodeCollection = tables[index].SelectNodes(".//tr");
              foreach(HtmlNode node in nodeCollection)
              {
                HtmlNodeCollection dataCollection = node.SelectNodes(".//td");
                if(null==dataCollection || 0==dataCollection.Count)continue;
                if(11==dataCollection.Count)
                {
                  Dictionary<String,String> dictionary=new Dictionary<String,String>();
                  dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
                  dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
                  dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
                  dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
                  dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
                  dictionary.Add("Price",dataCollection[7].InnerHtml);
                  dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
                  dictionaryList.Add(dictionary);
                }
              }
            }

            foreach(Dictionary<String,String> dictionary in dictionaryList)
            {
              String strItem=null;
              InsiderTransaction insiderTransaction = new InsiderTransaction();
              insiderTransaction.Symbol=secFiling.Symbol;
              insiderTransaction.SECAccessionNumber=secFiling.SECAccessionNumber;
              insiderTransaction.FormRowNumber=(secFiling.Sequence+1).ToString();
              insiderTransaction.Form=secFiling.Form;
              insiderTransaction.FilingDate=secFiling.FilingDate;
              insiderTransaction.InsiderName=insiderName;
              insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
              insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
              if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
              strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
              if(String.IsNullOrEmpty(strItem))continue;
              insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
              insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
              insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
              insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
              insiderTransactions.Add(insiderTransaction); 
            }
          }
          return insiderTransactions;
        }
        catch(Exception exception)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
          return new InsiderTransactions();
        }
      }
    }

    private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
    {
      if(null==tables)return null;
      for(int index=0;index<tables.Count;index++)
      {
        HtmlNodeCollection nameAndAddressRows = tables[index].SelectNodes(".//tr");
        for(int itemIndex=0;itemIndex<nameAndAddressRows.Count;itemIndex++)
        {
          String marker=nameAndAddressRows[itemIndex].InnerText;
          if(marker.StartsWith(startsWith))
          {
            return tables[index];
          }
        }
      }
      return null;
    }

    private void BuildTransactionCodes()
    {
      transactionCodes.Add("P","Open market or private purchase of non-derivative or derivative security");
      transactionCodes.Add("S","Open market or private sale of non-derivative or derivative security");
      transactionCodes.Add("V","Transaction voluntarily reported earlier than required");
      transactionCodes.Add("A","Grant, award or other acquisition pursuant to Rule 16b-3(d)");
      transactionCodes.Add("D","Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e)");
      transactionCodes.Add("F","Payment of exercise price or tax liability by delivering or withholding securities incident to the receipt, exercise or vesting of a security issued in accordance with Rule 16b-3");
      transactionCodes.Add("I","Discretionary transaction in accordance with Rule 16b-3(f) resulting in acquisition or disposition of issuer securities");
      transactionCodes.Add("M","Exercise or conversion of derivative security exempted pursuant to Rule 16b-3");
      transactionCodes.Add("C","Conversion of derivative security");
      transactionCodes.Add("E","Expiration of short derivative position");
      transactionCodes.Add("H","Expiration (or cancellation) of long derivative position with value received");
      transactionCodes.Add("O","Exercise of out-of-the-money derivative security");
      transactionCodes.Add("X","Exercise of in-the-money or at-the-money derivative security");
      transactionCodes.Add("G","Bona fide gift");
      transactionCodes.Add("L","Small acquisition under Rule 16a-6");
      transactionCodes.Add("W","Acquisition or disposition by will or the laws of descent and distribution");
      transactionCodes.Add("Z","Deposit into or withdrawal from voting trust");
      transactionCodes.Add("J","Other acquisition or disposition (describe transaction)");
      transactionCodes.Add("K","Transaction in equity swap or instrument with similar characteristics");
      transactionCodes.Add("U","Disposition pursuant to a tender of shares in a change of control transaction");
    }
  }
}