diff --git a/MarketDataLib/DataAccess/InsiderTransactionDA.cs b/MarketDataLib/DataAccess/InsiderTransactionDA.cs
index b4bb862..0c4df01 100644
--- a/MarketDataLib/DataAccess/InsiderTransactionDA.cs
+++ b/MarketDataLib/DataAccess/InsiderTransactionDA.cs
@@ -173,7 +173,7 @@ namespace MarketData.DataAccess
if(null!=sqlTransaction)sqlTransaction.Dispose();
}
}
- private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) //, MySqlTransaction sqlTransaction)
+ private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection)
{
StringBuilder sb = null;
String strQuery = null;
@@ -210,5 +210,45 @@ namespace MarketData.DataAccess
if(null!=sqlTransaction)sqlTransaction.Dispose();
}
}
+ ///
+ /// Delete InsiderTransaction for given symbol for all years>=given year.
+ ///
+ /// The symbol.
+ /// All years greater then or equal to the year.
+ public static bool DeleteInsiderTransactionsYearsGreaterEqual(String symbol, int yearsGreaterEqual)
+ {
+ MySqlConnection sqlConnection = null;
+ StringBuilder sb = null;
+ String strQuery = null;
+ MySqlTransaction sqlTransaction = null;
+
+ try
+ {
+ if (null == symbol) return false;
+ sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
+ sqlTransaction = sqlConnection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted);
+ sb = new StringBuilder();
+ sb.Append("DELETE FROM InsiderTransaction WHERE symbol=");
+ sb.Append("'").Append(symbol).Append("'");
+ sb.Append(" AND YEAR(filing_date)>=").Append(yearsGreaterEqual);
+ strQuery = sb.ToString();
+ MySqlCommand sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction);
+ sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
+ sqlCommand.ExecuteNonQuery();
+ sqlTransaction.Commit();
+ return true;
+ }
+ catch (Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,exception);
+ if (null != strQuery) MDTrace.WriteLine(LogLevel.DEBUG,"Query was " + strQuery);
+ return false;
+ }
+ finally
+ {
+ if(null!=sqlTransaction)sqlTransaction.Dispose();
+ if(null!=sqlConnection)sqlConnection.Close();
+ }
+ }
}
}
diff --git a/MarketDataLib/Helper/FundamentalMarketDataHelper.cs b/MarketDataLib/Helper/FundamentalMarketDataHelper.cs
index baaf7ae..3f4c002 100644
--- a/MarketDataLib/Helper/FundamentalMarketDataHelper.cs
+++ b/MarketDataLib/Helper/FundamentalMarketDataHelper.cs
@@ -11,7 +11,6 @@ namespace MarketData.Helper
{
public class FundamentalMarketDataHelper : MarketDataHelperBase
{
-// private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads;
private UpdateManager updateManager=new UpdateManager();
diff --git a/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs b/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs
index 16af8f2..f1bd217 100644
--- a/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs
+++ b/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs
@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
-using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
@@ -9,12 +8,23 @@ using MarketData.Utils;
namespace MarketData.Helper
{
+
+ public class InsiderTransactionThreadHelper : ThreadHelper
+ {
+ public InsiderTransactionThreadHelper(String symbol, ManualResetEvent resetEvent)
+ : base(symbol, resetEvent)
+ {
+ }
+ public int Year { get; set; }
+ }
+
public class InsiderTransactionMarketDataHelper
{
- private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads;
+ private static int MaxThreads = 10; // 10 threads avoids receiving HTTP Response 429 (Too many requests)
private static int WAIT_TIME_MS=500; // wait between request
private List symbols;
private int currentIndex = 0;
+ private UpdateManager UpdateManager = new UpdateManager();
public InsiderTransactionMarketDataHelper()
{
@@ -83,6 +93,45 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
}
}
+ public bool LoadInsiderTransactionsYear(List symbols,int year)
+ {
+ Profiler profiler=new Profiler();
+ try
+ {
+ this.symbols = symbols;
+ currentIndex = 0;
+
+ UpdateManager.Prepare("load_insider_transactions_year.txt", 7); // use max age 7 days
+ this.symbols=this.symbols.Except(new List(UpdateManager.Entries)).ToList();
+
+ while (true)
+ {
+ List queueSymbols = GetQueueSymbols();
+ if (null == queueSymbols || 0 == queueSymbols.Count) break;
+ ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
+ for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
+ {
+ resetEvents[eventIndex] = new ManualResetEvent(false);
+ }
+ for (int index = 0; index < queueSymbols.Count; index++)
+ {
+ InsiderTransactionThreadHelper threadHelper = new InsiderTransactionThreadHelper(queueSymbols[index], resetEvents[index]);
+ threadHelper.UpdateManager=UpdateManager;
+ threadHelper.Year=year;
+ ThreadPool.QueueUserWorkItem(ThreadPoolCallbackYearGreaterEqual, threadHelper);
+ try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // wait
+ }
+ MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete.");
+ WaitHandle.WaitAll(resetEvents);
+ }
+ return true;
+ }
+ finally
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End()));
+ }
+ }
+
private List GetQueueSymbols()
{
List queueSymbols = new List();
@@ -94,6 +143,7 @@ namespace MarketData.Helper
currentIndex = index;
return queueSymbols;
}
+
public void ThreadPoolCallback(Object threadHelperContext)
{
ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
@@ -102,6 +152,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
threadHelper.ResetEvent.Set();
}
+
public static void LoadInsiderTransactionsSymbolEx(String symbol)
{
symbol = symbol.ToUpper();
@@ -116,5 +167,31 @@ namespace MarketData.Helper
InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
}
+
+ public void ThreadPoolCallbackYearGreaterEqual(Object threadHelperContext)
+ {
+ InsiderTransactionThreadHelper threadHelper = (InsiderTransactionThreadHelper)threadHelperContext;
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
+ LoadInsiderTransactionsYearGreaterEqualEx(threadHelper.Symbol,threadHelper.Year);
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
+ threadHelper.UpdateManager.Add(threadHelper.Symbol);
+ threadHelper.ResetEvent.Set();
+ }
+
+ public static void LoadInsiderTransactionsYearGreaterEqualEx(String symbol, int year)
+ {
+ symbol = symbol.ToUpper();
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0} years>={1}", symbol,year));
+ InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactionsYear(symbol, year);
+ if (null == insiderTransactions || 0 == insiderTransactions.Count)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0} years>={1}", symbol,year));
+ return;
+ }
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1} years>={2}", insiderTransactions.Count, symbol, year));
+ InsiderTransactionDA.DeleteInsiderTransactionsYearsGreaterEqual(symbol, year);
+ InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions);
+ MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done.");
+ }
}
}
diff --git a/MarketDataLib/Helper/InsiderTransactionsParser.cs b/MarketDataLib/Helper/InsiderTransactionsParser.cs
index 316a200..2d6087c 100644
--- a/MarketDataLib/Helper/InsiderTransactionsParser.cs
+++ b/MarketDataLib/Helper/InsiderTransactionsParser.cs
@@ -6,12 +6,13 @@ using HtmlAgilityPack;
using MarketData.Utils;
using MarketData.MarketDataModel;
using MarketDataLib.Utility;
+using System.Collections.Concurrent;
namespace MarketData.Helper
{
public class InsiderTransactionsParser
{
- private static Dictionary transactionCodes=new Dictionary();
+ private static ConcurrentDictionary transactionCodes=new ConcurrentDictionary();
private static InsiderTransactionsParser instance=null;
private InsiderTransactionsParser()
{
@@ -26,6 +27,140 @@ namespace MarketData.Helper
return instance;
}
}
+
+ ///
+ /// Parse the SECFilings. Each SECFiling may contain a list of InsiderTransaction
+ ///
+ /// The SECFilings.
+ public InsiderTransactions Parse(SECFilings secFilings)
+ {
+ try
+ {
+ InsiderTransactions insiderTransactions=new InsiderTransactions();
+
+ if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
+ foreach(SECFiling secFiling in secFilings)
+ {
+ InsiderTransactions secFilinginsiderTransactions =
+ Parse(secFiling.FormText, secFiling.Symbol,
+ secFiling.SECAccessionNumber,
+ secFiling.Form, secFiling.FilingDate);
+ if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0)
+ insiderTransactions.AddRange(secFilinginsiderTransactions);
+ }
+ return insiderTransactions;
+ }
+ catch(Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
+ return new InsiderTransactions();
+ }
+ }
+
+ public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate)
+ {
+ try
+ {
+ InsiderTransactions insiderTransactions = new InsiderTransactions();
+ if(null==strHtml)return null;
+
+ List> dictionaryList=new List>();
+ String insiderName=null;
+
+ byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml);
+ MemoryStream memoryStream = new MemoryStream(streamBytes);
+ HtmlDocument htmlDocument = new HtmlDocument();
+ htmlDocument.Load(memoryStream);
+ HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
+
+ if(null==tables || tables.Count<5)return null;
+
+ HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
+ if(null==nameAndAddressTable)return null;
+ HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
+ if(nameAndAddressRows.Count<2)return null;
+ insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
+
+ for(int index=0;index dictionary=new Dictionary();
+ dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
+ dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
+ dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
+ dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
+ dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
+ dictionary.Add("Price",dataCollection[7].InnerHtml);
+ dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
+ dictionaryList.Add(dictionary);
+ }
+ }
+ }
+
+ foreach(Dictionary dictionary in dictionaryList)
+ {
+ String strItem=null;
+ InsiderTransaction insiderTransaction=new InsiderTransaction();
+ insiderTransaction.Symbol=symbol;
+ insiderTransaction.SECAccessionNumber=secAccessionNumber;
+ insiderTransaction.Form=form;
+ insiderTransaction.FilingDate=filingDate;
+ insiderTransaction.InsiderName=insiderName;
+ insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
+ insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
+ if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
+ strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
+ if(String.IsNullOrEmpty(strItem))continue;
+ insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
+ insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
+ insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
+ insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString();
+ insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
+ String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary);
+ if(null==acquiredOrDisposed)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber));
+ continue;
+ }
+ if(acquiredOrDisposed.Equals("D"))
+ {
+ insiderTransaction.NumberOrValueAcquiredDisposed*=-1;
+ }
+ insiderTransactions.Add(insiderTransaction);
+ }
+ return insiderTransactions;
+ }
+ catch(Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString()));
+ return null;
+ }
+ }
+
+ private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
+ {
+ if(null==tables)return null;
+ for(int index=0;index sections)
{
String item=GetFirstSection(sections);
+ if(null==item)item="";
+ item=item.ToUpper();
if(item.Equals("D"))return "Direct Ownership";
else if(item.Equals("I"))return "Indirect Ownership";
return "";
}
+
+ private String GetAcquiredOrDisposed(Dictionary dictionary)
+ {
+ if(!dictionary.ContainsKey("Acquired or Disposed"))return null;
+ List section=Sections.GetSections(dictionary["Acquired or Disposed"]);
+ if(null==section||0==section.Count)return null;
+ String acquiredOrDisposed=section[0];
+ if(null==acquiredOrDisposed)return null;
+ return acquiredOrDisposed.Trim().ToUpper();
+ }
+
private String GetFirstSection(List sections)
{
if(null==sections || 0==sections.Count)return "";
return sections[0];
}
+
private String CombineSections(List sections)
{
StringBuilder sb=new StringBuilder();
@@ -62,135 +212,29 @@ namespace MarketData.Helper
}
return sb.ToString();
}
- public InsiderTransactions Parse(SECFilings secFilings)
- {
- lock(instance)
- {
- try
- {
- InsiderTransactions insiderTransactions=new InsiderTransactions();
-
- if(null==secFilings || 0==secFilings.Count)return insiderTransactions;
- foreach(SECFiling secFiling in secFilings)
- {
- List> dictionaryList=new List>();
- String insiderName=null;
- String relationshipOfReportingPerson=null;
-
- byte[] streamBytes = Encoding.ASCII.GetBytes(secFiling.FormText);
- MemoryStream memoryStream = new MemoryStream(streamBytes);
- HtmlDocument htmlDocument = new HtmlDocument();
- htmlDocument.Load(memoryStream);
- HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table");
-
- if(null==tables || tables.Count<5)continue;
-
- HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person");
- if(null==nameAndAddressTable)continue;
- HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr");
- if(nameAndAddressRows.Count<2)continue;
- insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
-
-
- //HtmlNodeCollection nameAndAddressRows = tables[4].SelectNodes(".//tr");
- //if(nameAndAddressRows.Count<10)continue;
- //insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText);
- //relationshipOfReportingPerson=Utility.RemoveHtml(nameAndAddressRows[9].InnerText);
-
- for(int index=0;index dictionary=new Dictionary();
- dictionary.Add("Title of Security",dataCollection[0].InnerHtml);
- dictionary.Add("Transaction Date",dataCollection[1].InnerHtml);
- dictionary.Add("Transaction Code",dataCollection[3].InnerHtml);
- dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml);
- dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml);
- dictionary.Add("Price",dataCollection[7].InnerHtml);
- dictionary.Add("Ownership Form",dataCollection[9].InnerHtml);
- dictionaryList.Add(dictionary);
- }
- }
- }
-
- foreach(Dictionary dictionary in dictionaryList)
- {
- String strItem=null;
- InsiderTransaction insiderTransaction = new InsiderTransaction();
- insiderTransaction.Symbol=secFiling.Symbol;
- insiderTransaction.SECAccessionNumber=secFiling.SECAccessionNumber;
- insiderTransaction.FormRowNumber=(secFiling.Sequence+1).ToString();
- insiderTransaction.Form=secFiling.Form;
- insiderTransaction.FilingDate=secFiling.FilingDate;
- insiderTransaction.InsiderName=insiderName;
- insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"]));
- insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"]));
- if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue;
- strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"]));
- if(String.IsNullOrEmpty(strItem))continue;
- insiderTransaction.TransactionDate=Utility.ParseDate(strItem);
- insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION;
- insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"])));
- insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"])));
- insiderTransactions.Add(insiderTransaction);
- }
- }
- return insiderTransactions;
- }
- catch(Exception exception)
- {
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString()));
- return new InsiderTransactions();
- }
- }
- }
-
- private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith)
- {
- if(null==tables)return null;
- for(int index=0;index 0) continue;
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
@@ -1320,13 +1318,114 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
+ ///
+ /// Get InsiderTransactions for years greater than or equal to specified year.
+ /// For example. If 2023 is specified then get all for years>=2023
+ ///
+ /// The symbol.
+ /// the new y-coordinate.
+ public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
+ {
+ MemoryStream memoryStream = null;
+ HttpNetResponse httpNetResponse=null;
+ String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
+ int maxFilings=120;
+ bool continuationFlag=true;
+ int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
+
+ try
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
+ DateGenerator dateGenerator=new DateGenerator();
+ String cik=PricingDA.GetCIKForSymbol(symbol);
+ if(null==cik)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
+ return null;
+ }
+ SECFilings secFilings = new SECFilings();
+ StringBuilder sb = new StringBuilder();
+ String strRequest;
+ sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
+ strRequest = sb.ToString();
+ WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
+ httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ if(!httpNetResponse.Success)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+ return null;
+ }
+ byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
+ memoryStream = new MemoryStream(streamBytes);
+ HtmlDocument htmlDocument = new HtmlDocument();
+ htmlDocument.Load(memoryStream);
+ HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
+ if (null == tables || tables.Count < 1) return null;
+ HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
+ for (int row = 0; row < rows.Count && continuationFlag; row ++)
+ {
+ try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
+ HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
+ if (null != headerColumns && headerColumns.Count > 0) continue;
+ HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
+ if (null == dataColumns || 5 != dataColumns.Count) continue;
+ HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
+ if (null == nodes || 0 == nodes.Count) continue;
+ String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
+ SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
+ for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
+ {
+ SECFiling innerFiling = innerCollection[index];
+ if(innerFiling.FilingDate.YearinnerFiling.Description.StartsWith(x)))continue;
+ secFilings.Add(innerFiling);
+ }
+ }
+ return InsiderTransactionsParser.GetInstance().Parse(secFilings);
+ }
+ catch (Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,exception);
+ return null;
+ }
+ finally
+ {
+ if (null != memoryStream) memoryStream.Close();
+ if(null!=httpNetResponse)httpNetResponse.Dispose();
+ }
+ }
// ***************************************************************************************************************************************************************************************
// ********************************************************************** S E C F I L I N G S - S E C . G O V ********************************************************************
// ***************************************************************************************************************************************************************************************
+ public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
+ {
+ try
+ {
+ String cik=PricingDA.GetCIKForSymbol(symbol);
+ if(null==cik)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
+ return null;
+ }
+ return GetSECFilings(symbol,cik,maxFilings);
+ }
+ catch(Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,exception);
+ return null; }
+ }
+
public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
{
MemoryStream memoryStream = null;
HttpNetResponse httpNetResponse=null;
+ int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
+
try
{
SECFilings secFilings = new SECFilings();
@@ -1357,6 +1456,7 @@ namespace MarketData.Helper
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
+ try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;}
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
{
@@ -1384,11 +1484,12 @@ namespace MarketData.Helper
SECFilings secFilings = new SECFilings();
String secFilingDocument = null;
HttpNetResponse httpNetResponse=null;
+ int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
+
try
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl));
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
- try { Thread.Sleep(1000); }catch(Exception) { ;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
@@ -1409,11 +1510,11 @@ namespace MarketData.Helper
String formName = secNodes[0].InnerText;
formName = formName.Replace("\n", "");
formName = formName.Trim();
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName));
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
httpNetResponse.Dispose();
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Examining form {0} with {1} entries",formName,rows.Count));
// go through the rows in the table file, searching for the different submission form content
int sequence = 0;
for (int row = 0; row < rows.Count; row++)
@@ -1425,21 +1526,22 @@ namespace MarketData.Helper
String document = dataColumns[2].InnerText.Trim();
String type = dataColumns[3].InnerText.Trim();
if(null!=type && type.Equals("GRAPHIC"))continue;
-// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
+// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
if (document.Contains(".htm"))
{
+
HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
-// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl));
- try{Thread.Sleep(1000);}catch(Exception){;}
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Requesting {0}",secFilingDocumentXmlUrl));
+ try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
continue;
}
- secFilingDocument =httpNetResponse.ResponseString;
+ secFilingDocument = httpNetResponse.ResponseString;
if (null == secFilingDocument) continue;
SECFiling secFiling = new SECFiling();
secFiling.SECAccessionNumber = secAccessionNumber;
diff --git a/MarketDataLib/Helper/ThreadHelper.cs b/MarketDataLib/Helper/ThreadHelper.cs
index 03b4bac..78d9ff2 100644
--- a/MarketDataLib/Helper/ThreadHelper.cs
+++ b/MarketDataLib/Helper/ThreadHelper.cs
@@ -1,8 +1,4 @@
using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
using System.Threading;
using MarketData.Utils;
diff --git a/MarketDataLib/Utility/UpdateManager.cs b/MarketDataLib/Utility/UpdateManager.cs
index c95bd26..ec50ac8 100644
--- a/MarketDataLib/Utility/UpdateManager.cs
+++ b/MarketDataLib/Utility/UpdateManager.cs
@@ -16,28 +16,28 @@ namespace MarketData.Utils
public UpdateManager()
{
}
- public bool Prepare(String strPathFileName)
+ public bool Prepare(String strPathFileName, int maxAgeDays=5)
{
try
{
String currentWorkingDirectory=Directory.GetCurrentDirectory();
- if(!File.Exists(strPathFileName)||IsExpired(strPathFileName))
+ if(!File.Exists(strPathFileName)||IsExpired(strPathFileName, maxAgeDays))
{
if(File.Exists(strPathFileName))File.Delete(strPathFileName);
fileStream=new FileStream(strPathFileName,FileMode.Create);
streamWriter=new StreamWriter(fileStream);
- Console.WriteLine(String.Format("Creating session file:{0}",strPathFileName));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Creating session file:{0}",strPathFileName));
}
else
{
- FileStream fileStream=new FileStream(strPathFileName,FileMode.Open);
+ FileStream fileStream=new FileStream(strPathFileName,FileMode.Open,FileAccess.ReadWrite,FileShare.Read);
StreamReader streamReader=new StreamReader(fileStream);
String strLine=null;
while(null!=(strLine=streamReader.ReadLine()))
{
if(!entries.ContainsKey(strLine))entries.Add(strLine,strLine);
}
- Console.WriteLine(String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName));
streamReader.Close();
streamReader.Dispose();
fileStream.Close();
@@ -49,27 +49,28 @@ namespace MarketData.Utils
}
catch(Exception exception)
{
- Console.WriteLine(String.Format("Exception:{0}",exception));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
return false;
}
}
- private bool IsExpired(String strPathFileName)
+ private bool IsExpired(String strPathFileName, int maxAgeDays)
{
try
{
DateGenerator dateGenerator=new DateGenerator();
DateTime creationTime=File.GetCreationTime(strPathFileName);
int daysElapsed=Math.Abs(dateGenerator.DaysBetweenActual(creationTime,DateTime.Now));
- if(daysElapsed>5)
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is {1} days(s) old.",strPathFileName,daysElapsed));
+ if(daysElapsed>maxAgeDays)
{
- Console.WriteLine(String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed));
return true;
}
return false;
}
catch(Exception exception)
{
- Console.WriteLine(String.Format("Exception:{0}",exception));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception));
return true;
}
}
@@ -79,9 +80,12 @@ namespace MarketData.Utils
}
public void Add(String entry)
{
- if(null==streamWriter)return;
- streamWriter.WriteLine(entry);
- streamWriter.Flush();
+ lock(this)
+ {
+ if(null==streamWriter)return;
+ streamWriter.WriteLine(entry);
+ streamWriter.Flush();
+ }
}
public void Dispose()
{