From 661bd3d7ec0beb0fa2e4839ea4ca82258a7c0c5d Mon Sep 17 00:00:00 2001 From: Sean Kessler Date: Mon, 26 Feb 2024 19:48:35 -0500 Subject: [PATCH] InsiderTransactions refactorings. --- .../DataAccess/InsiderTransactionDA.cs | 42 ++- .../Helper/FundamentalMarketDataHelper.cs | 1 - .../InsiderTransactionMarketDataHelper.cs | 81 ++++- .../Helper/InsiderTransactionsParser.cs | 298 ++++++++++-------- MarketDataLib/Helper/MarketDataHelper.cs | 126 +++++++- MarketDataLib/Helper/ThreadHelper.cs | 4 - MarketDataLib/Utility/UpdateManager.cs | 30 +- 7 files changed, 422 insertions(+), 160 deletions(-) diff --git a/MarketDataLib/DataAccess/InsiderTransactionDA.cs b/MarketDataLib/DataAccess/InsiderTransactionDA.cs index b4bb862..0c4df01 100644 --- a/MarketDataLib/DataAccess/InsiderTransactionDA.cs +++ b/MarketDataLib/DataAccess/InsiderTransactionDA.cs @@ -173,7 +173,7 @@ namespace MarketData.DataAccess if(null!=sqlTransaction)sqlTransaction.Dispose(); } } - private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) //, MySqlTransaction sqlTransaction) + private static bool DeleteInsiderTransactions(InsiderTransactions insiderTransactions, MySqlConnection sqlConnection) { StringBuilder sb = null; String strQuery = null; @@ -210,5 +210,45 @@ namespace MarketData.DataAccess if(null!=sqlTransaction)sqlTransaction.Dispose(); } } + /// + /// Delete InsiderTransaction for given symbol for all years>=given year. + /// + /// The symbol. + /// All years greater then or equal to the year. + public static bool DeleteInsiderTransactionsYearsGreaterEqual(String symbol, int yearsGreaterEqual) + { + MySqlConnection sqlConnection = null; + StringBuilder sb = null; + String strQuery = null; + MySqlTransaction sqlTransaction = null; + + try + { + if (null == symbol) return false; + sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data")); + sqlTransaction = sqlConnection.BeginTransaction(System.Data.IsolationLevel.ReadCommitted); + sb = new StringBuilder(); + sb.Append("DELETE FROM InsiderTransaction WHERE symbol="); + sb.Append("'").Append(symbol).Append("'"); + sb.Append(" AND YEAR(filing_date)>=").Append(yearsGreaterEqual); + strQuery = sb.ToString(); + MySqlCommand sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); + sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT; + sqlCommand.ExecuteNonQuery(); + sqlTransaction.Commit(); + return true; + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception); + if (null != strQuery) MDTrace.WriteLine(LogLevel.DEBUG,"Query was " + strQuery); + return false; + } + finally + { + if(null!=sqlTransaction)sqlTransaction.Dispose(); + if(null!=sqlConnection)sqlConnection.Close(); + } + } } } diff --git a/MarketDataLib/Helper/FundamentalMarketDataHelper.cs b/MarketDataLib/Helper/FundamentalMarketDataHelper.cs index baaf7ae..3f4c002 100644 --- a/MarketDataLib/Helper/FundamentalMarketDataHelper.cs +++ b/MarketDataLib/Helper/FundamentalMarketDataHelper.cs @@ -11,7 +11,6 @@ namespace MarketData.Helper { public class FundamentalMarketDataHelper : MarketDataHelperBase { -// private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads; private static int MaxThreads = 5; //(int)ThreadHelperEnum.MaxThreads; private UpdateManager updateManager=new UpdateManager(); diff --git a/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs b/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs index 16af8f2..f1bd217 100644 --- a/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs +++ b/MarketDataLib/Helper/InsiderTransactionMarketDataHelper.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using MarketData.MarketDataModel; using MarketData.DataAccess; @@ -9,12 +8,23 @@ using MarketData.Utils; namespace MarketData.Helper { + + public class InsiderTransactionThreadHelper : ThreadHelper + { + public InsiderTransactionThreadHelper(String symbol, ManualResetEvent resetEvent) + : base(symbol, resetEvent) + { + } + public int Year { get; set; } + } + public class InsiderTransactionMarketDataHelper { - private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads; + private static int MaxThreads = 10; // 10 threads avoids receiving HTTP Response 429 (Too many requests) private static int WAIT_TIME_MS=500; // wait between request private List symbols; private int currentIndex = 0; + private UpdateManager UpdateManager = new UpdateManager(); public InsiderTransactionMarketDataHelper() { @@ -83,6 +93,45 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End())); } } + public bool LoadInsiderTransactionsYear(List symbols,int year) + { + Profiler profiler=new Profiler(); + try + { + this.symbols = symbols; + currentIndex = 0; + + UpdateManager.Prepare("load_insider_transactions_year.txt", 7); // use max age 7 days + this.symbols=this.symbols.Except(new List(UpdateManager.Entries)).ToList(); + + while (true) + { + List queueSymbols = GetQueueSymbols(); + if (null == queueSymbols || 0 == queueSymbols.Count) break; + ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count]; + for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++) + { + resetEvents[eventIndex] = new ManualResetEvent(false); + } + for (int index = 0; index < queueSymbols.Count; index++) + { + InsiderTransactionThreadHelper threadHelper = new InsiderTransactionThreadHelper(queueSymbols[index], resetEvents[index]); + threadHelper.UpdateManager=UpdateManager; + threadHelper.Year=year; + ThreadPool.QueueUserWorkItem(ThreadPoolCallbackYearGreaterEqual, threadHelper); + try { Thread.Sleep(WAIT_TIME_MS); } catch(Exception) { ;} // wait + } + MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions, waiting for queued items to complete."); + WaitHandle.WaitAll(resetEvents); + } + return true; + } + finally + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadInsiderTransactions]End, total took {0}(ms)",profiler.End())); + } + } + private List GetQueueSymbols() { List queueSymbols = new List(); @@ -94,6 +143,7 @@ namespace MarketData.Helper currentIndex = index; return queueSymbols; } + public void ThreadPoolCallback(Object threadHelperContext) { ThreadHelper threadHelper = (ThreadHelper)threadHelperContext; @@ -102,6 +152,7 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol)); threadHelper.ResetEvent.Set(); } + public static void LoadInsiderTransactionsSymbolEx(String symbol) { symbol = symbol.ToUpper(); @@ -116,5 +167,31 @@ namespace MarketData.Helper InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions); MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done."); } + + public void ThreadPoolCallbackYearGreaterEqual(Object threadHelperContext) + { + InsiderTransactionThreadHelper threadHelper = (InsiderTransactionThreadHelper)threadHelperContext; + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol)); + LoadInsiderTransactionsYearGreaterEqualEx(threadHelper.Symbol,threadHelper.Year); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions Thread {0} ended for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol)); + threadHelper.UpdateManager.Add(threadHelper.Symbol); + threadHelper.ResetEvent.Set(); + } + + public static void LoadInsiderTransactionsYearGreaterEqualEx(String symbol, int year) + { + symbol = symbol.ToUpper(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load insider transactions for {0} years>={1}", symbol,year)); + InsiderTransactions insiderTransactions = MarketDataHelper.GetInsiderTransactionsYear(symbol, year); + if (null == insiderTransactions || 0 == insiderTransactions.Count) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No insider transactions for {0} years>={1}", symbol,year)); + return; + } + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Insider Transactions, Saving {0} records for {1} years>={2}", insiderTransactions.Count, symbol, year)); + InsiderTransactionDA.DeleteInsiderTransactionsYearsGreaterEqual(symbol, year); + InsiderTransactionDA.InsertInsiderTransactions(insiderTransactions); + MDTrace.WriteLine(LogLevel.DEBUG,"Insider Transactions - Done."); + } } } diff --git a/MarketDataLib/Helper/InsiderTransactionsParser.cs b/MarketDataLib/Helper/InsiderTransactionsParser.cs index 316a200..2d6087c 100644 --- a/MarketDataLib/Helper/InsiderTransactionsParser.cs +++ b/MarketDataLib/Helper/InsiderTransactionsParser.cs @@ -6,12 +6,13 @@ using HtmlAgilityPack; using MarketData.Utils; using MarketData.MarketDataModel; using MarketDataLib.Utility; +using System.Collections.Concurrent; namespace MarketData.Helper { public class InsiderTransactionsParser { - private static Dictionary transactionCodes=new Dictionary(); + private static ConcurrentDictionary transactionCodes=new ConcurrentDictionary(); private static InsiderTransactionsParser instance=null; private InsiderTransactionsParser() { @@ -26,6 +27,140 @@ namespace MarketData.Helper return instance; } } + + /// + /// Parse the SECFilings. Each SECFiling may contain a list of InsiderTransaction + /// + /// The SECFilings. + public InsiderTransactions Parse(SECFilings secFilings) + { + try + { + InsiderTransactions insiderTransactions=new InsiderTransactions(); + + if(null==secFilings || 0==secFilings.Count)return insiderTransactions; + foreach(SECFiling secFiling in secFilings) + { + InsiderTransactions secFilinginsiderTransactions = + Parse(secFiling.FormText, secFiling.Symbol, + secFiling.SECAccessionNumber, + secFiling.Form, secFiling.FilingDate); + if(null!=secFilinginsiderTransactions && secFilinginsiderTransactions.Count>0) + insiderTransactions.AddRange(secFilinginsiderTransactions); + } + return insiderTransactions; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString())); + return new InsiderTransactions(); + } + } + + public InsiderTransactions Parse(String strHtml,String symbol, String secAccessionNumber, String form, DateTime filingDate) + { + try + { + InsiderTransactions insiderTransactions = new InsiderTransactions(); + if(null==strHtml)return null; + + List> dictionaryList=new List>(); + String insiderName=null; + + byte[] streamBytes = Encoding.ASCII.GetBytes(strHtml); + MemoryStream memoryStream = new MemoryStream(streamBytes); + HtmlDocument htmlDocument = new HtmlDocument(); + htmlDocument.Load(memoryStream); + HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table"); + + if(null==tables || tables.Count<5)return null; + + HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person"); + if(null==nameAndAddressTable)return null; + HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr"); + if(nameAndAddressRows.Count<2)return null; + insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText); + + for(int index=0;index dictionary=new Dictionary(); + dictionary.Add("Title of Security",dataCollection[0].InnerHtml); + dictionary.Add("Transaction Date",dataCollection[1].InnerHtml); + dictionary.Add("Transaction Code",dataCollection[3].InnerHtml); + dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml); + dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml); + dictionary.Add("Price",dataCollection[7].InnerHtml); + dictionary.Add("Ownership Form",dataCollection[9].InnerHtml); + dictionaryList.Add(dictionary); + } + } + } + + foreach(Dictionary dictionary in dictionaryList) + { + String strItem=null; + InsiderTransaction insiderTransaction=new InsiderTransaction(); + insiderTransaction.Symbol=symbol; + insiderTransaction.SECAccessionNumber=secAccessionNumber; + insiderTransaction.Form=form; + insiderTransaction.FilingDate=filingDate; + insiderTransaction.InsiderName=insiderName; + insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"])); + insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"])); + if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue; + strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"])); + if(String.IsNullOrEmpty(strItem))continue; + insiderTransaction.TransactionDate=Utility.ParseDate(strItem); + insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION; + insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"]))); + insiderTransaction.FormRowNumber=((decimal)insiderTransaction.NumberOrValueAcquiredDisposed).ToString(); + insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"]))); + String acquiredOrDisposed=GetAcquiredOrDisposed(dictionary); + if(null==acquiredOrDisposed) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Cannot determine acquied or disposed for accession#{0}",secAccessionNumber)); + continue; + } + if(acquiredOrDisposed.Equals("D")) + { + insiderTransaction.NumberOrValueAcquiredDisposed*=-1; + } + insiderTransactions.Add(insiderTransaction); + } + return insiderTransactions; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse - Got exception {0}",exception.ToString())); + return null; + } + } + + private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith) + { + if(null==tables)return null; + for(int index=0;index sections) { String item=GetFirstSection(sections); + if(null==item)item=""; + item=item.ToUpper(); if(item.Equals("D"))return "Direct Ownership"; else if(item.Equals("I"))return "Indirect Ownership"; return ""; } + + private String GetAcquiredOrDisposed(Dictionary dictionary) + { + if(!dictionary.ContainsKey("Acquired or Disposed"))return null; + List section=Sections.GetSections(dictionary["Acquired or Disposed"]); + if(null==section||0==section.Count)return null; + String acquiredOrDisposed=section[0]; + if(null==acquiredOrDisposed)return null; + return acquiredOrDisposed.Trim().ToUpper(); + } + private String GetFirstSection(List sections) { if(null==sections || 0==sections.Count)return ""; return sections[0]; } + private String CombineSections(List sections) { StringBuilder sb=new StringBuilder(); @@ -62,135 +212,29 @@ namespace MarketData.Helper } return sb.ToString(); } - public InsiderTransactions Parse(SECFilings secFilings) - { - lock(instance) - { - try - { - InsiderTransactions insiderTransactions=new InsiderTransactions(); - - if(null==secFilings || 0==secFilings.Count)return insiderTransactions; - foreach(SECFiling secFiling in secFilings) - { - List> dictionaryList=new List>(); - String insiderName=null; - String relationshipOfReportingPerson=null; - - byte[] streamBytes = Encoding.ASCII.GetBytes(secFiling.FormText); - MemoryStream memoryStream = new MemoryStream(streamBytes); - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.Load(memoryStream); - HtmlNodeCollection tables=htmlDocument.DocumentNode.SelectNodes("//table"); - - if(null==tables || tables.Count<5)continue; - - HtmlNode nameAndAddressTable=FindTable(tables,"1. Name and Address of Reporting Person"); - if(null==nameAndAddressTable)continue; - HtmlNodeCollection nameAndAddressRows = nameAndAddressTable.SelectNodes(".//tr"); - if(nameAndAddressRows.Count<2)continue; - insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText); - - - //HtmlNodeCollection nameAndAddressRows = tables[4].SelectNodes(".//tr"); - //if(nameAndAddressRows.Count<10)continue; - //insiderName=ApplyNameCase(nameAndAddressRows[1].InnerText); - //relationshipOfReportingPerson=Utility.RemoveHtml(nameAndAddressRows[9].InnerText); - - for(int index=0;index dictionary=new Dictionary(); - dictionary.Add("Title of Security",dataCollection[0].InnerHtml); - dictionary.Add("Transaction Date",dataCollection[1].InnerHtml); - dictionary.Add("Transaction Code",dataCollection[3].InnerHtml); - dictionary.Add("Securities Acquired or Disposed",dataCollection[5].InnerHtml); - dictionary.Add("Acquired or Disposed",dataCollection[6].InnerHtml); - dictionary.Add("Price",dataCollection[7].InnerHtml); - dictionary.Add("Ownership Form",dataCollection[9].InnerHtml); - dictionaryList.Add(dictionary); - } - } - } - - foreach(Dictionary dictionary in dictionaryList) - { - String strItem=null; - InsiderTransaction insiderTransaction = new InsiderTransaction(); - insiderTransaction.Symbol=secFiling.Symbol; - insiderTransaction.SECAccessionNumber=secFiling.SECAccessionNumber; - insiderTransaction.FormRowNumber=(secFiling.Sequence+1).ToString(); - insiderTransaction.Form=secFiling.Form; - insiderTransaction.FilingDate=secFiling.FilingDate; - insiderTransaction.InsiderName=insiderName; - insiderTransaction.Securities=GetFirstSection(Sections.GetSections(dictionary["Title of Security"])); - insiderTransaction.OwnershipType=GetOwnershipForm(Sections.GetSections(dictionary["Ownership Form"])); - if(String.IsNullOrEmpty(insiderTransaction.OwnershipType))continue; - strItem=GetFirstSection(Sections.GetSections(dictionary["Transaction Date"])); - if(String.IsNullOrEmpty(strItem))continue; - insiderTransaction.TransactionDate=Utility.ParseDate(strItem); - insiderTransaction.NatureOfTransaction=transactionCodes.ContainsKey(GetFirstSection(Sections.GetSections(dictionary["Transaction Code"])))?transactionCodes[GetFirstSection(Sections.GetSections(dictionary["Transaction Code"]))]:Constants.CONST_QUESTION; - insiderTransaction.NumberOrValueAcquiredDisposed=FeedParser.ParseValue(GetFirstSection(Sections.GetSections(dictionary["Securities Acquired or Disposed"]))); - insiderTransaction.Price=FeedParser.ParseValue(CombineSections(Sections.GetSections(dictionary["Price"]))); - insiderTransactions.Add(insiderTransaction); - } - } - return insiderTransactions; - } - catch(Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Parse SECFiling got exception {0}",exception.ToString())); - return new InsiderTransactions(); - } - } - } - - private HtmlNode FindTable(HtmlNodeCollection tables,String startsWith) - { - if(null==tables)return null; - for(int index=0;index 0) continue; HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); @@ -1320,13 +1318,114 @@ namespace MarketData.Helper if(null!=httpNetResponse)httpNetResponse.Dispose(); } } + /// + /// Get InsiderTransactions for years greater than or equal to specified year. + /// For example. If 2023 is specified then get all for years>=2023 + /// + /// The symbol. + /// the new y-coordinate. + public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo) + { + MemoryStream memoryStream = null; + HttpNetResponse httpNetResponse=null; + String[] descriptionStartsWith=new String[]{"Form 4","Form 5"}; + int maxFilings=120; + bool continuationFlag=true; + int TIMEOUT_MS_BETWEEN_REQUESTS=1000; + + try + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo)); + DateGenerator dateGenerator=new DateGenerator(); + String cik=PricingDA.GetCIKForSymbol(symbol); + if(null==cik) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol)); + return null; + } + SECFilings secFilings = new SECFilings(); + StringBuilder sb = new StringBuilder(); + String strRequest; + sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); + strRequest = sb.ToString(); + WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions"); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return null; + } + byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + memoryStream = new MemoryStream(streamBytes); + HtmlDocument htmlDocument = new HtmlDocument(); + htmlDocument.Load(memoryStream); + HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]"); + if (null == tables || tables.Count < 1) return null; + HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count)); + for (int row = 0; row < rows.Count && continuationFlag; row ++) + { + try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;} + HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th"); + if (null != headerColumns && headerColumns.Count > 0) continue; + HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td"); + if (null == dataColumns || 5 != dataColumns.Count) continue; + HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a"); + if (null == nodes || 0 == nodes.Count) continue; + String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); + SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl); + for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++) + { + SECFiling innerFiling = innerCollection[index]; + if(innerFiling.FilingDate.YearinnerFiling.Description.StartsWith(x)))continue; + secFilings.Add(innerFiling); + } + } + return InsiderTransactionsParser.GetInstance().Parse(secFilings); + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception); + return null; + } + finally + { + if (null != memoryStream) memoryStream.Close(); + if(null!=httpNetResponse)httpNetResponse.Dispose(); + } + } // *************************************************************************************************************************************************************************************** // ********************************************************************** S E C F I L I N G S - S E C . G O V ******************************************************************** // *************************************************************************************************************************************************************************************** + public static SECFilings GetSECFilings(String symbol,int maxFilings=80) + { + try + { + String cik=PricingDA.GetCIKForSymbol(symbol); + if(null==cik) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol)); + return null; + } + return GetSECFilings(symbol,cik,maxFilings); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception); + return null; } + } + public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80) { MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; + int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000; + try { SECFilings secFilings = new SECFilings(); @@ -1357,6 +1456,7 @@ namespace MarketData.Helper HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); + try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;} SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl); for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++) { @@ -1384,11 +1484,12 @@ namespace MarketData.Helper SECFilings secFilings = new SECFilings(); String secFilingDocument = null; HttpNetResponse httpNetResponse=null; + int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000; + try { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl)); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments"); - try { Thread.Sleep(1000); }catch(Exception) { ;} httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { @@ -1409,11 +1510,11 @@ namespace MarketData.Helper String formName = secNodes[0].InnerText; formName = formName.Replace("\n", ""); formName = formName.Trim(); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName)); HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]"); if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); httpNetResponse.Dispose(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Examining form {0} with {1} entries",formName,rows.Count)); // go through the rows in the table file, searching for the different submission form content int sequence = 0; for (int row = 0; row < rows.Count; row++) @@ -1425,21 +1526,22 @@ namespace MarketData.Helper String document = dataColumns[2].InnerText.Trim(); String type = dataColumns[3].InnerText.Trim(); if(null!=type && type.Equals("GRAPHIC"))continue; -// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type)); +// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type)); if (document.Contains(".htm")) { + HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); -// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl)); - try{Thread.Sleep(1000);}catch(Exception){;} + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Requesting {0}",secFilingDocumentXmlUrl)); + try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); continue; } - secFilingDocument =httpNetResponse.ResponseString; + secFilingDocument = httpNetResponse.ResponseString; if (null == secFilingDocument) continue; SECFiling secFiling = new SECFiling(); secFiling.SECAccessionNumber = secAccessionNumber; diff --git a/MarketDataLib/Helper/ThreadHelper.cs b/MarketDataLib/Helper/ThreadHelper.cs index 03b4bac..78d9ff2 100644 --- a/MarketDataLib/Helper/ThreadHelper.cs +++ b/MarketDataLib/Helper/ThreadHelper.cs @@ -1,8 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Threading; using MarketData.Utils; diff --git a/MarketDataLib/Utility/UpdateManager.cs b/MarketDataLib/Utility/UpdateManager.cs index c95bd26..ec50ac8 100644 --- a/MarketDataLib/Utility/UpdateManager.cs +++ b/MarketDataLib/Utility/UpdateManager.cs @@ -16,28 +16,28 @@ namespace MarketData.Utils public UpdateManager() { } - public bool Prepare(String strPathFileName) + public bool Prepare(String strPathFileName, int maxAgeDays=5) { try { String currentWorkingDirectory=Directory.GetCurrentDirectory(); - if(!File.Exists(strPathFileName)||IsExpired(strPathFileName)) + if(!File.Exists(strPathFileName)||IsExpired(strPathFileName, maxAgeDays)) { if(File.Exists(strPathFileName))File.Delete(strPathFileName); fileStream=new FileStream(strPathFileName,FileMode.Create); streamWriter=new StreamWriter(fileStream); - Console.WriteLine(String.Format("Creating session file:{0}",strPathFileName)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Creating session file:{0}",strPathFileName)); } else { - FileStream fileStream=new FileStream(strPathFileName,FileMode.Open); + FileStream fileStream=new FileStream(strPathFileName,FileMode.Open,FileAccess.ReadWrite,FileShare.Read); StreamReader streamReader=new StreamReader(fileStream); String strLine=null; while(null!=(strLine=streamReader.ReadLine())) { if(!entries.ContainsKey(strLine))entries.Add(strLine,strLine); } - Console.WriteLine(String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Loaded {0} entries from session file:{1}",entries.Count,strPathFileName)); streamReader.Close(); streamReader.Dispose(); fileStream.Close(); @@ -49,27 +49,28 @@ namespace MarketData.Utils } catch(Exception exception) { - Console.WriteLine(String.Format("Exception:{0}",exception)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception)); return false; } } - private bool IsExpired(String strPathFileName) + private bool IsExpired(String strPathFileName, int maxAgeDays) { try { DateGenerator dateGenerator=new DateGenerator(); DateTime creationTime=File.GetCreationTime(strPathFileName); int daysElapsed=Math.Abs(dateGenerator.DaysBetweenActual(creationTime,DateTime.Now)); - if(daysElapsed>5) + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is {1} days(s) old.",strPathFileName,daysElapsed)); + if(daysElapsed>maxAgeDays) { - Console.WriteLine(String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} is expired. {1} days old.",strPathFileName,daysElapsed)); return true; } return false; } catch(Exception exception) { - Console.WriteLine(String.Format("Exception:{0}",exception)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception)); return true; } } @@ -79,9 +80,12 @@ namespace MarketData.Utils } public void Add(String entry) { - if(null==streamWriter)return; - streamWriter.WriteLine(entry); - streamWriter.Flush(); + lock(this) + { + if(null==streamWriter)return; + streamWriter.WriteLine(entry); + streamWriter.Flush(); + } } public void Dispose() {