Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

InsiderTransactions refactorings.

Browse Source
This commit is contained in:
Sean Kessler
2024-02-26 19:48:35 -05:00
parent dc73a51156
commit 661bd3d7ec
7 changed files with 422 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -3,19 +3,15 @@ using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Text.RegularExpressions;
using System.Data;
using System.Threading;
using System.Net;
using System.Collections;
using System.Reflection;
using System.Web;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
using HtmlAgilityPack;
using MarketData.MarketDataModel;
using MarketData.Utils;
using MarketData.Numerical;
using MarketData.CSVHelper;
using MarketData.DataAccess;
using MarketData.Integration;
@@ -1245,7 +1241,6 @@ namespace MarketData.Helper
// ***************************************************************************************************************************************************************************************
// ************************************ S E C F I L I N G S - F O R M 4 & F O R M 5 - F O R I N S I D E R T R A N S A C T I O N S S E C . G O V ***************************S
// ***************************************************************************************************************************************************************************************
// public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=7)
public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=30)
{
MemoryStream memoryStream = null;
@@ -1254,6 +1249,7 @@ namespace MarketData.Helper
DateTime minFilingDate=DateTime.Now;
int maxFilings=80;
bool continuationFlag=true;
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
try
{
@@ -1285,8 +1281,10 @@ namespace MarketData.Helper
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count));
for (int row = 0; row < rows.Count && continuationFlag; row ++)
{
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
if (null != headerColumns && headerColumns.Count > 0) continue;
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
@@ -1320,13 +1318,114 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
/// <summary>
/// Get InsiderTransactions for years greater than or equal to specified year.
/// For example. If 2023 is specified then get all for years>=2023
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="yearGreaterThanEqualTo">the new y-coordinate.</param>
public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
{
MemoryStream memoryStream = null;
HttpNetResponse httpNetResponse=null;
String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
int maxFilings=120;
bool continuationFlag=true;
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
DateGenerator dateGenerator=new DateGenerator();
String cik=PricingDA.GetCIKForSymbol(symbol);
if(null==cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
return null;
}
SECFilings secFilings = new SECFilings();
StringBuilder sb = new StringBuilder();
String strRequest;
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
for (int row = 0; row < rows.Count && continuationFlag; row ++)
{
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
if (null != headerColumns && headerColumns.Count > 0) continue;
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
if (null == dataColumns || 5 != dataColumns.Count) continue;
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
{
SECFiling innerFiling = innerCollection[index];
if(innerFiling.FilingDate.Year<yearGreaterThanEqualTo)
{
continuationFlag=false;
break;
}
if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
secFilings.Add(innerFiling);
}
}
return InsiderTransactionsParser.GetInstance().Parse(secFilings);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
return null;
}
finally
{
if (null != memoryStream) memoryStream.Close();
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// ***************************************************************************************************************************************************************************************
// ********************************************************************** S E C F I L I N G S - S E C . G O V ********************************************************************
// ***************************************************************************************************************************************************************************************
public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
{
try
{
String cik=PricingDA.GetCIKForSymbol(symbol);
if(null==cik)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
return null;
}
return GetSECFilings(symbol,cik,maxFilings);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
return null; }
}
public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
{
MemoryStream memoryStream = null;
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
try
{
SECFilings secFilings = new SECFilings();
@@ -1357,6 +1456,7 @@ namespace MarketData.Helper
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;}
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
{
@@ -1384,11 +1484,12 @@ namespace MarketData.Helper
SECFilings secFilings = new SECFilings();
String secFilingDocument = null;
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
try
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl));
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl));
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
try { Thread.Sleep(1000); }catch(Exception) { ;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
@@ -1409,11 +1510,11 @@ namespace MarketData.Helper
String formName = secNodes[0].InnerText;
formName = formName.Replace("\n", "");
formName = formName.Trim();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName));
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
if (null == tables || tables.Count < 1) return null;
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
httpNetResponse.Dispose();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Examining form {0} with {1} entries",formName,rows.Count));
// go through the rows in the table file, searching for the different submission form content
int sequence = 0;
for (int row = 0; row < rows.Count; row++)
@@ -1425,21 +1526,22 @@ namespace MarketData.Helper
String document = dataColumns[2].InnerText.Trim();
String type = dataColumns[3].InnerText.Trim();
if(null!=type && type.Equals("GRAPHIC"))continue;
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
if (document.Contains(".htm"))
{
HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
if (null == nodes || 0 == nodes.Count) continue;
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl));
try{Thread.Sleep(1000);}catch(Exception){;}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Requesting {0}",secFilingDocumentXmlUrl));
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
continue;
}
secFilingDocument =httpNetResponse.ResponseString;
secFilingDocument = httpNetResponse.ResponseString;
if (null == secFilingDocument) continue;
SECFiling secFiling = new SECFiling();
secFiling.SECAccessionNumber = secAccessionNumber;