diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 9b7e8dd..2b89288 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -1074,10 +1074,10 @@ namespace MarketData.Helper MemoryStream memoryStream = null; HttpNetResponse httpNetResponse=null; int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000; + SECFilings secFilings = new SECFilings(); try { - SECFilings secFilings = new SECFilings(); StringBuilder sb = new StringBuilder(); String strRequest; sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString()); @@ -1086,7 +1086,7 @@ namespace MarketData.Helper httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); @@ -1124,11 +1124,13 @@ namespace MarketData.Helper { if (null != memoryStream) memoryStream.Close(); if(null!=httpNetResponse)httpNetResponse.Dispose(); + MDTrace.WriteLine(LogLevel.DEBUG,$"[GetSECFilings] Retrieved {secFilings.Count} SECFilings for {symbol}"); } } // Get the filing document in XML format private static SECFilings GetSECFilingDocuments(String symbol,DateTime filingDate,String secFilingDocumentUrl) { + Profiler profiler = new Profiler(); MemoryStream memoryStream = null; SECFilings secFilings = new SECFilings(); String secFilingDocument = null; @@ -1137,12 +1139,13 @@ namespace MarketData.Helper try { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl)); WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments"); + if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy."); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Request:{0} failed with status {1}]",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); @@ -1163,7 +1166,7 @@ namespace MarketData.Helper if (null == tables || tables.Count < 1) return null; HtmlNodeCollection rows = tables[0].SelectNodes(".//tr"); httpNetResponse.Dispose(); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Examining form {0} with {1} entries",formName,rows.Count)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. {0} Examining form {1} with {2} entries",symbol,formName,rows.Count)); // go through the rows in the table file, searching for the different submission form content int sequence = 0; for (int row = 0; row < rows.Count; row++) @@ -1181,12 +1184,12 @@ namespace MarketData.Helper HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a"); if (null == nodes || 0 == nodes.Count) continue; String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown"); - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Requesting {0}",secFilingDocumentXmlUrl)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl)); try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;} httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode)); continue; } secFilingDocument = httpNetResponse.ResponseString; @@ -1203,7 +1206,7 @@ namespace MarketData.Helper secFiling.FormText = secFilingDocument; secFiling.FormText = secFiling.FormText.Replace("\n",""); if (null == secFiling.FormText) continue; - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString())); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString())); secFilings.Add(secFiling); httpNetResponse.Dispose(); } @@ -1222,6 +1225,7 @@ namespace MarketData.Helper memoryStream.Close(); memoryStream = null; } + MDTrace.WriteLine(LogLevel.DEBUG,string.Format("[GetSECFilingDocuments] Done, retrieved {0} documents for {1} in {2}(ms)",secFilings.Count,symbol,profiler.End())); } } // ***************************************************************************************************************************************************************************************