diff --git a/App.config b/App.config
index deee84b..6149667 100644
--- a/App.config
+++ b/App.config
@@ -16,7 +16,7 @@
-
+
diff --git a/MarketData.v12.suo b/MarketData.v12.suo
index 8ac9d02..e799a75 100644
Binary files a/MarketData.v12.suo and b/MarketData.v12.suo differ
diff --git a/MarketDataLib/DataAccess/InsiderTransactionDA.cs b/MarketDataLib/DataAccess/InsiderTransactionDA.cs
index 13ed0fb..04e8fef 100644
--- a/MarketDataLib/DataAccess/InsiderTransactionDA.cs
+++ b/MarketDataLib/DataAccess/InsiderTransactionDA.cs
@@ -54,6 +54,59 @@ namespace MarketData.DataAccess
if (null != sqlConnection) sqlConnection.Close();
}
}
+
+ public static InsiderTransactions GetLatestInsiderTransactions()
+ {
+ MySqlConnection sqlConnection = null;
+ MySqlDataReader sqlDataReader = null;
+ MySqlCommand sqlCommand =null;
+ InsiderTransactions insiderTransactions=new InsiderTransactions();
+ String strQuery = null;
+
+ try
+ {
+ StringBuilder sb = new StringBuilder();
+ sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
+ sb.Append("SELECT symbol,filing_date,transaction_date,insider_name,ownership_type,securities,nature_of_transaction,number_or_value_acquired_disposed,price,form,sec_accession_number,form_row_number,modified FROM insidertransaction WHERE TRANSACTION_DATE=(SELECT MAX(TRANSACTION_DATE) FROM insidertransaction WHERE TRANSACTION_DATE<=NOW() LIMIT 1)");
+ strQuery = sb.ToString(); ;
+ sqlCommand = new MySqlCommand(strQuery, sqlConnection);
+ sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
+ sqlDataReader = sqlCommand.ExecuteReader();
+ while (sqlDataReader.Read())
+ {
+ InsiderTransaction insiderTransaction=new InsiderTransaction();
+ insiderTransaction.Symbol=sqlDataReader.GetString(0);
+ insiderTransaction.FilingDate=sqlDataReader.GetDateTime(1);
+ insiderTransaction.TransactionDate=sqlDataReader.GetDateTime(2);
+ insiderTransaction.InsiderName=sqlDataReader.GetString(3);
+ insiderTransaction.OwnershipType=sqlDataReader.GetString(4);
+ insiderTransaction.Securities=sqlDataReader.GetString(5);
+ insiderTransaction.NatureOfTransaction=sqlDataReader.GetString(6);
+ if(!sqlDataReader.IsDBNull(7))insiderTransaction.NumberOrValueAcquiredDisposed=sqlDataReader.GetDouble(7);
+ else insiderTransaction.NumberOrValueAcquiredDisposed=Double.NaN;
+ if(!sqlDataReader.IsDBNull(8))insiderTransaction.Price=sqlDataReader.GetDouble(8);
+ else insiderTransaction.Price=Double.NaN;
+ insiderTransaction.Form=sqlDataReader.GetString(9);
+ insiderTransaction.SECAccessionNumber=sqlDataReader.GetString(10);
+ insiderTransaction.FormRowNumber=sqlDataReader.GetString(11);
+ insiderTransaction.Modified=sqlDataReader.GetDateTime(12);
+ insiderTransactions.Add(insiderTransaction);
+ }
+ return insiderTransactions;
+ }
+ catch (Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestInsiderTransactions Exception: {0}",exception.ToString()));
+ return null;
+ }
+ finally
+ {
+ if(null!=sqlCommand)sqlCommand.Dispose();
+ if (null != sqlDataReader) sqlDataReader.Close();
+ if (null != sqlConnection) sqlConnection.Close();
+ }
+ }
+
public static InsiderTransactions GetInsiderTransactions(String symbol)
{
MySqlConnection sqlConnection = null;
@@ -107,6 +160,7 @@ namespace MarketData.DataAccess
}
}
+
public static bool InsertInsiderTransactions(InsiderTransactions insiderTransactions)
{
MySqlConnection sqlConnection = null;
diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs
index fba6310..6029b63 100644
--- a/MarketDataLib/Helper/MarketDataHelper.cs
+++ b/MarketDataLib/Helper/MarketDataHelper.cs
@@ -298,124 +298,6 @@ namespace MarketData.Helper
if(null!=httpNetResponse) httpNetResponse.Dispose();
}
}
- //public static PremarketElements GetPremarketData()
- //{
- // HttpNetResponse httpNetResponse=null;
- // MemoryStream memoryStream=null;
- // PremarketElements premarketElements=new PremarketElements();
- // DateTime timestamp=DateTime.Now;
- // try
- // {
- // StringBuilder sb=new StringBuilder();
- // String strRequest=null;
-
- // sb.Append("https://money.cnn.com/data/premarket/");
- // strRequest=sb.ToString();
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
- // httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
- // if(!httpNetResponse.Success)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}' failed with {1}",strRequest,httpNetResponse.ErrorMessage));
- // return null;
- // }
- // byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
- // memoryStream=new MemoryStream(streamBytes);
- // HtmlDocument htmlDocument=new HtmlDocument();
- // htmlDocument.Load(memoryStream);
- // HtmlNodeCollection divSection=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"wsod_dataTable\"]");
- // if(null==divSection||0==divSection.Count)
- // {
- // if(!ParsePremarketElementsBySection(httpNetResponse.ResponseString,premarketElements))
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, cannot locate data in result for request '{0}'.",strRequest));
- // return null;
- // }
- // return premarketElements;
- // }
- // HtmlNodeCollection rows=divSection[0].SelectNodes(".//tr");
- // for(int index=0;index0) continue;
- // HtmlNodeCollection dataColumns=rows[index].SelectNodes(".//td");
- // if(dataColumns.Count<2) continue;
- // String market=dataColumns[0].InnerText.Trim().ToUpper();
- // market=market.Replace("&","&");
- // String quoteHtml=dataColumns[1].InnerHtml;
- // List sections=Sections.GetSections(quoteHtml);
- // if(3!=sections.Count)continue;
- // String changeValue=sections[0];
- // String changePercent=sections[2];
- // premarketElement.Market=market;
- // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
- // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
- // premarketElement.Timestamp=timestamp;
- // premarketElements.Add(premarketElement);
- // }
- // if(0==premarketElements.Count)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, Did not find any results for request '{0}'.",strRequest));
- // }
- // return premarketElements;
- // }
- // catch(Exception exception)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,exception);
- // return null;
- // }
- // finally
- // {
- // if(null!=memoryStream) memoryStream.Close();
- // if(null!=httpNetResponse) httpNetResponse.Dispose();
- // }
- //}
- //private static bool ParsePremarketElementsBySection(String strResponseString,PremarketElements premarketElements)
- //{
- // if(null==strResponseString)return false;
- // List sections=Sections.GetSections(strResponseString);
- // if(null==sections||0==sections.Count)return false;
- // DateTime timestamp=DateTime.Now;
- // PremarketElement premarketElement=null;
-
- // String changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",8);
- // String changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",4);
- // if(!(null==changePercent||null==changeValue))
- // {
- // premarketElement=new PremarketElement();
- // premarketElement.Market="S&P";
- // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
- // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
- // premarketElement.Timestamp=timestamp;
- // premarketElements.Add(premarketElement);
- // }
-
- // changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",8);
- // changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",4);
- // if(!(null==changePercent||null==changeValue))
- // {
- // premarketElement=new PremarketElement();
- // premarketElement.Market="DOW";
- // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
- // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
- // premarketElement.Timestamp=timestamp;
- // premarketElements.Add(premarketElement);
- // }
-
- // changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",8);
- // changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",4);
- // if(!(null==changePercent||null==changeValue))
- // {
- // premarketElement=new PremarketElement();
- // premarketElement.Market="NASDAQ";
- // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
- // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
- // premarketElement.Timestamp=timestamp;
- // premarketElements.Add(premarketElement);
- // }
-
- // return true;
- //}
// ******************************************************************************************************************************************************************************
// ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S *******************************************
// ******************************************************************************************************************************************************************************
@@ -691,11 +573,11 @@ namespace MarketData.Helper
if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf");
else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks");
strRequest = sb.ToString();
- httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest));
+ httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy);
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy);
- //httpNetResponse = HttpNetRequest.GetRequestNoEncodingV7(strRequest);
+// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1646,6 +1528,68 @@ namespace MarketData.Helper
// ***************************************************************************************************************************************************************************************
// ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E *********************************************************
// ***************************************************************************************************************************************************************************************
+ //public static ETFHoldings GetETFHoldings(String etfSymbol)
+ //{
+ // MemoryStream memoryStream = null;
+ // ETFHoldings etfHoldings = new ETFHoldings();
+ // HttpNetResponse httpNetResponse=null;
+ // DateTime modified=DateTime.Now;
+
+ // try
+ // {
+ // StringBuilder sb = new StringBuilder();
+ // String strRequest;
+ // etfSymbol = etfSymbol.ToUpper();
+ // sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings");
+ // strRequest = sb.ToString();
+
+ // WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
+ // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
+ // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ // if(!httpNetResponse.Success)
+ // {
+ // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+ // return null;
+ // }
+ // String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
+ // strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\"");
+ // List> items=LocateJSONItems(strResponse);
+ // if(null==items||0==items.Count)return null;
+ // for(int index=0;indexitems.Count)break;
+ // List symbolPair=items[index];
+ // List holdingNamePair=items[index+1];
+ // List percentOfAssetsPair=items[index+2];
+ // if(!symbolPair[0].Equals("symbol"))continue;
+ // if(!holdingNamePair[0].Equals("holdingName"))continue;
+ // if(!percentOfAssetsPair[0].Equals("fmt"))continue;
+ // if(String.IsNullOrEmpty(symbolPair[1]))continue;
+ // ETFHolding etfHolding = new ETFHolding();
+ // etfHolding.ETFSymbol = etfSymbol;
+ // etfHolding.HoldingSymbolShareClass=null;
+ // etfHolding.HoldingCompanyName=holdingNamePair[1];
+ // etfHolding.HoldingSymbolShareClass=symbolPair[1];
+ // etfHolding.HoldingSymbol=symbolPair[1];
+ // etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]);
+ // etfHolding.Modified=modified;
+ // etfHoldings.Add(etfHolding);
+ // }
+ // return etfHoldings;
+ // }
+ // catch (Exception exception)
+ // {
+ // MDTrace.WriteLine(LogLevel.DEBUG,exception);
+ // return null;
+ // }
+ // finally
+ // {
+ // if (null != memoryStream) memoryStream.Close();
+ // if(null!=httpNetResponse)httpNetResponse.Dispose();
+ // }
+ //}
+
+
public static ETFHoldings GetETFHoldings(String etfSymbol)
{
MemoryStream memoryStream = null;
@@ -1658,40 +1602,39 @@ namespace MarketData.Helper
StringBuilder sb = new StringBuilder();
String strRequest;
etfSymbol = etfSymbol.ToUpper();
- sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings");
+ sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings");
strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
-// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
- httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy);
+// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
- String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
- strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\"");
- List> items=LocateJSONItems(strResponse);
- if(null==items||0==items.Count)return null;
- for(int index=0;index sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
+ if(null == sections || 0==sections.Count)return null;
+
+ byte[] streamBytes=Encoding.ASCII.GetBytes(sections[0]);
+ memoryStream=new MemoryStream(streamBytes);
+ HtmlDocument htmlDocument=new HtmlDocument();
+ htmlDocument.Load(memoryStream);
+ HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes(".//td");
+ if(null==rows || 0==rows.Count)return null;
+ for(int rowIndex=0;rowIndexitems.Count)break;
- List symbolPair=items[index];
- List holdingNamePair=items[index+1];
- List percentOfAssetsPair=items[index+2];
- if(!symbolPair[0].Equals("symbol"))continue;
- if(!holdingNamePair[0].Equals("holdingName"))continue;
- if(!percentOfAssetsPair[0].Equals("fmt"))continue;
- if(String.IsNullOrEmpty(symbolPair[1]))continue;
+ HtmlNode node = rows[rowIndex];
ETFHolding etfHolding = new ETFHolding();
etfHolding.ETFSymbol = etfSymbol;
- etfHolding.HoldingSymbolShareClass=null;
- etfHolding.HoldingCompanyName=holdingNamePair[1];
- etfHolding.HoldingSymbolShareClass=symbolPair[1];
- etfHolding.HoldingSymbol=symbolPair[1];
- etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]);
- etfHolding.Modified=modified;
+ etfHolding.HoldingSymbolShareClass = null;
+ etfHolding.HoldingCompanyName = rows[rowIndex].InnerText;
+ etfHolding.HoldingSymbolShareClass = rows[rowIndex+1].InnerText;
+ etfHolding.HoldingSymbol = rows[rowIndex+1].InnerText;
+ etfHolding.PercentOfAssets = FeedParser.ParseValue(rows[rowIndex+2].InnerText);
+ etfHolding.Modified = modified;
etfHoldings.Add(etfHolding);
}
return etfHoldings;
@@ -2265,50 +2208,6 @@ namespace MarketData.Helper
}
}
-
- //private static AnalystPriceTarget GetAnalystPriceTargetYahoo(String symbol)
- //{
- // AnalystPriceTarget analystPriceTarget = null;
- // HttpNetResponse httpNetResponse=null;
- // try
- // {
- // StringBuilder sb = new StringBuilder();
- // String strRequest;
- // symbol = symbol.ToUpper();
- // sb.Append("http://finance.yahoo.com/q/ao?s=").Append(symbol).Append("+Analyst+Opinion");
- // WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetYahoo");
- // strRequest=sb.ToString();
- // MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetYahoo: {0}", strRequest));
- // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
- // if(!httpNetResponse.Success)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
- // return null;
- // }
- // analystPriceTarget=new AnalystPriceTarget();
- // analystPriceTarget.Symbol=symbol;
- // analystPriceTarget.Date=DateTime.Now.Date;
- // String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
- // analystPriceTarget.HighTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetHighPrice\":{\"raw\":",','));
- // analystPriceTarget.MeanTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMeanPrice\":{\"raw\":",','));
- // analystPriceTarget.LowTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetLowPrice\":{\"raw\":",','));
- // analystPriceTarget.MedianTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMedianPrice\":{\"raw\":",','));
- // if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
- // double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
- // double.IsNaN(analystPriceTarget.LowTargetPrice)&&
- // double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
- // return analystPriceTarget;
- // }
- // catch (Exception exception)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,exception);
- // return null;
- // }
- // finally
- // {
- // if(null!=httpNetResponse)httpNetResponse.Dispose();
- // }
- //}
// ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
// ****************************************************************************************************************************************************************************
@@ -2804,15 +2703,6 @@ namespace MarketData.Helper
return currency;
}
}
- //public static String GetMStarSecurityId(String symbol,String responseString)
- //{
- // String securityIdentifier=GetMStarSecurityId_1(responseString);
- // if(null!=securityIdentifier)return securityIdentifier;
- // securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
- // if(null!=securityIdentifier)return securityIdentifier;
- // return null;
- //}
-
public static String GetMStarSecurityId(String symbol,String responseString)
{
@@ -2838,23 +2728,6 @@ namespace MarketData.Helper
//byId:{\"0P000003MU\":$}
// AAPL:0P000000GY
- //private static String GetMStarSecurityId_2(String responseString)
- //{
- // try
- // {
- // string token="byId";
- // int index=responseString.IndexOf(token);
- // if(-1==index)return null;
- // string securityIdentifier=responseString.Substring(index+token.Length);
- // securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
- // return securityIdentifier;
- // }
- // catch(Exception exception)
- // {
- // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId encountered an exception:{0}",exception.ToString()));
- // return null;
- // }
- //}
// 5","0P000003MU","MIDD",60
private static String GetMStarSecurityId_2(String symbol,String responseString)
@@ -4424,7 +4297,7 @@ namespace MarketData.Helper
}
// *****************************************************************************************************************************************************************************
-// **************************************************************************** B A L A N C E S H E E T - N A S D A Q ****************************************************
+// **************************************************************************** C U R R E N C Y C O N V E R S I O N ****************************************************
// *****************************************************************************************************************************************************************************
public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate)
{
@@ -4711,7 +4584,8 @@ namespace MarketData.Helper
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx");
- httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
+// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy); // user agents refreshed in this version
//MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
if(!httpNetResponse.Success)
{
@@ -4890,10 +4764,12 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol));
return null;
}
+// ***************************************************************************************************************
+// ******************** L A T E S T P R I C E R E T R I E V A L F O R I N T R A - D A Y F E E D ********
+// ***************************************************************************************************************
-// This should be proxied through TOR because Yahoo tracks IP address.
-// This is used in the intra-day price feed
- private static Price GetLatestPriceYahoo(String symbol)
+
+ public static Price GetLatestPriceYahoo(String symbol)
{
HttpNetResponse httpNetResponse=null;
try
@@ -4904,15 +4780,16 @@ namespace MarketData.Helper
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol);
strRequest=sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo");
- httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
+ httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy);
if(!httpNetResponse.Success)
{
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
Price price=new Price();
price.Symbol=symbol.ToUpper();
price.Date=DateTime.Now;
+ price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN;
price.Source=Price.PriceSource.Yahoo;
return BuildPriceFromResponse(httpNetResponse.ResponseString,price);
}
@@ -4926,8 +4803,250 @@ namespace MarketData.Helper
if(null!=httpNetResponse) httpNetResponse.Dispose();
}
}
+
+ public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
+ {
+ if(httpNetResponse.Contains("HTML_TAG_END"))
+ {
+ price = BuildPriceFromSections(httpNetResponse, price);
+ }
+ else
+ {
+ price = BuildPriceFromTable(httpNetResponse, price);
+ }
+ return price;
+ }
+
+ private static Price BuildPriceFromSections(String httpNetResponse,Price price)
+ {
+ try
+ {
+ String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END");
+ List sections = null;
+
+ sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
+
+ String marker ="data-symbol=\""+price.Symbol+"\"";
+ List dataSections = sections.Where(x => x.Contains(marker)).ToList();
+ dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
+ if(dataSections.Count>0)
+ {
+ dataSections = Sections.GetSections(dataSections[0]);
+ if(null!=dataSections && dataSections.Count>0)
+ {
+ price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]);
+ }
+ }
+
+ marker = " x.StartsWith(marker)).ToList();
+ for(int index=0;index sections = null;
+ double bid = double.NaN;
+ double bidContracts = double.NaN;
+ double ask = double.NaN;
+ double askContracts = double.NaN;
+
+ String symbol=price.Symbol.ToUpper();
+ byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse);
+ memoryStream = new MemoryStream(streamBytes);
+ HtmlDocument htmlDocument = new HtmlDocument();
+ htmlDocument.Load(memoryStream);
+
+ HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]");
+ if(null == table || 0 == table.Count)return null;
+ HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
+ if (null == rows || 0 == rows.Count) return null;
+ for(int index=0;index< rows.Count; index++)
+ {
+ int itemIndex=0;
+ String itemValue=null;
+ String innerHtml = rows[index].InnerHtml;
+ sections=Sections.GetSections(innerHtml);
+
+ if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ }
+ else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ price.Open=FeedParser.ParseValue(itemValue);
+ }
+ else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ String[] bidContractSplit = itemValue.Split('x');
+ bid=FeedParser.ParseValue(bidContractSplit[0]);
+ bidContracts=FeedParser.ParseValue(bidContractSplit[1]);
+ }
+ else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ String[] askContractSplit = itemValue.Split('x');
+ ask = FeedParser.ParseValue(askContractSplit[0]);
+ askContracts = FeedParser.ParseValue(askContractSplit[1]);
+ }
+ else if(Sections.FindInSections(sections, "Day's Range",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ String[] subItems=itemValue.Split('-');
+ price.Low=FeedParser.ParseValue(subItems[0]);
+ price.High=FeedParser.ParseValue(subItems[1]);
+ }
+ else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ }
+ else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+4];
+ price.Volume=FeedParser.ParseValueLong(itemValue);
+ }
+ else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true))
+ {
+ itemValue = sections[itemIndex+3];
+ }
+ }
+
+ if (double.IsNaN(price.Close))
+ {
+ sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
+ String marker = "data-symbol=\"" + price.Symbol + "\"";
+ sections = sections.Where(x => x.Contains(marker)).ToList();
+ sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
+ if (sections.Count > 0)
+ {
+ sections = Sections.GetSections(sections[0]);
+ if (null != sections && sections.Count > 0)
+ {
+ price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]);
+ }
+ }
+ }
+
+ if(double.IsNaN(price.Close))
+ {
+ double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts);
+ if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice;
+ if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00;
+ }
+
+ CheckPrice(price);
+ return price;
+ }
+ catch(Exception exception)
+ {
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: {0}",exception.ToString()));
+ return null;
+ }
+ finally
+ {
+ if(null!=memoryStream)
+ {
+ memoryStream.Close();
+ memoryStream.Dispose();
+ memoryStream=null;
+ }
+ }
+ }
+
+ private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts)
+ {
+ if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN;
+ double totalContracts = bidContracts + askContracts;
+ double bidWeight = bidContracts/totalContracts;
+ double askWeight = askContracts/totalContracts;
+ double weightedBid = bid*bidWeight;
+ double weightedAsk = ask*askWeight;
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2)));
+ return weightedBid + weightedAsk;
+ }
+
+ public static void CheckPrice(Price price)
+ {
+ if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
+ {
+ price.Close=price.AdjClose=(price.High+price.Low)/2.00;
+ }
+ if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
+ { // if the close price is garbage set it to the midpoint of the high and the low
+ if(price.Closeprice.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
+ }
+ if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close;
+ if(double.IsNaN(price.Open))price.Open=0.00;
+ if(double.IsNaN(price.Low))price.Low=0.00;
+ if(double.IsNaN(price.High))price.High=0.00;
+ if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
+ {
+ price.Open=price.High=price.Low=price.Close;
+ }
+ }
+
// This is used in the intra-day price feed.
- private static Price GetLatestPriceBigCharts(String symbol)
+ public static Price GetLatestPriceBigCharts(String symbol)
{
HttpNetResponse httpNetResponse=null;
MemoryStream memoryStream=null;
@@ -5123,6 +5242,12 @@ namespace MarketData.Helper
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
}
}
+// *******************************************************************************************************************************************************************************
+// *******************************************************************************************************************************************************************************
+// *******************************************************************************************************************************************************************************
+
+
+
// *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
// *******************************************************************************************************************************************************************************
@@ -5349,88 +5474,6 @@ namespace MarketData.Helper
// ******************************************************************************************************************************************************************************************************************
-// This function attempts to locate the quote data within the raw data returned from Yahoo. It should be interpreted as intraday data because
-// the market date cannot be located within the raw data response.
- public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
- {
- String strResponse=Utility.KeepAfter(httpNetResponse,"/* -- Data -- */");
- String symbol=price.Symbol.ToUpper();
- bool isMutualFund=false;
- String findPattern1="\"quoteType\":\"MUTUALFUND\",\"symbol\":"+"\""+symbol+"\"";
- String findPattern2="\"quoteType\":\"EQUITY\",\"symbol\":"+"\""+symbol+"\"";
- String findPattern3="\"quoteType\":\"ETF\",\"symbol\":"+"\""+symbol+"\"";
- String findPattern4="\"quoteType\":\"INDEX\",\"symbol\":"+"\""+symbol+"\"";
- strResponse=Utility.KeepAfter(httpNetResponse,findPattern1);
- if(null!=strResponse)isMutualFund=true;
- if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern2);
- if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern3);
- if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern4);
- if(null==strResponse)return null;
- strResponse=Utility.KeepBefore(strResponse,"MOST_ACTIVE_TITLE");
- price.Open=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketOpen\":{\"raw\":",','));
- price.High=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayHigh\":{\"raw\":",','));
- price.Low=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayLow\":{\"raw\":",','));
- price.Close=FeedParser.ParseValue(Utility.FindFirst(Utility.KeepAfterLast(strResponse,"\"regularMarketDayLow\":{\"raw\":"),"\"regularMarketPrice\":{\"raw\":",','));
- price.Volume=FeedParser.ParseValueLong(Utility.FindFirst(strResponse,"\"regularMarketVolume\":{\"raw\":",','));
- if(double.NaN.Equals(price.Close))price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPrice\":{\"raw\":",','));
- if(double.IsNaN(price.Close)) // try to find raw quote data
- {
- String strQuote=Utility.KeepAfter(httpNetResponse,"",'<'));
- else if(null==strQuote&&httpNetResponse.Contains("data-reactid=\"38\">As of"))
- {
- strQuote=Utility.KeepAfter(httpNetResponse,"class=\"Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)\"");
- if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,">",'<'));
- }
- if(!double.IsNaN(price.Close))MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Found quote data for {0}->{1}",symbol,Utility.FormatCurrency(price.Close)));
- }
- if(double.IsNaN(price.Close))
- {
- double bid=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"bid\":{\"raw\":",','));
- double ask=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"ask\":{\"raw\":",','));
- if(!double.IsNaN(bid)&&!double.IsNaN(ask))price.Close=(bid+ask)/2.00;
- }
- if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
- {
- price.Close=price.AdjClose=(price.High+price.Low)/2.00;
- }
- if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
- {
-// if this is a mutual fund then the regularMarketPreviousClose may actually be the close from two days ago and not last night. Goto BigCharts in this case
- if(isMutualFund)price=GetPriceAsOf(symbol,DateTime.Now);
- else price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPreviousClose\":{\"raw\":",','));
- if(null==price||double.IsNaN(price.Close))
- {
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",symbol));
- return null;
- }
- price.Open=price.High=price.Low=price.AdjClose=price.Close;
- return price;
- }
- if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
- { // if the close price is garbage set it to the midpoint of the high and the low
- if(price.Closeprice.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
- }
- price.AdjClose=price.Close;
- if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
- {
- MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No open,high,low,close price for {0}",price.Symbol));
- return null;
- }
- if(double.IsNaN(price.Open))price.Open=0.00;
- if(double.IsNaN(price.Low))price.Low=0.00;
- if(double.IsNaN(price.High))price.High=0.00;
- if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
- {
- price.Open=price.High=price.Low=price.Close;
- }
- return price;
- }
// ****************************************************************************************************************************************************************************************
// **************************************************************************** E N D D A T A F E E D S *********************************************************************************
// ****************************************************************************************************************************************************************************************
diff --git a/MarketDataLib/Integration/HttpNetRequest.cs b/MarketDataLib/Integration/HttpNetRequest.cs
index 3aa35cc..1aac6e5 100644
--- a/MarketDataLib/Integration/HttpNetRequest.cs
+++ b/MarketDataLib/Integration/HttpNetRequest.cs
@@ -688,6 +688,97 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]");
}
}
+ public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null)
+ {
+ HttpWebResponse webResponse=null;
+ WebException lastWebException=null;
+ ServicePointManager.Expect100Continue=true;
+ ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
+ Random random=new Random();
+
+ try
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
+ String[] userAgents=
+ {
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3"
+ };
+ int MAX_RETRIES=5;
+ int TIMEOUT_BETWEEN_RETRIES=1000;
+ int charCount=0;
+ byte[] buffer=new byte[8192];
+ StringBuilder sb=new StringBuilder();
+
+ for(int count=0;count=userAgents.Length) index=userAgents.Length-1;
+ String userAgent=userAgents[index];
+ webRequest.UserAgent=userAgent;
+
+ try
+ {
+ webResponse=(HttpWebResponse)webRequest.GetResponse();
+ Stream responseStream=webResponse.GetResponseStream();
+ if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
+ {
+ responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
+ StreamReader reader=new StreamReader(responseStream,Encoding.Default);
+ sb.Append(reader.ReadToEnd());
+ reader.Close();
+ }
+ else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
+ {
+ responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
+ StreamReader reader=new StreamReader(responseStream,Encoding.Default);
+ sb.Append(reader.ReadToEnd());
+ reader.Close();
+ }
+ else
+ {
+ while(true)
+ {
+ charCount=responseStream.Read(buffer,0,buffer.Length);
+ if(0==charCount) break;
+ sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
+ }
+ }
+ webResponse.Close();
+ return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
+ }
+ catch(WebException webException)
+ {
+ if(webException.Message.Contains("(404) Not Found")) return new HttpNetResponse(webResponse,strRequest,false,null==lastWebException?"":lastWebException.Message);
+ lastWebException=webException;
+ Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
+ }
+ }
+ MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
+ return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
+ }
+ catch(Exception exception)
+ {
+ return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
+ }
+ finally
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
+ }
+ }
public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
@@ -796,6 +887,7 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
}
}
+
public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
@@ -844,55 +936,73 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
-// public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
-// {
-// HttpWebResponse webResponse = null;
-// try
-// {
-// MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
-// int charCount = 0;
-// byte[] buffer = new byte[8192];
-// StringBuilder sb = new StringBuilder();
-// bool expect100Condition = ServicePointManager.Expect100Continue;
-// SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
-// ServicePointManager.Expect100Continue = true;
-// ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
-// HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
-// if(null!=webProxy)webRequest.Proxy=webProxy;
-// webRequest.Timeout = webRequestTimeoutMS;
-// webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
-// webRequest.Headers.Add("Accept-Encoding: *");
-// webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
-//// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
-// webRequest.UserAgent = GetUserAgent(true);
-// webRequest.KeepAlive = true;
-// webRequest.CookieContainer = new CookieContainer();
-// webResponse = (HttpWebResponse)webRequest.GetResponse();
-// Stream responseStream = webResponse.GetResponseStream();
-// while (true)
-// {
-// charCount = responseStream.Read(buffer, 0, buffer.Length);
-// if (0 == charCount) break;
-// sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
-// }
-// webResponse.Close();
-// ServicePointManager.Expect100Continue = expect100Condition;
-// ServicePointManager.SecurityProtocol = securityProtocolType;
-// return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
-// }
-// catch (WebException webException)
-// {
-// return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
-// }
-// catch (Exception exception)
-// {
-// return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
-// }
-// finally
-// {
-// MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
-// }
-// }
+
+ public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
+ {
+ HttpWebResponse webResponse = null;
+ try
+ {
+ Random random = new Random();
+ String[] userAgents=
+ {
+ "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
+ };
+
+ MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
+ int charCount = 0;
+ byte[] buffer = new byte[8192];
+ StringBuilder sb = new StringBuilder();
+ bool expect100Condition = ServicePointManager.Expect100Continue;
+ SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
+ ServicePointManager.Expect100Continue = true;
+ ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
+ HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
+ if(null!=webProxy)webRequest.Proxy=webProxy;
+ webRequest.Timeout = webRequestTimeoutMS;
+ webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
+ webRequest.Headers.Add("Accept-Encoding: None");
+ webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
+// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
+ webRequest.KeepAlive = true;
+ webRequest.CookieContainer = new CookieContainer();
+
+ double randomNumber=random.NextDouble(); // number between 0 and 1
+ int index=(int)(randomNumber*((double)userAgents.Length));
+ if(index>=userAgents.Length) index=userAgents.Length-1;
+ String userAgent=userAgents[index];
+ webRequest.UserAgent=userAgent;
+
+
+ webResponse = (HttpWebResponse)webRequest.GetResponse();
+ Stream responseStream = webResponse.GetResponseStream();
+ while (true)
+ {
+ charCount = responseStream.Read(buffer, 0, buffer.Length);
+ if (0 == charCount) break;
+ sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
+ }
+ webResponse.Close();
+ ServicePointManager.Expect100Continue = expect100Condition;
+ ServicePointManager.SecurityProtocol = securityProtocolType;
+ return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
+ }
+ catch (WebException webException)
+ {
+ return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
+ }
+ catch (Exception exception)
+ {
+ return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
+ }
+ finally
+ {
+ MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
+ }
+ }
+
+
+
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
{
HttpWebResponse webResponse = null;
diff --git a/MarketDataLib/Utility/FeedParser.cs b/MarketDataLib/Utility/FeedParser.cs
index d70b937..c0bee2f 100644
--- a/MarketDataLib/Utility/FeedParser.cs
+++ b/MarketDataLib/Utility/FeedParser.cs
@@ -231,6 +231,12 @@ namespace MarketData.Utils
multiplier = -1.00;
}
if (strText.Equals("-")) return double.NaN;
+ if (strText[strText.Length - 1].Equals('T'))
+ {
+ strText = strText.Replace("T", "");
+ value = double.Parse(strText);
+ value *= 1000000000000;
+ }
if (strText[strText.Length - 1].Equals('B'))
{
strText = strText.Replace("B", "");
diff --git a/MarketDataLib/Utility/Sections.cs b/MarketDataLib/Utility/Sections.cs
index 2fc1965..3d50c66 100644
--- a/MarketDataLib/Utility/Sections.cs
+++ b/MarketDataLib/Utility/Sections.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
+using MarketData.Utils;
namespace MarketDataLib.Utility
{
@@ -28,6 +29,24 @@ namespace MarketDataLib.Utility
if(String.IsNullOrEmpty(strContainingString))return null;
return strContainingString;
}
+
+ public static List GetAllItemsInSections(String strInput, String sectionName)
+ {
+ int searchIndex = 0;
+ List sectionItems = new List();
+
+ while (true)
+ {
+ String itemsInSection = GetItemsInSection(strInput, sectionName, ref searchIndex);
+ if(null==itemsInSection)break;
+ sectionItems.Add(itemsInSection);
+ searchIndex++;
+ }
+ return sectionItems;
+ }
+
+
+
public static List GetSections(String strInput)
{
try
@@ -36,6 +55,7 @@ namespace MarketDataLib.Utility
int index=0;
String strLiteral=null;
+ if(null==strInput)return null;
while(index", "<");
+ return strItem;
+ }
+
private static String ScanSection(String strInput, ref int index)
{
try