Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Feeds. InsiderTransactions, ETFHoldings, Yahoo Price Feed

Browse Source
This commit is contained in:
Sean Kessler
2024-03-04 19:28:21 -05:00
parent dc893be526
commit c4fa727c54
7 changed files with 596 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
App.config
View File

@@ -16,7 +16,7 @@
<add key="proxy_GetAnalystPriceTargetYahoo" value="true"/> <add key="proxy_GetAnalystPriceTargetYahoo" value="true"/>
<add key="proxy_GetDailyPrices" value="false"/> <add key="proxy_GetDailyPrices" value="false"/>
<add key="proxy_GetCompanyHeadlines" value="true"/> <add key="proxy_GetCompanyHeadlines" value="true"/>
<add key="proxy_GetFundamentalEx" value="true"/> <add key="proxy_GetFundamentalEx" value="false"/>
<add key="proxy_GetDividendHistory" value="false"/> <add key="proxy_GetDividendHistory" value="false"/>
<add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/> <add key="proxy_GetAnalystPriceTargetMarketBeat" value="false"/>
</appSettings> </appSettings>

BIN
MarketData.v12.suo
View File

Binary file not shown.

54
MarketDataLib/DataAccess/InsiderTransactionDA.cs
View File

@@ -54,6 +54,59 @@ namespace MarketData.DataAccess
if (null != sqlConnection) sqlConnection.Close(); if (null != sqlConnection) sqlConnection.Close();
} }
} }
public static InsiderTransactions GetLatestInsiderTransactions()
{
MySqlConnection sqlConnection = null;
MySqlDataReader sqlDataReader = null;
MySqlCommand sqlCommand =null;
InsiderTransactions insiderTransactions=new InsiderTransactions();
String strQuery = null;
try
{
StringBuilder sb = new StringBuilder();
sqlConnection = SqlUtils.CreateMySqlConnection(MainDataSource.Instance.LocateDataSource("market_data"));
sb.Append("SELECT symbol,filing_date,transaction_date,insider_name,ownership_type,securities,nature_of_transaction,number_or_value_acquired_disposed,price,form,sec_accession_number,form_row_number,modified FROM insidertransaction WHERE TRANSACTION_DATE=(SELECT MAX(TRANSACTION_DATE) FROM insidertransaction WHERE TRANSACTION_DATE<=NOW() LIMIT 1)");
strQuery = sb.ToString(); ;
sqlCommand = new MySqlCommand(strQuery, sqlConnection);
sqlCommand.CommandTimeout = SqlUtils.COMMAND_TIMEOUT;
sqlDataReader = sqlCommand.ExecuteReader();
while (sqlDataReader.Read())
{
InsiderTransaction insiderTransaction=new InsiderTransaction();
insiderTransaction.Symbol=sqlDataReader.GetString(0);
insiderTransaction.FilingDate=sqlDataReader.GetDateTime(1);
insiderTransaction.TransactionDate=sqlDataReader.GetDateTime(2);
insiderTransaction.InsiderName=sqlDataReader.GetString(3);
insiderTransaction.OwnershipType=sqlDataReader.GetString(4);
insiderTransaction.Securities=sqlDataReader.GetString(5);
insiderTransaction.NatureOfTransaction=sqlDataReader.GetString(6);
if(!sqlDataReader.IsDBNull(7))insiderTransaction.NumberOrValueAcquiredDisposed=sqlDataReader.GetDouble(7);
else insiderTransaction.NumberOrValueAcquiredDisposed=Double.NaN;
if(!sqlDataReader.IsDBNull(8))insiderTransaction.Price=sqlDataReader.GetDouble(8);
else insiderTransaction.Price=Double.NaN;
insiderTransaction.Form=sqlDataReader.GetString(9);
insiderTransaction.SECAccessionNumber=sqlDataReader.GetString(10);
insiderTransaction.FormRowNumber=sqlDataReader.GetString(11);
insiderTransaction.Modified=sqlDataReader.GetDateTime(12);
insiderTransactions.Add(insiderTransaction);
}
return insiderTransactions;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestInsiderTransactions Exception: {0}",exception.ToString()));
return null;
}
finally
{
if(null!=sqlCommand)sqlCommand.Dispose();
if (null != sqlDataReader) sqlDataReader.Close();
if (null != sqlConnection) sqlConnection.Close();
}
}
public static InsiderTransactions GetInsiderTransactions(String symbol) public static InsiderTransactions GetInsiderTransactions(String symbol)
{ {
MySqlConnection sqlConnection = null; MySqlConnection sqlConnection = null;
@@ -107,6 +160,7 @@ namespace MarketData.DataAccess
} }
} }
public static bool InsertInsiderTransactions(InsiderTransactions insiderTransactions) public static bool InsertInsiderTransactions(InsiderTransactions insiderTransactions)
{ {
MySqlConnection sqlConnection = null; MySqlConnection sqlConnection = null;

647
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -298,124 +298,6 @@ namespace MarketData.Helper
if(null!=httpNetResponse) httpNetResponse.Dispose(); if(null!=httpNetResponse) httpNetResponse.Dispose();
} }
} }
//public static PremarketElements GetPremarketData()
//{
// HttpNetResponse httpNetResponse=null;
// MemoryStream memoryStream=null;
// PremarketElements premarketElements=new PremarketElements();
// DateTime timestamp=DateTime.Now;
// try
// {
// StringBuilder sb=new StringBuilder();
// String strRequest=null;
// sb.Append("https://money.cnn.com/data/premarket/");
// strRequest=sb.ToString();
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
// httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
// if(!httpNetResponse.Success)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}' failed with {1}",strRequest,httpNetResponse.ErrorMessage));
// return null;
// }
// byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
// memoryStream=new MemoryStream(streamBytes);
// HtmlDocument htmlDocument=new HtmlDocument();
// htmlDocument.Load(memoryStream);
// HtmlNodeCollection divSection=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"wsod_dataTable\"]");
// if(null==divSection||0==divSection.Count)
// {
// if(!ParsePremarketElementsBySection(httpNetResponse.ResponseString,premarketElements))
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, cannot locate data in result for request '{0}'.",strRequest));
// return null;
// }
// return premarketElements;
// }
// HtmlNodeCollection rows=divSection[0].SelectNodes(".//tr");
// for(int index=0;index<rows.Count;index++)
// {
// PremarketElement premarketElement=new PremarketElement();
// HtmlNodeCollection headerColumns=rows[index].SelectNodes(".//th");
// if(null!=headerColumns&&headerColumns.Count>0) continue;
// HtmlNodeCollection dataColumns=rows[index].SelectNodes(".//td");
// if(dataColumns.Count<2) continue;
// String market=dataColumns[0].InnerText.Trim().ToUpper();
// market=market.Replace("&AMP;","&");
// String quoteHtml=dataColumns[1].InnerHtml;
// List<String> sections=Sections.GetSections(quoteHtml);
// if(3!=sections.Count)continue;
// String changeValue=sections[0];
// String changePercent=sections[2];
// premarketElement.Market=market;
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
// premarketElement.Timestamp=timestamp;
// premarketElements.Add(premarketElement);
// }
// if(0==premarketElements.Count)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, Did not find any results for request '{0}'.",strRequest));
// }
// return premarketElements;
// }
// catch(Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception);
// return null;
// }
// finally
// {
// if(null!=memoryStream) memoryStream.Close();
// if(null!=httpNetResponse) httpNetResponse.Dispose();
// }
//}
//private static bool ParsePremarketElementsBySection(String strResponseString,PremarketElements premarketElements)
//{
// if(null==strResponseString)return false;
// List<String> sections=Sections.GetSections(strResponseString);
// if(null==sections||0==sections.Count)return false;
// DateTime timestamp=DateTime.Now;
// PremarketElement premarketElement=null;
// String changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",8);
// String changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",4);
// if(!(null==changePercent||null==changeValue))
// {
// premarketElement=new PremarketElement();
// premarketElement.Market="S&P";
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
// premarketElement.Timestamp=timestamp;
// premarketElements.Add(premarketElement);
// }
// changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",8);
// changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",4);
// if(!(null==changePercent||null==changeValue))
// {
// premarketElement=new PremarketElement();
// premarketElement.Market="DOW";
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
// premarketElement.Timestamp=timestamp;
// premarketElements.Add(premarketElement);
// }
// changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",8);
// changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",4);
// if(!(null==changePercent||null==changeValue))
// {
// premarketElement=new PremarketElement();
// premarketElement.Market="NASDAQ";
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
// premarketElement.Timestamp=timestamp;
// premarketElements.Add(premarketElement);
// }
// return true;
//}
// ****************************************************************************************************************************************************************************** // ******************************************************************************************************************************************************************************
// ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S ******************************************* // ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S *******************************************
// ****************************************************************************************************************************************************************************** // ******************************************************************************************************************************************************************************
@@ -691,11 +573,11 @@ namespace MarketData.Helper
if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf"); if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf");
else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks"); else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks");
strRequest = sb.ToString(); strRequest = sb.ToString();
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest));
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy); // httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy);
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy); // httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy);
//httpNetResponse = HttpNetRequest.GetRequestNoEncodingV7(strRequest); // httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest);
if(!httpNetResponse.Success) if(!httpNetResponse.Success)
{ {
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1646,6 +1528,68 @@ namespace MarketData.Helper
// *************************************************************************************************************************************************************************************** // ***************************************************************************************************************************************************************************************
// ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E ********************************************************* // ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E *********************************************************
// *************************************************************************************************************************************************************************************** // ***************************************************************************************************************************************************************************************
//public static ETFHoldings GetETFHoldings(String etfSymbol)
//{
// MemoryStream memoryStream = null;
// ETFHoldings etfHoldings = new ETFHoldings();
// HttpNetResponse httpNetResponse=null;
// DateTime modified=DateTime.Now;
// try
// {
// StringBuilder sb = new StringBuilder();
// String strRequest;
// etfSymbol = etfSymbol.ToUpper();
// sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings");
// strRequest = sb.ToString();
// WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
// if(!httpNetResponse.Success)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
// return null;
// }
// String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
// strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\"");
// List<List<String>> items=LocateJSONItems(strResponse);
// if(null==items||0==items.Count)return null;
// for(int index=0;index<items.Count;index+=3)
// {
// if(index+3>items.Count)break;
// List<String> symbolPair=items[index];
// List<String> holdingNamePair=items[index+1];
// List<String> percentOfAssetsPair=items[index+2];
// if(!symbolPair[0].Equals("symbol"))continue;
// if(!holdingNamePair[0].Equals("holdingName"))continue;
// if(!percentOfAssetsPair[0].Equals("fmt"))continue;
// if(String.IsNullOrEmpty(symbolPair[1]))continue;
// ETFHolding etfHolding = new ETFHolding();
// etfHolding.ETFSymbol = etfSymbol;
// etfHolding.HoldingSymbolShareClass=null;
// etfHolding.HoldingCompanyName=holdingNamePair[1];
// etfHolding.HoldingSymbolShareClass=symbolPair[1];
// etfHolding.HoldingSymbol=symbolPair[1];
// etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]);
// etfHolding.Modified=modified;
// etfHoldings.Add(etfHolding);
// }
// return etfHoldings;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception);
// return null;
// }
// finally
// {
// if (null != memoryStream) memoryStream.Close();
// if(null!=httpNetResponse)httpNetResponse.Dispose();
// }
//}
public static ETFHoldings GetETFHoldings(String etfSymbol) public static ETFHoldings GetETFHoldings(String etfSymbol)
{ {
MemoryStream memoryStream = null; MemoryStream memoryStream = null;
@@ -1658,40 +1602,39 @@ namespace MarketData.Helper
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
String strRequest; String strRequest;
etfSymbol = etfSymbol.ToUpper(); etfSymbol = etfSymbol.ToUpper();
sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings"); sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings");
strRequest = sb.ToString(); strRequest = sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
if(!httpNetResponse.Success) if(!httpNetResponse.Success)
{ {
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null; return null;
} }
String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\""); List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
List<List<String>> items=LocateJSONItems(strResponse); if(null == sections || 0==sections.Count)return null;
if(null==items||0==items.Count)return null;
for(int index=0;index<items.Count;index+=3) byte[] streamBytes=Encoding.ASCII.GetBytes(sections[0]);
memoryStream=new MemoryStream(streamBytes);
HtmlDocument htmlDocument=new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes(".//td");
if(null==rows || 0==rows.Count)return null;
for(int rowIndex=0;rowIndex<rows.Count;rowIndex+=3)
{ {
if(index+3>items.Count)break; HtmlNode node = rows[rowIndex];
List<String> symbolPair=items[index];
List<String> holdingNamePair=items[index+1];
List<String> percentOfAssetsPair=items[index+2];
if(!symbolPair[0].Equals("symbol"))continue;
if(!holdingNamePair[0].Equals("holdingName"))continue;
if(!percentOfAssetsPair[0].Equals("fmt"))continue;
if(String.IsNullOrEmpty(symbolPair[1]))continue;
ETFHolding etfHolding = new ETFHolding(); ETFHolding etfHolding = new ETFHolding();
etfHolding.ETFSymbol = etfSymbol; etfHolding.ETFSymbol = etfSymbol;
etfHolding.HoldingSymbolShareClass=null; etfHolding.HoldingSymbolShareClass = null;
etfHolding.HoldingCompanyName=holdingNamePair[1]; etfHolding.HoldingCompanyName = rows[rowIndex].InnerText;
etfHolding.HoldingSymbolShareClass=symbolPair[1]; etfHolding.HoldingSymbolShareClass = rows[rowIndex+1].InnerText;
etfHolding.HoldingSymbol=symbolPair[1]; etfHolding.HoldingSymbol = rows[rowIndex+1].InnerText;
etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]); etfHolding.PercentOfAssets = FeedParser.ParseValue(rows[rowIndex+2].InnerText);
etfHolding.Modified=modified; etfHolding.Modified = modified;
etfHoldings.Add(etfHolding); etfHoldings.Add(etfHolding);
} }
return etfHoldings; return etfHoldings;
@@ -2265,50 +2208,6 @@ namespace MarketData.Helper
} }
} }
//private static AnalystPriceTarget GetAnalystPriceTargetYahoo(String symbol)
//{
// AnalystPriceTarget analystPriceTarget = null;
// HttpNetResponse httpNetResponse=null;
// try
// {
// StringBuilder sb = new StringBuilder();
// String strRequest;
// symbol = symbol.ToUpper();
// sb.Append("http://finance.yahoo.com/q/ao?s=").Append(symbol).Append("+Analyst+Opinion");
// WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetYahoo");
// strRequest=sb.ToString();
// MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetYahoo: {0}", strRequest));
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
// if(!httpNetResponse.Success)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
// return null;
// }
// analystPriceTarget=new AnalystPriceTarget();
// analystPriceTarget.Symbol=symbol;
// analystPriceTarget.Date=DateTime.Now.Date;
// String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
// analystPriceTarget.HighTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetHighPrice\":{\"raw\":",','));
// analystPriceTarget.MeanTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMeanPrice\":{\"raw\":",','));
// analystPriceTarget.LowTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetLowPrice\":{\"raw\":",','));
// analystPriceTarget.MedianTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMedianPrice\":{\"raw\":",','));
// if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
// double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
// double.IsNaN(analystPriceTarget.LowTargetPrice)&&
// double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
// return analystPriceTarget;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception);
// return null;
// }
// finally
// {
// if(null!=httpNetResponse)httpNetResponse.Dispose();
// }
//}
// **************************************************************************************************************************************************************************** // ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N ******************************************* // ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
// **************************************************************************************************************************************************************************** // ****************************************************************************************************************************************************************************
@@ -2804,15 +2703,6 @@ namespace MarketData.Helper
return currency; return currency;
} }
} }
//public static String GetMStarSecurityId(String symbol,String responseString)
//{
// String securityIdentifier=GetMStarSecurityId_1(responseString);
// if(null!=securityIdentifier)return securityIdentifier;
// securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
// if(null!=securityIdentifier)return securityIdentifier;
// return null;
//}
public static String GetMStarSecurityId(String symbol,String responseString) public static String GetMStarSecurityId(String symbol,String responseString)
{ {
@@ -2838,23 +2728,6 @@ namespace MarketData.Helper
//byId:{\"0P000003MU\":$} //byId:{\"0P000003MU\":$}
// AAPL:0P000000GY // AAPL:0P000000GY
//private static String GetMStarSecurityId_2(String responseString)
//{
// try
// {
// string token="byId";
// int index=responseString.IndexOf(token);
// if(-1==index)return null;
// string securityIdentifier=responseString.Substring(index+token.Length);
// securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
// return securityIdentifier;
// }
// catch(Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId encountered an exception:{0}",exception.ToString()));
// return null;
// }
//}
// 5","0P000003MU","MIDD",60 // 5","0P000003MU","MIDD",60
private static String GetMStarSecurityId_2(String symbol,String responseString) private static String GetMStarSecurityId_2(String symbol,String responseString)
@@ -4424,7 +4297,7 @@ namespace MarketData.Helper
} }
// ***************************************************************************************************************************************************************************** // *****************************************************************************************************************************************************************************
// **************************************************************************** B A L A N C E S H E E T - N A S D A Q **************************************************** // **************************************************************************** C U R R E N C Y C O N V E R S I O N ****************************************************
// ***************************************************************************************************************************************************************************** // *****************************************************************************************************************************************************************************
public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate) public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate)
{ {
@@ -4711,7 +4584,8 @@ namespace MarketData.Helper
strRequest = sb.ToString(); strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx"); WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx");
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); // httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy); // user agents refreshed in this version
//MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); //MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
if(!httpNetResponse.Success) if(!httpNetResponse.Success)
{ {
@@ -4890,10 +4764,12 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol));
return null; return null;
} }
// ***************************************************************************************************************
// ******************** L A T E S T P R I C E R E T R I E V A L F O R I N T R A - D A Y F E E D ********
// ***************************************************************************************************************
// This should be proxied through TOR because Yahoo tracks IP address.
// This is used in the intra-day price feed public static Price GetLatestPriceYahoo(String symbol)
private static Price GetLatestPriceYahoo(String symbol)
{ {
HttpNetResponse httpNetResponse=null; HttpNetResponse httpNetResponse=null;
try try
@@ -4904,15 +4780,16 @@ namespace MarketData.Helper
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol); sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol);
strRequest=sb.ToString(); strRequest=sb.ToString();
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo"); WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo");
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy);
if(!httpNetResponse.Success) if(!httpNetResponse.Success)
{ {
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null; return null;
} }
Price price=new Price(); Price price=new Price();
price.Symbol=symbol.ToUpper(); price.Symbol=symbol.ToUpper();
price.Date=DateTime.Now; price.Date=DateTime.Now;
price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN;
price.Source=Price.PriceSource.Yahoo; price.Source=Price.PriceSource.Yahoo;
return BuildPriceFromResponse(httpNetResponse.ResponseString,price); return BuildPriceFromResponse(httpNetResponse.ResponseString,price);
} }
@@ -4926,8 +4803,250 @@ namespace MarketData.Helper
if(null!=httpNetResponse) httpNetResponse.Dispose(); if(null!=httpNetResponse) httpNetResponse.Dispose();
} }
} }
public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
{
if(httpNetResponse.Contains("HTML_TAG_END"))
{
price = BuildPriceFromSections(httpNetResponse, price);
}
else
{
price = BuildPriceFromTable(httpNetResponse, price);
}
return price;
}
private static Price BuildPriceFromSections(String httpNetResponse,Price price)
{
try
{
String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END");
List<String> sections = null;
sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
String marker ="data-symbol=\""+price.Symbol+"\"";
List<String> dataSections = sections.Where(x => x.Contains(marker)).ToList();
dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
if(dataSections.Count>0)
{
dataSections = Sections.GetSections(dataSections[0]);
if(null!=dataSections && dataSections.Count>0)
{
price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]);
}
}
marker = "<fin-streamer data-symbol=\""+price.Symbol+"\"";
sections = sections.Where(x => x.StartsWith(marker)).ToList();
for(int index=0;index<sections.Count;index++)
{
String sectionText = sections[index];
String itemValue = null;
if(sectionText.Contains("regularMarketPreviousClose"))
{
itemValue = Sections.GetSections(sections[index])[0];
}
else if(sectionText.Contains("regularMarketOpen"))
{
itemValue = Sections.GetSections(sections[index])[0];
price.Open=FeedParser.ParseValue(itemValue);
}
else if(sectionText.Contains("regularMarketDayRange"))
{
itemValue = Sections.GetSections(sections[index])[0];
String[] subItems = itemValue.Split('-');
price.Low=FeedParser.ParseValue(subItems[0]);
price.High=FeedParser.ParseValue(subItems[1]);
if(double.IsNaN(price.Close))
{
price.Close=price.AdjClose=(price.Low+price.High)/2.00;
}
}
else if(sectionText.Contains("fiftyTwoWeekRange"))
{
itemValue = Sections.GetSections(sections[index])[0];
}
else if(sectionText.Contains("regularMarketVolume"))
{
itemValue = Sections.GetSections(sections[index])[0];
price.Volume=FeedParser.ParseValueLong(itemValue);
}
else if(sectionText.Contains("averageVolume"))
{
itemValue = Sections.GetSections(sections[index])[0];
}
else if(sectionText.Contains("averageVolume"))
{
itemValue = Sections.GetSections(sections[index])[0];
}
else if(sectionText.Contains("marketCap"))
{
itemValue = Sections.GetSections(sections[index])[0];
}
}
CheckPrice(price);
return price;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: {0}",exception.ToString()));
return null;
}
}
private static Price BuildPriceFromTable(String httpNetResponse,Price price)
{
MemoryStream memoryStream = null;
try
{
List<String> sections = null;
double bid = double.NaN;
double bidContracts = double.NaN;
double ask = double.NaN;
double askContracts = double.NaN;
String symbol=price.Symbol.ToUpper();
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse);
memoryStream = new MemoryStream(streamBytes);
HtmlDocument htmlDocument = new HtmlDocument();
htmlDocument.Load(memoryStream);
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]");
if(null == table || 0 == table.Count)return null;
HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
if (null == rows || 0 == rows.Count) return null;
for(int index=0;index< rows.Count; index++)
{
int itemIndex=0;
String itemValue=null;
String innerHtml = rows[index].InnerHtml;
sections=Sections.GetSections(innerHtml);
if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
}
else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
price.Open=FeedParser.ParseValue(itemValue);
}
else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
String[] bidContractSplit = itemValue.Split('x');
bid=FeedParser.ParseValue(bidContractSplit[0]);
bidContracts=FeedParser.ParseValue(bidContractSplit[1]);
}
else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
String[] askContractSplit = itemValue.Split('x');
ask = FeedParser.ParseValue(askContractSplit[0]);
askContracts = FeedParser.ParseValue(askContractSplit[1]);
}
else if(Sections.FindInSections(sections, "Day&#x27;s Range",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
String[] subItems=itemValue.Split('-');
price.Low=FeedParser.ParseValue(subItems[0]);
price.High=FeedParser.ParseValue(subItems[1]);
}
else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
}
else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+4];
price.Volume=FeedParser.ParseValueLong(itemValue);
}
else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true))
{
itemValue = sections[itemIndex+3];
}
}
if (double.IsNaN(price.Close))
{
sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
String marker = "data-symbol=\"" + price.Symbol + "\"";
sections = sections.Where(x => x.Contains(marker)).ToList();
sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
if (sections.Count > 0)
{
sections = Sections.GetSections(sections[0]);
if (null != sections && sections.Count > 0)
{
price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]);
}
}
}
if(double.IsNaN(price.Close))
{
double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts);
if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice;
if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00;
}
CheckPrice(price);
return price;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: {0}",exception.ToString()));
return null;
}
finally
{
if(null!=memoryStream)
{
memoryStream.Close();
memoryStream.Dispose();
memoryStream=null;
}
}
}
private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts)
{
if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN;
double totalContracts = bidContracts + askContracts;
double bidWeight = bidContracts/totalContracts;
double askWeight = askContracts/totalContracts;
double weightedBid = bid*bidWeight;
double weightedAsk = ask*askWeight;
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2)));
return weightedBid + weightedAsk;
}
public static void CheckPrice(Price price)
{
if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
{
price.Close=price.AdjClose=(price.High+price.Low)/2.00;
}
if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
{ // if the close price is garbage set it to the midpoint of the high and the low
if(price.Close<price.Low||price.Close>price.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
}
if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close;
if(double.IsNaN(price.Open))price.Open=0.00;
if(double.IsNaN(price.Low))price.Low=0.00;
if(double.IsNaN(price.High))price.High=0.00;
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
{
price.Open=price.High=price.Low=price.Close;
}
}
// This is used in the intra-day price feed. // This is used in the intra-day price feed.
private static Price GetLatestPriceBigCharts(String symbol) public static Price GetLatestPriceBigCharts(String symbol)
{ {
HttpNetResponse httpNetResponse=null; HttpNetResponse httpNetResponse=null;
MemoryStream memoryStream=null; MemoryStream memoryStream=null;
@@ -5123,6 +5242,12 @@ namespace MarketData.Helper
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();} if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
} }
} }
// *******************************************************************************************************************************************************************************
// *******************************************************************************************************************************************************************************
// *******************************************************************************************************************************************************************************
// ******************************************************************************************************************************************************************************* // *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ******************************************************** // ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
// ******************************************************************************************************************************************************************************* // *******************************************************************************************************************************************************************************
@@ -5349,88 +5474,6 @@ namespace MarketData.Helper
// ****************************************************************************************************************************************************************************************************************** // ******************************************************************************************************************************************************************************************************************
// This function attempts to locate the quote data within the raw data returned from Yahoo. It should be interpreted as intraday data because
// the market date cannot be located within the raw data response.
public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
{
String strResponse=Utility.KeepAfter(httpNetResponse,"/* -- Data -- */");
String symbol=price.Symbol.ToUpper();
bool isMutualFund=false;
String findPattern1="\"quoteType\":\"MUTUALFUND\",\"symbol\":"+"\""+symbol+"\"";
String findPattern2="\"quoteType\":\"EQUITY\",\"symbol\":"+"\""+symbol+"\"";
String findPattern3="\"quoteType\":\"ETF\",\"symbol\":"+"\""+symbol+"\"";
String findPattern4="\"quoteType\":\"INDEX\",\"symbol\":"+"\""+symbol+"\"";
strResponse=Utility.KeepAfter(httpNetResponse,findPattern1);
if(null!=strResponse)isMutualFund=true;
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern2);
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern3);
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern4);
if(null==strResponse)return null;
strResponse=Utility.KeepBefore(strResponse,"MOST_ACTIVE_TITLE");
price.Open=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketOpen\":{\"raw\":",','));
price.High=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayHigh\":{\"raw\":",','));
price.Low=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayLow\":{\"raw\":",','));
price.Close=FeedParser.ParseValue(Utility.FindFirst(Utility.KeepAfterLast(strResponse,"\"regularMarketDayLow\":{\"raw\":"),"\"regularMarketPrice\":{\"raw\":",','));
price.Volume=FeedParser.ParseValueLong(Utility.FindFirst(strResponse,"\"regularMarketVolume\":{\"raw\":",','));
if(double.NaN.Equals(price.Close))price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPrice\":{\"raw\":",','));
if(double.IsNaN(price.Close)) // try to find raw quote data
{
String strQuote=Utility.KeepAfter(httpNetResponse,"<!-- react-text: 36");
if(null==strQuote&&httpNetResponse.Contains("<span data-reactid=\"38\">At close:"))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***Retrieving price from BigCharts for {0}***",symbol));
return GetPriceAsOf(symbol,DateTime.Now); // if the market is closed then get the price from BigCharts
}
if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,"-->",'<'));
else if(null==strQuote&&httpNetResponse.Contains("data-reactid=\"38\">As of"))
{
strQuote=Utility.KeepAfter(httpNetResponse,"class=\"Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)\"");
if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,">",'<'));
}
if(!double.IsNaN(price.Close))MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Found quote data for {0}->{1}",symbol,Utility.FormatCurrency(price.Close)));
}
if(double.IsNaN(price.Close))
{
double bid=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"bid\":{\"raw\":",','));
double ask=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"ask\":{\"raw\":",','));
if(!double.IsNaN(bid)&&!double.IsNaN(ask))price.Close=(bid+ask)/2.00;
}
if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
{
price.Close=price.AdjClose=(price.High+price.Low)/2.00;
}
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
{
// if this is a mutual fund then the regularMarketPreviousClose may actually be the close from two days ago and not last night. Goto BigCharts in this case
if(isMutualFund)price=GetPriceAsOf(symbol,DateTime.Now);
else price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPreviousClose\":{\"raw\":",','));
if(null==price||double.IsNaN(price.Close))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",symbol));
return null;
}
price.Open=price.High=price.Low=price.AdjClose=price.Close;
return price;
}
if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
{ // if the close price is garbage set it to the midpoint of the high and the low
if(price.Close<price.Low||price.Close>price.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
}
price.AdjClose=price.Close;
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No open,high,low,close price for {0}",price.Symbol));
return null;
}
if(double.IsNaN(price.Open))price.Open=0.00;
if(double.IsNaN(price.Low))price.Low=0.00;
if(double.IsNaN(price.High))price.High=0.00;
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
{
price.Open=price.High=price.Low=price.Close;
}
return price;
}
// **************************************************************************************************************************************************************************************** // ****************************************************************************************************************************************************************************************
// **************************************************************************** E N D D A T A F E E D S ********************************************************************************* // **************************************************************************** E N D D A T A F E E D S *********************************************************************************
// **************************************************************************************************************************************************************************************** // ****************************************************************************************************************************************************************************************

208
MarketDataLib/Integration/HttpNetRequest.cs
View File

@@ -688,6 +688,97 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]"); MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]");
} }
} }
public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
WebException lastWebException=null;
ServicePointManager.Expect100Continue=true;
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
Random random=new Random();
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
String[] userAgents=
{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3"
};
int MAX_RETRIES=5;
int TIMEOUT_BETWEEN_RETRIES=1000;
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
for(int count=0;count<MAX_RETRIES;count++)
{
CookieContainer cookieContainer=new CookieContainer();
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.ProtocolVersion=HttpVersion.Version10;
webRequest.CookieContainer=cookieContainer;
webRequest.KeepAlive=false;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
double randomNumber=random.NextDouble(); // number between 0 and 1
int index=(int)(randomNumber*((double)userAgents.Length));
if(index>=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
try
{
webResponse=(HttpWebResponse)webRequest.GetResponse();
Stream responseStream=webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0==charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
}
webResponse.Close();
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(WebException webException)
{
if(webException.Message.Contains("(404) Not Found")) return new HttpNetResponse(webResponse,strRequest,false,null==lastWebException?"":lastWebException.Message);
lastWebException=webException;
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null) public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{ {
HttpWebResponse webResponse=null; HttpWebResponse webResponse=null;
@@ -796,6 +887,7 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]"); MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
} }
} }
public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{ {
HttpWebResponse webResponse = null; HttpWebResponse webResponse = null;
@@ -844,55 +936,73 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
} }
} }
// public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
// { public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
// HttpWebResponse webResponse = null; {
// try HttpWebResponse webResponse = null;
// { try
// MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); {
// int charCount = 0; Random random = new Random();
// byte[] buffer = new byte[8192]; String[] userAgents=
// StringBuilder sb = new StringBuilder(); {
// bool expect100Condition = ServicePointManager.Expect100Continue; "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
// SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
// ServicePointManager.Expect100Continue = true; };
// ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
// if(null!=webProxy)webRequest.Proxy=webProxy; int charCount = 0;
// webRequest.Timeout = webRequestTimeoutMS; byte[] buffer = new byte[8192];
// webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); StringBuilder sb = new StringBuilder();
// webRequest.Headers.Add("Accept-Encoding: *"); bool expect100Condition = ServicePointManager.Expect100Continue;
// webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
//// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; ServicePointManager.Expect100Continue = true;
// webRequest.UserAgent = GetUserAgent(true); ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// webRequest.KeepAlive = true; HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
// webRequest.CookieContainer = new CookieContainer(); if(null!=webProxy)webRequest.Proxy=webProxy;
// webResponse = (HttpWebResponse)webRequest.GetResponse(); webRequest.Timeout = webRequestTimeoutMS;
// Stream responseStream = webResponse.GetResponseStream(); webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
// while (true) webRequest.Headers.Add("Accept-Encoding: None");
// { webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
// charCount = responseStream.Read(buffer, 0, buffer.Length); // webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
// if (0 == charCount) break; webRequest.KeepAlive = true;
// sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); webRequest.CookieContainer = new CookieContainer();
// }
// webResponse.Close(); double randomNumber=random.NextDouble(); // number between 0 and 1
// ServicePointManager.Expect100Continue = expect100Condition; int index=(int)(randomNumber*((double)userAgents.Length));
// ServicePointManager.SecurityProtocol = securityProtocolType; if(index>=userAgents.Length) index=userAgents.Length-1;
// return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); String userAgent=userAgents[index];
// } webRequest.UserAgent=userAgent;
// catch (WebException webException)
// {
// return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); webResponse = (HttpWebResponse)webRequest.GetResponse();
// } Stream responseStream = webResponse.GetResponseStream();
// catch (Exception exception) while (true)
// { {
// return new HttpNetResponse(webResponse, strRequest, false, exception.Message); charCount = responseStream.Read(buffer, 0, buffer.Length);
// } if (0 == charCount) break;
// finally sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
// { }
// MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); webResponse.Close();
// } ServicePointManager.Expect100Continue = expect100Condition;
// } ServicePointManager.SecurityProtocol = securityProtocolType;
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS) public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
{ {
HttpWebResponse webResponse = null; HttpWebResponse webResponse = null;

6
MarketDataLib/Utility/FeedParser.cs
View File

@@ -231,6 +231,12 @@ namespace MarketData.Utils
multiplier = -1.00; multiplier = -1.00;
} }
if (strText.Equals("-")) return double.NaN; if (strText.Equals("-")) return double.NaN;
if (strText[strText.Length - 1].Equals('T'))
{
strText = strText.Replace("T", "");
value = double.Parse(strText);
value *= 1000000000000;
}
if (strText[strText.Length - 1].Equals('B')) if (strText[strText.Length - 1].Equals('B'))
{ {
strText = strText.Replace("B", ""); strText = strText.Replace("B", "");

31
MarketDataLib/Utility/Sections.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using MarketData.Utils;
namespace MarketDataLib.Utility namespace MarketDataLib.Utility
{ {
@@ -28,6 +29,24 @@ namespace MarketDataLib.Utility
if(String.IsNullOrEmpty(strContainingString))return null; if(String.IsNullOrEmpty(strContainingString))return null;
return strContainingString; return strContainingString;
} }
public static List<String> GetAllItemsInSections(String strInput, String sectionName)
{
int searchIndex = 0;
List<String> sectionItems = new List<string>();
while (true)
{
String itemsInSection = GetItemsInSection(strInput, sectionName, ref searchIndex);
if(null==itemsInSection)break;
sectionItems.Add(itemsInSection);
searchIndex++;
}
return sectionItems;
}
public static List<String> GetSections(String strInput) public static List<String> GetSections(String strInput)
{ {
try try
@@ -36,6 +55,7 @@ namespace MarketDataLib.Utility
int index=0; int index=0;
String strLiteral=null; String strLiteral=null;
if(null==strInput)return null;
while(index<strInput.Length) while(index<strInput.Length)
{ {
char ch=strInput[index]; char ch=strInput[index];
@@ -379,6 +399,17 @@ namespace MarketDataLib.Utility
return null; return null;
} }
} }
private static String GetItem(String strInput, String item)
{
if (null == strInput) return null;
int startIndex = strInput.IndexOf(item);
if (-1 == startIndex) return null;
strInput = strInput.Substring(startIndex);
String strItem = MarketData.Utils.Utility.BetweenString(strInput, ">", "<");
return strItem;
}
private static String ScanSection(String strInput, ref int index) private static String ScanSection(String strInput, ref int index)
{ {
try try