diff --git a/MarketDataLib/Helper/ETFHoldingsMarketDataHelper.cs b/MarketDataLib/Helper/ETFHoldingsMarketDataHelper.cs index c11eab7..2729892 100644 --- a/MarketDataLib/Helper/ETFHoldingsMarketDataHelper.cs +++ b/MarketDataLib/Helper/ETFHoldingsMarketDataHelper.cs @@ -1,7 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using MarketData.MarketDataModel; using MarketData.DataAccess; @@ -11,9 +9,10 @@ namespace MarketData.Helper { public class ETFHoldingsMarketDataHelper { - private static int MaxThreads = (int)ThreadHelperEnum.MaxThreads; + private static int MaxThreads = 10; // 5 (int)ThreadHelperEnum.MaxThreads; private List symbols; private int currentIndex = 0; + private int WAIT_BETWEEN_REQUESTS=250; // 500 public ETFHoldingsMarketDataHelper() { @@ -39,6 +38,7 @@ namespace MarketData.Helper { ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadETFHoldings, threadHelper); + try{Thread.Sleep(WAIT_BETWEEN_REQUESTS);}catch(Exception){;} } MDTrace.WriteLine(LogLevel.DEBUG,"Load ETF Holdings, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); @@ -70,6 +70,7 @@ namespace MarketData.Helper { ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]); ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadETFHoldings, threadHelper); + try{Thread.Sleep(WAIT_BETWEEN_REQUESTS);}catch(Exception){;} } MDTrace.WriteLine(LogLevel.DEBUG,"Load ETF Holdings, waiting for queued items to complete."); WaitHandle.WaitAll(resetEvents); @@ -120,9 +121,10 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,"Error adding ETF holdings for " + symbol); return; } + MDTrace.WriteLine(LogLevel.DEBUG,"********* E T F H O L D I N G S ********"); foreach (ETFHolding etfHolding in etfHoldings) { - MDTrace.WriteLine(LogLevel.DEBUG,etfHolding.HoldingSymbol + "," + etfHolding.HoldingSymbolShareClass + "," + etfHolding.HoldingCompanyName); + MDTrace.WriteLine(LogLevel.DEBUG,etfHolding.ETFSymbol + ","+etfHolding.HoldingSymbol + "," + etfHolding.HoldingSymbolShareClass + "," + etfHolding.HoldingCompanyName); } MDTrace.WriteLine(LogLevel.DEBUG,"************************"); } diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index 6029b63..2215ce9 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -52,176 +52,48 @@ namespace MarketData.Helper public static String SEC_BASE_URL="https://www.sec.gov"; - public static Dictionary RunTests() - { - DateGenerator dateGenerator=new DateGenerator(); - Dictionary items=new Dictionary(); - String symbol="MIDD"; - String dividendSymbol="AAPL"; - String insiderTransactionSymbol="AAVL"; - String etfSymbol="JFNNX"; - String analystPriceTargetSymbol="MGPI"; - String sourceCurrency="USD"; - DateTime analysisDate=dateGenerator.FindPrevBusinessDay(DateTime.Now); + public static bool GetWorldTime() + { + HttpNetResponse httpNetResponse=null; - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetDividendHistory"); - DividendHistory dividendHistory=GetDividendHistory(dividendSymbol); - if(null==dividendHistory||0==dividendHistory.Count)items.Add("GetDividendHistory",false); - else items.Add("GetDividendHistory",true); + try + { + StringBuilder sb = new StringBuilder(); + sb.Append("https://www.timeanddate.com/worldclock/"); + String strRequest = sb.ToString(); + WebProxy webProxy=HttpNetRequest.GetProxy(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime:{0}",strRequest)); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4A(strRequest, null, webProxy); + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return false; + } - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetPremarketData"); - PremarketElements premarketElements=GetPremarketData(); - if(null==premarketElements||0==premarketElements.Count)items.Add("GetPremarketData",false); - else items.Add("GetPremarketData",true); + if(String.IsNullOrEmpty(httpNetResponse.ResponseString)) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest)); + return false; + } - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetEarningsAnnouncements"); - EarningsAnnouncements earningsAnnouncements=GetEarningsAnnouncements(symbol); - if(null==earningsAnnouncements||0==earningsAnnouncements.Count)items.Add("GetEarningsAnnouncements",false); - else items.Add("GetEarningsAnnouncements",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetZacksRank"); - ZacksRank zacksRank=GetZacksRank(symbol); - if(null==zacksRank)items.Add("GetZacksRank",false); - else items.Add("GetZacksRank",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetSplits"); - Splits splits=GetSplits(); - if(null==splits)items.Add("GetSplits",false); - else items.Add("GetSplits",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetGDPPerCapita"); - EconomicIndicators economicIndicators=GetGDPPerCapita(); - if(null==economicIndicators)items.Add("GetGDPPerCapita",false); - else items.Add("GetGDPPerCapita",true); - - //MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetOptions"); - //Options options=GetOptions(symbol); - //if(null==options||0==options.Count) items.Add("GetOptions",false); - //else items.Add("GetOptions",true); - items.Add("GetOptions",false); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetLatestAnalystRatings"); - AnalystRatings analystRatings=GetLatestAnalystRatings(); - if(null==analystRatings||0==analystRatings.Count)items.Add("GetLatestAnalystRatings",false); - else items.Add("GetLatestAnalystRatings",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystRatingsMarketBeat"); - analystRatings=GetAnalystRatingsMarketBeat(symbol); - if(null==analystRatings||0==analystRatings.Count)items.Add("GetAnalystRatingsMarketBeat",false); - else items.Add("GetAnalystRatingsMarketBeat",true);; - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCIK"); - String strCik=GetCIK(symbol); - if(null==strCik)items.Add("GetCIK",false); - else items.Add("GetCIK",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetSECFilings"); - String cik=PricingDA.GetCIKForSymbol(symbol); - SECFilings secFilings=GetSECFilings(symbol,cik); - if(null==secFilings||0==secFilings.Count)items.Add("GetSECFilings",false); - else items.Add("GetSECFilings",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetYieldCurve"); - YieldCurve yieldCurve=GetYieldCurve(analysisDate.Year); - if(null==yieldCurve||0==yieldCurve.Count)items.Add("GetYieldCurve",false); - else items.Add("GetYieldCurve",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetETFHoldings"); - ETFHoldings etfHoldings=GetETFHoldings(etfSymbol); - if(null==etfHoldings||0==etfHoldings.Count)items.Add("GetETFHoldings",false); - else items.Add("GetETFHoldings",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetInsiderTransactions"); - InsiderTransactions insiderTransactions=GetInsiderTransactions(insiderTransactionSymbol); - if(null==insiderTransactions||0==insiderTransactions.Count)items.Add("GetInsiderTransactions",false); - else items.Add("GetInsiderTransactions",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCompanyProfile"); - CompanyProfile companyProfile=GetCompanyProfile(symbol); - if(null==companyProfile)items.Add("GetCompanyProfile",false); - else items.Add("GetCompanyProfile",true); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length)); - //MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCompanyDescription"); - //String companyDescription=GetCompanyDescription(symbol); - //if(null==companyDescription)items.Add("GetCompanyDescription",false); - //else items.Add("GetCompanyDescription",true); + return true; - Headlines headlines=HeadlinesMarketDataHelper.GetHeadlinesEx(symbol); - if(null==headlines || 0==headlines.Count)items.Add("GetHeadlines",false); - else items.Add("GetHeadlines",true); - - //MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystPriceTargetYahoo"); - //AnalystPriceTarget analystPriceTarget=GetAnalystPriceTargetYahoo(analystPriceTargetSymbol); - //if(null==analystPriceTarget)items.Add("GetAnalystPriceTargetYahoo",false); - //else items.Add("GetAnalystPriceTargetYahoo",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystPriceTargetMarketBeat"); - AnalystPriceTarget analystPriceTarget=GetAnalystPriceTargetMarketBeat(analystPriceTargetSymbol); - if(null==analystPriceTarget)items.Add("GetAnalystPriceTargetYahoo",false); - else items.Add("GetAnalystPriceTargetMarketBeat",true); + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception); + return false; + } + finally + { + if(null!=httpNetResponse)httpNetResponse.Dispose(); + } + } - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetHistoricalValues"); - Dictionary timeSeries=GetHistoricalValues(symbol); - if(null==timeSeries)items.Add("GetHistoricalValues",false); - else items.Add("GetHistoricalValues",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetIncomeStatement"); - List incomeStatement=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual); - if(null==incomeStatement||0==incomeStatement.Count)items.Add("GetIncomeStatement",false); - else items.Add("GetIncomeStatement",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetBalanceSheetNASDAQ"); - List balanceSheet=GetBalanceSheetNASDAQ(symbol,BalanceSheet.PeriodType.Annual); - if(null==balanceSheet||0==balanceSheet.Count)items.Add("GetBalanceSheetNASDAQ",false); - else items.Add("GetBalanceSheetNASDAQ",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCashflowStatement"); - List cashflowStatement=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual); - if(null==cashflowStatement||0==cashflowStatement.Count)items.Add("GetCashflowStatement",false); - else items.Add("GetCashflowStatement",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCurrencyConversion"); - CurrencyConversionCollection currencyConversionCollection=GetCurrencyConversion(sourceCurrency,analysisDate); - if(null==currencyConversionCollection||0==currencyConversionCollection.Count)items.Add("GetCurrencyConversion",false); - else items.Add("GetCurrencyConversion",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetFundamentalFinViz"); - Fundamental fundamental=GetFundamentalFinViz(symbol); - if(null==fundamental)items.Add("GetFundamentalFinViz",false); - else items.Add("GetFundamentalFinViz",true); - - MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetFundamental"); - fundamental=GetFundamental(symbol); - if(null==fundamental)items.Add("GetFundamental",false); - else items.Add("GetFundamental",true); - - fundamental=FundamentalDA.GetFundamental(symbol); - if(!PopulateKeyStatistics(fundamental))items.Add("PopulateKeyStatistics",false); - else items.Add("PopulateKeyStatistics",true); - - if(!PopulateFinancials(fundamental))items.Add("PopulateFinancials",false); - else items.Add("PopulateFinancials",true); - - Price price=GetLatestPriceYahoo(symbol); - if(null==price)items.Add("GetLatestPriceYahoo",false); - else items.Add("GetLatestPriceYahoo",true); - - price=GetLatestPriceBigCharts(symbol); - if(null==price)items.Add("GetLatestPriceBigCharts",false); - else items.Add("GetLatestPriceBigCharts",true); - - price=GetPriceAsOf(symbol,analysisDate); - if(null==price)items.Add("GetPriceAsOf",false); - else items.Add("GetPriceAsOf",true); - - Prices prices=GetDailyPrices(symbol,analysisDate,analysisDate); - if(null==prices||0==prices.Count)items.Add("GetDailyPrices",false); - else items.Add("GetDailyPrices",true); - - return items; - } // ****************************************************************************************************************************************************************************** // ************************************************************************************ P R E M A R K E T D A T A *********************************************************** @@ -1528,67 +1400,6 @@ namespace MarketData.Helper // *************************************************************************************************************************************************************************************** // ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E ********************************************************* // *************************************************************************************************************************************************************************************** - //public static ETFHoldings GetETFHoldings(String etfSymbol) - //{ - // MemoryStream memoryStream = null; - // ETFHoldings etfHoldings = new ETFHoldings(); - // HttpNetResponse httpNetResponse=null; - // DateTime modified=DateTime.Now; - - // try - // { - // StringBuilder sb = new StringBuilder(); - // String strRequest; - // etfSymbol = etfSymbol.ToUpper(); - // sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings"); - // strRequest = sb.ToString(); - - // WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); - // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); - // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy); - // if(!httpNetResponse.Success) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - // return null; - // } - // String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */"); - // strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\""); - // List> items=LocateJSONItems(strResponse); - // if(null==items||0==items.Count)return null; - // for(int index=0;indexitems.Count)break; - // List symbolPair=items[index]; - // List holdingNamePair=items[index+1]; - // List percentOfAssetsPair=items[index+2]; - // if(!symbolPair[0].Equals("symbol"))continue; - // if(!holdingNamePair[0].Equals("holdingName"))continue; - // if(!percentOfAssetsPair[0].Equals("fmt"))continue; - // if(String.IsNullOrEmpty(symbolPair[1]))continue; - // ETFHolding etfHolding = new ETFHolding(); - // etfHolding.ETFSymbol = etfSymbol; - // etfHolding.HoldingSymbolShareClass=null; - // etfHolding.HoldingCompanyName=holdingNamePair[1]; - // etfHolding.HoldingSymbolShareClass=symbolPair[1]; - // etfHolding.HoldingSymbol=symbolPair[1]; - // etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]); - // etfHolding.Modified=modified; - // etfHoldings.Add(etfHolding); - // } - // return etfHoldings; - // } - // catch (Exception exception) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,exception); - // return null; - // } - // finally - // { - // if (null != memoryStream) memoryStream.Close(); - // if(null!=httpNetResponse)httpNetResponse.Dispose(); - // } - //} - public static ETFHoldings GetETFHoldings(String etfSymbol) { @@ -1596,6 +1407,7 @@ namespace MarketData.Helper ETFHoldings etfHoldings = new ETFHoldings(); HttpNetResponse httpNetResponse=null; DateTime modified=DateTime.Now; + int TIMEOUT_MS=1000*30; try { @@ -1607,8 +1419,7 @@ namespace MarketData.Helper WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy); -// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS , webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1616,7 +1427,17 @@ namespace MarketData.Helper } List sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table"); - if(null == sections || 0==sections.Count)return null; + if(null == sections || 1!=sections.Count) + { + MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unexpected items in section"); + return null; + } + String marker=" divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div"); + if(null==divSections || 0==divSections.Count)return null; + List searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList(); + if(null!=searchPattern && searchPattern.Count>0) + { + httpNetResponse.Dispose(); + sb = new StringBuilder(); + sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNYSE); + strRequest = sb.ToString(); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy); + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return null; + } + divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div"); + searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList(); + if(null!=searchPattern && searchPattern.Count>0) + { + return null; + } + } + + List sections = Sections.GetSections(httpNetResponse.ResponseString); + sections = Sections.FlattenSection(sections); + + int index=0; + Price price = new Price(); price.Symbol = symbol.ToUpper(); - price.Source=Price.PriceSource.Fidelity; + price.Source=Price.PriceSource.Google; price.Close=price.Open=price.High=price.Low=price.AdjClose=double.NaN; + price.Open=0.00; price.Date = DateTime.Now; - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - memoryStream = new MemoryStream(streamBytes); - HtmlDocument htmlDocument = new HtmlDocument(); - htmlDocument.Load(memoryStream); - HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quotesDetailsTable\"]"); - if(null==table||0==table.Count)return null; - HtmlNodeCollection rows = table[0].SelectNodes(".//tr"); - if(null==rows || 0==rows.Count)return null; - double bid=double.NaN; - double ask=double.NaN; - for(int rowIndex=0;rowIndex sections=Sections.GetSections(rows[rowIndex].InnerHtml); - if(null==sections || 0==sections.Count)continue; + if(Sections.FindInSections(sections, "After Hours:", 0,ref index, true)) + { + price.Close=price.AdjClose=FeedParser.ParseValue(sections[index-1]); + } + else if(Sections.FindInSections(sections, "Share", 0,ref index, true)) + { + price.Close=price.AdjClose=FeedParser.ParseValue(sections[index+1]); + } - itemIndex=sections.FindIndex(delegate(string str){return str.StartsWith("As of:");}); - if(-1!=itemIndex) - { - String strDate=Utility.BetweenString(sections[itemIndex],":",","); - if(null==strDate)continue; - { - strDate=strDate.Trim(); - price.Date=Utility.ParseDate(strDate); - } - } - itemIndex=sections.IndexOf("Last Trade [tick]"); - if(-1!=itemIndex) - { - price.Close=FeedParser.ParseValue(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Volume"); - if(-1!=itemIndex) - { - price.Volume=FeedParser.ParseValueLong(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Bid"); - if(-1!=itemIndex) - { - bid=FeedParser.ParseValue(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Ask"); - if(-1!=itemIndex) - { - ask=FeedParser.ParseValue(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Open"); - if(-1!=itemIndex) - { - price.Open=FeedParser.ParseValue(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Day High"); - if(-1!=itemIndex) - { - price.High=FeedParser.ParseValue(sections[itemIndex+2]); - } - itemIndex=sections.IndexOf("Day Low"); - if(-1!=itemIndex) - { - price.Low=FeedParser.ParseValue(sections[itemIndex+2]); - } - } - if((double.IsNaN(price.Close) || 0==price.Close) && !double.IsNaN(bid) && !double.IsNaN(ask)) - { - price.Close=(bid+ask)/2.00; - } - if(0!=price.Close && 0==price.Open && 0==price.High && 0==price.Low) - { - price.Open=price.High=price.Low=price.Close; - } - price.AdjClose=price.Close; - if(double.IsNaN(price.Open))price.Open=0.00; - if(double.IsNaN(price.Low))price.Low=0.00; - if(double.IsNaN(price.High))price.High=0.00; - if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close)) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",price.Symbol)); - return null; - } - if(!price.IsValid)return null; - return price; + if(Sections.FindInSections(sections, "Previous close", 0,ref index, true)) + { + price.Open=FeedParser.ParseValue(sections[index+2]); + } + + if(Sections.FindInSections(sections, "Day range", 0,ref index, true)) + { + String itemValue = sections[index+2]; + String[] highLow = itemValue.Split('-'); + price.Low=FeedParser.ParseValue(highLow[0]); + price.High=FeedParser.ParseValue(highLow[1]); + } + if(!price.IsValid)return null; + return price; } catch (Exception exception) { @@ -5239,9 +5037,9 @@ namespace MarketData.Helper finally { if(null!=httpNetResponse)httpNetResponse.Dispose(); - if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();} } } + // ******************************************************************************************************************************************************************************* // ******************************************************************************************************************************************************************************* // ******************************************************************************************************************************************************************************* @@ -5477,26 +5275,10 @@ namespace MarketData.Helper // **************************************************************************************************************************************************************************************** // **************************************************************************** E N D D A T A F E E D S ********************************************************************************* // **************************************************************************************************************************************************************************************** -// **************************************************************************************************************************************************************************************** -// **************************************************************************** H E L P E R M E T H O D S ******************************************************************************* -// **************************************************************************************************************************************************************************************** - private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName) - { - StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt"); - for (int row = 0; row < rows.Count; row++) - { - streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml); - } - streamWriter.Flush(); - streamWriter.Close(); - } - private static void WriteToDisk(String responseString, String strPathFileName) - { - StreamWriter streamWriter = File.CreateText(strPathFileName); - streamWriter.WriteLine(responseString); - streamWriter.Flush(); - streamWriter.Close(); - } + + + + // *************************************************************************************************************************************************************************************** // *************************************************************************************************************************************************************************************** // *************************************************************************************************************************************************************************************** @@ -5635,5 +5417,39 @@ namespace MarketData.Helper if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch; return referenceDate; } + +// **************************************************************************************************************************************************************************************** +// **************************************************************************** H E L P E R M E T H O D S ******************************************************************************* +// **************************************************************************************************************************************************************************************** + private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName) + { + StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt"); + for (int row = 0; row < rows.Count; row++) + { + streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml); + } + streamWriter.Flush(); + streamWriter.Close(); + } + + private static void WriteToDisk(String responseString, String strPathFileName) + { + StreamWriter streamWriter = File.CreateText(strPathFileName); + streamWriter.WriteLine(responseString); + streamWriter.Flush(); + streamWriter.Close(); + } + + private static void WriteToDisk(List sections, String strPathFileName) + { + StreamWriter streamWriter = File.CreateText(strPathFileName); + for(int index=0;index + /// Get the proxy for the specified service or null if it is not congfigured to use the proxy + /// + ///The service name public static WebProxy GetProxy(String serviceName) { try @@ -1224,6 +1243,32 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetProxy: Exception:{0}",exception.ToString())); return null; } + } + + /// + /// Get the proxy. This is only to be used as part of unit tests to test out the TOR Proxy. User code should use the GetProxy(serviceName) variation + /// + public static WebProxy GetProxy() + { + String proxyAddress=ConfigurationManager.AppSettings["proxy_address"]; + String[] addressParts=proxyAddress.Split(':'); + SocketControl socketControl=null; + try + { + socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2])); + if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY. {0} {1} ",proxyAddress,exception.ToString())); + return null; + } + socketControl.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} ",proxyAddress)); + Uri newUri=new Uri(proxyAddress); + WebProxy proxy=new WebProxy(); + proxy.Address=newUri; + return proxy; } } } diff --git a/MarketDataLib/MarketDataModel/Prices.cs b/MarketDataLib/MarketDataModel/Prices.cs index 1a16ead..9e6667f 100644 --- a/MarketDataLib/MarketDataModel/Prices.cs +++ b/MarketDataLib/MarketDataModel/Prices.cs @@ -315,7 +315,7 @@ namespace MarketData.MarketDataModel // *************************************************************************************************************************************************************************** public class Price { - public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3}; + public enum PriceSource{Other=0,BigCharts=1,Yahoo=2,Fidelity=3,Google=4}; private String symbol; private DateTime date; private double open; diff --git a/MarketDataLib/Utility/Sections.cs b/MarketDataLib/Utility/Sections.cs index 3d50c66..8555ade 100644 --- a/MarketDataLib/Utility/Sections.cs +++ b/MarketDataLib/Utility/Sections.cs @@ -81,6 +81,19 @@ namespace MarketDataLib.Utility return null; } } + + public static List FlattenSection(List section) + { + List flattened = new List(); + + foreach(String item in section) + { + if("".Equals(item))continue; + flattened.Add(item); + } + return flattened; + } + public static bool FindInSections(List sections,String startsWith,int startingIndex,ref int indexOfItem,bool findExact=true) { for(int index=startingIndex;index