using System;                                                                                                                 
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Data;
using System.Threading;
using System.Net;
using System.Reflection;
using System.Xml.Linq;
using Newtonsoft.Json.Linq;
using HtmlAgilityPack;
using MarketData.MarketDataModel;
using MarketData.Utils;                                                                                                         
using MarketData.CSVHelper;
using MarketData.DataAccess;
using MarketData.Integration;
using System.Globalization;
using MarketDataLib.Utility;

//Zacks Rank - Zacks
//Splits - EODDATA
//GDP Data - WorldBank
//Dividend History - NASDAQ
//Options - NASDAQ
//Analyst Ratings - Briefing.com
//SEC Filings - SEC.GOV (EDGAR)
//Yield Curve - Treasury.Gov
//ETF Holdings - Yahoo Finance
//Insider Transactions - Insider Tracking.com
//Company Profile - MorningStar + Reuters
//Headlines - Seeking Alpha + NASDAQ
//CIK Codes - SEC.GOV
//Analyst Price Target - Yahoo Finance , Market Beat as backup
//Historical - MorningStar
//Income Statement - NASDAQ
//Balance Sheet - NASDAQ
//Statement of Cashflows - MorningStar
//Fundamentals - 2 Sources....Yahoo Finance & FINVIZ
//  a) Key Statistics
//  b) Financials
//Intraday Pricing - Yahoo Finance + BigCharts
//Historical Pricing - BigCharts + Yahoo Finance
//CompanyDescription - Reuters
//Premarket  - CNN

namespace MarketData.Helper
{
  public class MarketDataHelper
  {
    public static int DEFAULT_TIMEOUT_MS=30000;
    public static String SEC_BASE_URL="https://www.sec.gov";


    public static bool GetWorldTime()
    {
      HttpNetResponse httpNetResponse=null;

      try
      {
        StringBuilder sb = new StringBuilder();
        sb.Append("https://www.timeanddate.com/worldclock/");
        String strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime:{0}",strRequest));
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4A(strRequest, null, webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return false;
        }

        if(String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest));
          return false;
        }

        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length));


        return true;

      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return false;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }



// ******************************************************************************************************************************************************************************
// ************************************************************************************ P R E M A R K E T  D A T A    ***********************************************************
// ******************************************************************************************************************************************************************************
  public static PremarketElements GetPremarketData()
  {
    HttpNetResponse httpNetResponse=null;
    MemoryStream memoryStream=null;
    PremarketElements premarketElements=new PremarketElements();
    DateTime timestamp=DateTime.Now;

    try
    {
      StringBuilder sb=new StringBuilder();
      String strRequest=null;

      sb.Append("https://production.dataviz.cnn.io/markets/futures/summary");
      strRequest=sb.ToString();
      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
      httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
      if(!httpNetResponse.Success)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'.  Failed with {1}",strRequest,httpNetResponse.ErrorMessage));
        return null;
      }

      List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
      if(null==items || 0==items.Count)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'.  No Data",strRequest));
        return premarketElements;
      }
      for(int index=0;index<items.Count;index++)
      {
        List<String> pair=items[index];
        if(pair[1].Equals("S&P 500"))
        {
          PremarketElement premarketElement=new PremarketElement();
          premarketElement.Market="S&P";
          premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
          premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
          premarketElement.Timestamp=timestamp;
          premarketElements.Add(premarketElement);
        }
        else if(pair[1].Equals("NASDAQ"))
        {
          PremarketElement premarketElement=new PremarketElement();
          premarketElement.Market="NASDAQ";
          premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
          premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
          premarketElement.Timestamp=timestamp;
          premarketElements.Add(premarketElement);
        }
        else if(pair[1].Equals("DOW"))
        {
          PremarketElement premarketElement=new PremarketElement();
          premarketElement.Market="DOW";
          premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
          premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
          premarketElement.Timestamp=timestamp;
          premarketElements.Add(premarketElement);
        }
      }
      return premarketElements;
    }
    catch(Exception exception)
    {
      MDTrace.WriteLine(LogLevel.DEBUG,exception);
      return null;
    }
    finally
    {
      if(null!=memoryStream) memoryStream.Close();
      if(null!=httpNetResponse) httpNetResponse.Dispose();
    }
  }
// ******************************************************************************************************************************************************************************
// ************************************************************************** Z A C K S  E A R N I N G S  A N N O U N C E M E N T S   *******************************************
// ******************************************************************************************************************************************************************************
  public static EarningsAnnouncements GetEarningsAnnouncements(String symbol)
  {
      HttpNetResponse httpNetResponse=null;
      EarningsAnnouncements earningsAnnouncements=new EarningsAnnouncements();
			MemoryStream memoryStream=null;
      try
      {
        StringBuilder sb=new StringBuilder();
        sb.Append("https://www.zacks.com/stock/research/").Append(symbol).Append("/earnings-announcements");
        String strRequest=sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        String responseString=httpNetResponse.ResponseString;

// get next earnings date
				byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"key-expected-earnings-data-module\"]");
				if (null != table && 0 != table.Count)
				{
					HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
					String[] requiredElements={"Report Date","Period Ending","Zacks Consensus Estimate","Earnings ESP"};
					if(2==rows.Count)
					{
						HtmlNodeCollection headerColumns = rows[0].SelectNodes(".//th");
						if (null!=headerColumns && 0!=headerColumns.Count && HeaderContains(headerColumns,requiredElements))
						{
							HtmlNodeCollection dataColumns = rows[1].ChildNodes;
							String[] reportElements=dataColumns[1].InnerText.Split(' ');
							String reportDate=reportElements[0];
							if(!"NA".Equals(reportDate))
							{
							  String[] formats=new[] {"MM/yyyy","M/yyyy"};
								String periodEnding=dataColumns[3].InnerText;
								EarningsAnnouncement earningsAnnouncement=new EarningsAnnouncement();
								earningsAnnouncement.Symbol=symbol;
								earningsAnnouncement.Date=Utility.ParseDate(reportDate);
								earningsAnnouncement.PeriodEnding=DateTime.ParseExact(periodEnding, formats, new System.Globalization.CultureInfo("en-US"),DateTimeStyles.AssumeLocal);
								if(reportElements[1].Equals("(AMC)"))earningsAnnouncement.Time="After Close";
								else if(reportElements[1].Equals("(NONE)"))earningsAnnouncement.Time="Before Open";
								else earningsAnnouncement.Time=reportElements[1];
								earningsAnnouncements.Add(earningsAnnouncement);
							}
						}
					}
				}

// get historical
        responseString=Utility.KeepAfter(responseString,"earnings_announcements_earnings_table");
        if(null==responseString)return null;
        responseString=Utility.KeepBefore(responseString,"]  ]");
        if(null==responseString)return null;
        responseString=responseString+"]";
        responseString=Utility.KeepAfter(responseString,"[  [");
        if(null==responseString)return null;
        responseString="["+responseString;                 
        responseString=Utility.RemoveDivs(responseString);
        if(null==responseString)return null;
        String[] items=responseString.Split(']');
        for(int index=0;index<items.Length;index++)
        {
          EarningsAnnouncement earningsAnnouncement=new EarningsAnnouncement();
          String itemText=items[index];
          if(null==itemText||0==itemText.Length)continue;
          itemText=Utility.BetweenString(itemText,"[",null);
          itemText=itemText.Replace("\"",null);
          String[] columnData=itemText.Split(',');
          for(int columnIndex=0;columnIndex<columnData.Length;columnIndex++)columnData[columnIndex]=columnData[columnIndex].Trim();
          if(null==columnData||7!=columnData.Length)continue;
          earningsAnnouncement.Symbol=symbol;
          earningsAnnouncement.Date=Utility.ParseDate(columnData[0]);

          String[] periodEndingItems=columnData[1].Split('/');
          if(null!=periodEndingItems&&2==periodEndingItems.Length)
          {
            int month=(int)FeedParser.ParseValueLong(periodEndingItems[0]);
            int year=(int)FeedParser.ParseValueLong(periodEndingItems[1]);
            earningsAnnouncement.PeriodEnding=new DateTime(year,month,DateTime.DaysInMonth(year,month));
          }
          earningsAnnouncement.Estimate=FeedParser.ParseValue(columnData[2]);
          earningsAnnouncement.Reported=FeedParser.ParseValue(columnData[3]);
          earningsAnnouncement.Surprise=FeedParser.ParseValue(columnData[4]);
          earningsAnnouncement.SurprisePct=FeedParser.ParseValue(columnData[5]);
          earningsAnnouncement.Time=columnData[6];
          earningsAnnouncements.Add(earningsAnnouncement);
        }
        return earningsAnnouncements;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
				if(null!=memoryStream)memoryStream.Dispose();
      }
  }
// ******************************************************************************************************************************************************************************
// ************************************************************************** Z A C K S  R A N K   ******************************************************************************
// ******************************************************************************************************************************************************************************
    public static ZacksRank GetZacksRank(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      try
      {
        StringBuilder sb=new StringBuilder();
        sb.Append("https://www.zacks.com/stock/quote/");
        sb.Append(symbol);
        String strRequest=sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          if(httpNetResponse.StatusCode.Equals("Forbidden"))
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
            try{Thread.Sleep(15000);}catch(Exception){;}
            httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
            if(!httpNetResponse.Success)
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
              return null;
            }
          }
          else return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"rank_view\"]");
        if(null==table||0==table.Count)return null;
        String rank=Utility.BetweenString(table[0].InnerHtml,null,"<").Trim();
        if(null==rank||"".Equals(rank))return null;
        ZacksRank zacksRank=new ZacksRank();
        zacksRank.Symbol=symbol;
        zacksRank.Date=DateTime.Now.Date;
        zacksRank.Rank=rank;
        return zacksRank;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ************************************************************************** S P L I T S  F R O M  E O D D A T A   **********************************************************************
// ***************************************************************************************************************************************************************************************
    public static Splits GetSplits()
    {
      HttpNetResponse httpNetResponse=null;
			try
			{
        Splits splits=new Splits();
				StringBuilder sb=new StringBuilder();
				String strRequest;
				sb.Append("http://eoddata.com/splits.aspx");
				strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Getting splits {0}",strRequest));
        httpNetResponse=HttpNetRequest.GetRequestStreamZIPV2(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        HtmlDocument htmlDocument=new HtmlDocument();
        MemoryStream memoryStream=new MemoryStream(httpNetResponse.ResponseStream.GetBuffer());
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quotes\"]");
        if (null == divSection || 0 == divSection.Count) return null;
        HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
        for (int index = 0; index < rows.Count; index++)
        {
          Split split=new Split();
          HtmlNodeCollection headerColumns = rows[index].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td");
          if(dataColumns.Count<4)continue;
          if(null!=dataColumns[0])split.Exchange=dataColumns[0].InnerText.Trim().ToUpper();            // exchange
          if(null!=dataColumns[1])split.Symbol=dataColumns[1].InnerText.Trim().ToUpper();
          if(null!=dataColumns[2])split.EffectiveDate=Utility.ParseDate(dataColumns[2].InnerText.Trim().ToUpper());
          if(null!=dataColumns[3])split.StrRatio=dataColumns[3].InnerText.Trim().ToUpper();
          splits.Add(split);
        }
        memoryStream.Close();
        memoryStream.Dispose();
        return splits;
			}
			catch(Exception exception)
			{
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
			}
			finally
			{
        if(null!=httpNetResponse)httpNetResponse.Dispose();
			}
    }
// ***************************************************************************************************************************************************************************************
// ************************************************************************** G D P  P E R  C A P I T A   D A T A  W O R L D  B A N K    *************************************************
// ***************************************************************************************************************************************************************************************
		public static EconomicIndicators GetGDPPerCapita(bool debug=false)
		{
      HttpNetResponse httpNetResponse=null;
			try
			{
// Retrieve compressed CSV from World Bank web site and write to disk
				StringBuilder sb=new StringBuilder();
				String strRequest;
				String currentWorkingDirectory=Directory.GetCurrentDirectory();
				String strExtractFolder=currentWorkingDirectory+"\\"+"extracts";
				String strFileName="API_NY.GDP.MKTP.CD_DS2_V2_USD.zip";
				String strPathFileName=currentWorkingDirectory+"\\"+strFileName;
				sb.Append("http://api.worldbank.org/v2/en/indicator/NY.GDP.MKTP.CD?downloadformat=csv");
				strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Downloading {0} from {1}",strFileName,strRequest));
        httpNetResponse=HttpNetRequest.GetRequestStreamZIP(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
				if(File.Exists(strPathFileName))File.Delete(strPathFileName);
				FileStream outStream=new FileStream(strPathFileName,FileMode.Create);
				byte[] streamBytes=httpNetResponse.ResponseStream.GetBuffer();
				outStream.Write(streamBytes,0,streamBytes.Length);
				outStream.Flush();
				outStream.Close();
				return EconomicIndicators.FromZipFile(strPathFileName,strExtractFolder,debug);
			}
			catch(Exception exception)
			{
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
			}
			finally
			{
        if(null!=httpNetResponse)httpNetResponse.Dispose();
			}
		}
// ***************************************************************************************************************************************************************************************
// ************************************************************************** D I V I D E N D   H I S T O R Y   N A S D A Q    ***********************************************************
// ***************************************************************************************************************************************************************************************
    public static DividendHistory GetDividendHistory(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      DividendHistory dividendHistory = new DividendHistory();
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        CookieCollection cookieCollection = new CookieCollection();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetDividendHistory");
				CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
        if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf");
				else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks"); 
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest));
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
//        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy);
//        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy);
//        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          if(httpNetResponse.StatusCode.Equals(System.Net.HttpStatusCode.Forbidden))
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
            try{Thread.Sleep(15000);}catch(Exception){;}
            httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest, cookieCollection, webProxy);
            if(!httpNetResponse.Success)
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
              return null;
            }
          }
          else return null;
        }
        
			  int groupBy=6;

				List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
				if(null==items || 0==items.Count || items.Count<groupBy)return dividendHistory;
				for(int index=0;index<items.Count;index+=groupBy)
				{
				  if(items.Count<index+groupBy)break;
          String strExEffDate = items[index][1];
          String strType = items[index+1][1];
          String strCashAmount = items[index+2][1];
          String strDeclarationDate = items[index+3][1];
          String strRecordDate = items[index+4][1];
          String strPaymentDate = items[index+5][1];

					try
					{
						if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
            if("EX-DIVIDEND DATE".Equals(strExEffDate, StringComparison.InvariantCultureIgnoreCase))
            {
              strExEffDate = items[index+1][1];
              strType = items[index+4][1];
              strCashAmount = items[index+5][1];
              strDeclarationDate = items[index+1][1];
              strRecordDate = items[index+1][1];
              strPaymentDate = items[index+1][1];
            }
						if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
						DividendHistoryItem dividendHistoryItem = new DividendHistoryItem();
            dividendHistoryItem.Modified=DateTime.Now;
						dividendHistoryItem.Symbol = symbol.ToUpper();
						try{dividendHistoryItem.DivExDate = Utility.ParseDate(strExEffDate);}
            catch(Exception exception){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception parsing value '{0}', {1}",strExEffDate,exception.ToString()));continue;}
						dividendHistoryItem.DividendType = strType;
						if (!"--".Equals(strCashAmount)&&!"".Equals(strCashAmount)) dividendHistoryItem.CashAmount = FeedParser.ParseValue(strCashAmount);
						else dividendHistoryItem.CashAmount = null;
						if (!"--".Equals(strDeclarationDate)&&!"N/A".Equals(strDeclarationDate)) dividendHistoryItem.DeclarationDate = DateTime.Parse(strDeclarationDate);
						else dividendHistoryItem.DeclarationDate=null;
						if (!"--".Equals(strRecordDate)&&!"N/A".Equals(strRecordDate)) dividendHistoryItem.RecordDate = DateTime.Parse(strRecordDate);
						if (!"--".Equals(strPaymentDate)&&!"N/A".Equals(strPaymentDate)) dividendHistoryItem.PaymentDate = DateTime.Parse(strPaymentDate);
						else dividendHistoryItem.PaymentDate = null;

            if(null==dividendHistoryItem.CashAmount || double.IsNaN(dividendHistoryItem.CashAmount.Value))
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Encountered invalid CashAmount in DividendHistory record for {0} ",symbol));
              continue;
            }
            if(dividendHistoryItem.DividendType.Equals("ANNUAL DIVIDEND"))dividendHistoryItem.DividendType="Cash";
						dividendHistory.Add(dividendHistoryItem);
					}
					catch(Exception exception)
					{
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Exception:{0}",exception.ToString()));
					  continue;
					}
				}
				return dividendHistory;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return dividendHistory;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ************************************************************************** O P T I O N S   N A S D A Q    *****************************************************************************
// ***************************************************************************************************************************************************************************************
// month=0=current month
// month=1=current month
// month=2=current month
// month=3=current month
// month=4=current month
// month=5=current month
//  http://www.nasdaq.com/symbol/msft/option-chain?money=all&dateindex=-1
    public static Options GetOptions(String symbol)
    {
      Options options = new Options();
      Dictionary<int,Option> optionsDictionary=new Dictionary<int,Option>();

      for (int dateIndex = 0; dateIndex < 5 ; dateIndex++)
      {
        bool morePages=true;
        for(int pageIndex=1;pageIndex<10 && morePages;pageIndex++)
        {
          Options datedOptions = GetOptions(symbol, dateIndex,pageIndex);
          if (null == datedOptions || 0 == datedOptions.Count) continue;
          for(int optionIndex=0;optionIndex<datedOptions.Count && morePages;optionIndex++)
          {
            Option option=datedOptions[optionIndex];
            if(!optionsDictionary.ContainsKey(option.GetHashCode()))
            {
              optionsDictionary.Add(option.GetHashCode(),option);
              options.Add(option);
            }
            else 
            {
              morePages=false;
              continue;
            }
          }
        }
      }
      return options;
    }

    private static Options GetOptions(String symbol, int dateIndex,int page)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse=null;

      try
      {
        Options options = new Options();
        StringBuilder sb = new StringBuilder();
        String strRequest;
//        sb.Append("http://www.nasdaq.com/symbol/").Append(symbol).Append("/option-chain?money=all&dateindex=").Append(dateIndex).Append("&page=").Append(page);
        sb.Append("http://old.nasdaq.com/symbol/").Append(symbol).Append("/option-chain?money=all&dateindex=").Append(dateIndex).Append("&page=").Append(page);
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
//        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV2(strRequest);
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"OptionsChain-chart borderAll thin\"]");
        if (null == divSection || 0 == divSection.Count) return null;
        HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
        if (null == rows) return null;
        for (int index = 0; index < rows.Count; index++)
        {
          HtmlNodeCollection headerColumns = rows[index].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td");
          Option callOption = new Option();
          Option putOption = new Option();
          callOption.Expiration=FeedParser.ParseValueDateTimeMonthFormat(dataColumns[0].InnerText);
          callOption.Last = FeedParser.ParseValue(dataColumns[1].InnerText);
          callOption.Change = FeedParser.ParseValue(dataColumns[2].InnerText);
          callOption.Bid = FeedParser.ParseValue(dataColumns[3].InnerText);
          callOption.Ask = FeedParser.ParseValue(dataColumns[4].InnerText);
          callOption.Volume = FeedParser.ParseValueLong(dataColumns[5].InnerText);
          callOption.OpenInterest = FeedParser.ParseValue(dataColumns[6].InnerText);
          callOption.Symbol=dataColumns[7].InnerText.Trim().ToUpper();
          callOption.Strike = FeedParser.ParseValue(dataColumns[8].InnerText);
          callOption.Type = OptionTypeEnum.CallOption;
          if(!double.IsNaN(callOption.Bid)&&0!=callOption.Strike)options.Add(callOption);
          putOption.Symbol = dataColumns[7].InnerText.Trim().ToUpper();
          putOption.Strike = FeedParser.ParseValue(dataColumns[8].InnerText);
          putOption.Expiration = FeedParser.ParseValueDateTimeMonthFormat(dataColumns[9].InnerText);
          putOption.Last = FeedParser.ParseValue(dataColumns[10].InnerText);
          putOption.Change = FeedParser.ParseValue(dataColumns[11].InnerText);
          putOption.Bid = FeedParser.ParseValue(dataColumns[12].InnerText);
          putOption.Ask = FeedParser.ParseValue(dataColumns[13].InnerText);
          putOption.Volume = FeedParser.ParseValueLong(dataColumns[14].InnerText);
          putOption.OpenInterest = FeedParser.ParseValue(dataColumns[15].InnerText);
          putOption.Type = OptionTypeEnum.PutOption;
          if (!double.IsNaN(putOption.Bid)&&0!=putOption.Strike) options.Add(putOption);
        }
        return options;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ********************************************************************** A N A L Y S T   R A T I N G S  -  B R I E F I N G  .  C O M    *************************************************
// ***************************************************************************************************************************************************************************************
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Upgrades/2014/08/04
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/downgrades/2014/08/04
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Reiterated/2014/08/04
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Initiated/2014/08/04
// The feed no longer accepts a date parameter
    public static AnalystRatings GetLatestAnalystRatings()
    {
      AnalystRatings analystRatings = GetLatestAnalystRatingsBriefing();
      return analystRatings;
    }
    public static AnalystRatings GetLatestAnalystRatingsBriefing()
    {
      Dictionary<String, String> uniqueSymbols = new Dictionary<String, String>();
      AnalystRatings analystRatings = new AnalystRatings();
      AnalystRatings ratings = null;

      ratings = GetLatestAnalystRatingsBriefing("Upgrades");
      if (null != ratings)
      {
        foreach (AnalystRating analystRating in ratings)
        {
          if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
          {
            analystRatings.Add(analystRating);
            uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
          }
        }
      }
      ratings = GetLatestAnalystRatingsBriefing("Downgrades");
      if (null != ratings)
      {
        foreach (AnalystRating analystRating in ratings)
        {
          if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
          {
            analystRatings.Add(analystRating);
            uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
          }
        }
      }
      ratings = GetLatestAnalystRatingsBriefing("Reiterated");
      if (null != ratings)
      {
        foreach (AnalystRating analystRating in ratings)
        {
          if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
          {
            analystRatings.Add(analystRating);
            uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
          }
        }
      }
      ratings = GetLatestAnalystRatingsBriefing("Initiated");
      if (null != ratings)
      {
        foreach (AnalystRating analystRating in ratings)
        {
          if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
          {
            analystRatings.Add(analystRating);
            uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
          }
        }
      }
      return analystRatings;
    }
    private static AnalystRatings GetLatestAnalystRatingsBriefing(String type)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse = null;
      AnalystRatings analystRatings = new AnalystRatings();
      try
      {
        String strRequest = null;
        if ("Upgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/upgrade.htm";
        else if ("Downgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/downgrade.htm";
        else if ("Reiterated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/reiterated.htm";
        else if ("Initiated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/initiated.htm";
        else return null;

        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if (!httpNetResponse.Success || null == httpNetResponse.ResponseString)
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection divSectionDate = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calDATE\"]");
        HtmlNodeCollection divSectionHeader = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calHDR2\"]");
        HtmlNodeCollection divSections = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calEVENT\"]");

        if (null == divSectionDate || 0 == divSectionDate.Count) return null;
        if (null == divSectionHeader || 0 == divSectionHeader.Count) return null;
        if (null == divSections || 0 == divSections.Count) return null;

        DateTime ratingsDate = Utility.ParseDate(divSectionDate[0].InnerText);
        String ratingsType = divSectionHeader[0].InnerText;
        if (!ratingsType.StartsWith(type.ToUpper())) return null;

        //  calDATA
        for (int index = 0; index < divSections.Count; index++)
        {

          List<String> items = Sections.GetSections(divSections[index].InnerHtml);
          if (null == items || 0 == items.Count) continue;

          AnalystRating analystRating = new AnalystRating();
          analystRating.CompanyName = items.Count > 0 ? items[0] : null;
          analystRating.Date = ratingsDate;
          analystRating.Symbol = items.Count > 4 ? items[4] : null;
          analystRating.Type = type;
          if (items.Count > 10 && items[10].StartsWith("Brokerage Firm")) analystRating.BrokerageFirm = items.Count > 12 ? items[12] : null;
          if (items.Count > 22 && items[22].StartsWith("Ratings Change")) analystRating.RatingsChange = items.Count > 24 ? items[24] : null;
          if (null != analystRating.RatingsChange & analystRating.RatingsChange.Contains("&#187;")) analystRating.RatingsChange = analystRating.RatingsChange.Replace("&#187;", "->");
          if (items.Count > 28 && items[28].StartsWith("Price Tgt"))
          {
            String priceTarget = items.Count > 31 ? items[31] : null;
            if (null != priceTarget && priceTarget.Contains("&#187;")) priceTarget = priceTarget.Replace("&#187;", "-");
            if (priceTarget.Contains("-"))
            {
              String[] prices = priceTarget.Split('-');
              priceTarget = prices[1];
            }
            analystRating.PriceTarget = FeedParser.ParseValue(priceTarget);
          }
          analystRatings.Add(analystRating);
        }
        return analystRatings;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG, exception);
        return analystRatings;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if (null != httpNetResponse) httpNetResponse.Dispose();
      }
    }
// ******************************************************************************************************************************************************************************************
// ****************************************************************************** A N A L Y S T   R A T I N G S  M A R K E T  B E A T *******************************************************
// ******************************************************************************************************************************************************************************************
    public static AnalystRatings GetAnalystRatingsMarketBeat(String symbol)
    {
      AnalystRatings analystRatings=new AnalystRatings();
      String nasdaq="NASDAQ";
      String nyse="NYSE";

      try
      {
        StringBuilder sb=new StringBuilder();
        CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
        HtmlNodeCollection ratingsTable=null;
        ratingsTable=GetRatingsTableOnMarketBeat(symbol,nyse);
        if(null==ratingsTable) ratingsTable=GetRatingsTableOnMarketBeat(symbol,nasdaq);
        if(null==ratingsTable)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No ratings for {0} on MarketBeat",symbol));
          return null;
        }
        HtmlNodeCollection header=ratingsTable[0].SelectNodes(".//th");
				string[] requiredElements=new string[]{"Date","Brokerage","Action","Rating","Price Target","Details"};
				if(!HeaderContains(header,requiredElements))return null;
        HtmlNodeCollection data=ratingsTable[0].SelectNodes(".//tr//td");
				if(null==data)return analystRatings;
        for(int index=0,rowIndex=0;index<data.Count&&index+(header.Count-1)<data.Count;index+=header.Count,rowIndex++)
        {
				  String strDate=GetElementData(header,data,rowIndex,"Date");
          if(null==strDate || "".Equals(strDate))break;
					String strBrokerage=GetElementData(header,data,rowIndex,"Brokerage");
					if(null!=strBrokerage)strBrokerage=strBrokerage.Trim();
          String strAction=GetElementData(header,data,rowIndex,"Action");
          String strRating=GetElementData(header,data,rowIndex,"Rating");     
          String strPriceTarget=GetElementData(header,data,rowIndex,"Price Target");
          AnalystRating analystRating=new AnalystRating();
          analystRating.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;
          analystRating.Date=Utility.ParseDate(strDate);
          analystRating.Symbol=symbol;
          analystRating.Type=strAction;
          analystRating.BrokerageFirm=strBrokerage;
          if(null!=strRating&strRating.Contains("&#x279D;")) strRating=strRating.Replace("&#x279D;","->");
          analystRating.RatingsChange=strRating;
          if(null!=strPriceTarget)
          {

            if(strPriceTarget.Contains("&#x279D;")) strPriceTarget=strPriceTarget.Replace("&#x279D;","-");
            if(strPriceTarget.Contains("-"))
            {
              String[] prices=strPriceTarget.Split('-');
              strPriceTarget=prices[1];
            }
            analystRating.PriceTarget=FeedParser.ParseValue(strPriceTarget);
          }
          if(analystRating.Type.Equals("Boost Price Target")) analystRating.Type="Upgrades";
          if(analystRating.Type.Equals("Upgrade")) analystRating.Type="Upgrades";
          if(analystRating.Type.Equals("Downgrade")) analystRating.Type="Downgrades";
          if(analystRating.Type.StartsWith("Reiterated")) analystRating.Type="Reiterated";
          if(analystRating.Type.StartsWith("Initiated")) analystRating.Type="Initiated";
					if(analystRating.Type.StartsWith("Set Price Target"))analystRating.Type="Initiated";
          if(!analystRating.Type.Equals("Upgrades")&&!analystRating.Type.Equals("Downgrades")&&!analystRating.Type.Equals("Reiterated")&&!analystRating.Type.Equals("Initiated")) continue;
          analystRatings.Add(analystRating);
        }
        return new AnalystRatings(analystRatings.Distinct().ToList());
//        return analystRatings;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return analystRatings;
      }
      finally
      {
      }
    }
		public static bool HeaderContains(HtmlNodeCollection header,String[] requiredElements)
		{
		  List<String> headerElements=new List<String>();
		  if(null==header)return false;
			for(int index=0;index<header.Count;index++)
			{
			  headerElements.Add(header[index].InnerText);
			}
      List<String> exceptions=requiredElements.Except(headerElements).ToList();
			return exceptions.Count()>0?false:true;
		}
		public static int HeaderIndexOf(HtmlNodeCollection header,String identifier)
		{
		  if(null==header)return -1;
			for(int index=0;index<header.Count;index++)
			{
			  if(header[index].InnerText.Equals(identifier))return index;
			}
			return -1;
		}
		public static String GetElementData(HtmlNodeCollection header,HtmlNodeCollection data,int row,String identifier)
		{
		  int index=HeaderIndexOf(header,identifier);
			if(-1==index)return null;
      int indexer=(header.Count*row)+index;
			List<String> sections=Sections.GetSections(data[indexer].InnerHtml);
			if(null==sections || 0==sections.Count)return data[indexer].InnerText;
			return sections[0];
		}
    private static HtmlNodeCollection GetRatingsTableOnMarketBeat(String symbol,String exchange)
    {
      MemoryStream memoryStream=null;
      HttpNetResponse httpNetResponse=null;

      try
      {
        String strRequest=null;
        StringBuilder sb=new StringBuilder();
        sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3(strRequest, "www.marketbeat.com");
        if(!httpNetResponse.Success||String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream=new MemoryStream(streamBytes);
        HtmlDocument htmlDocument=new HtmlDocument();
        htmlDocument.Load(memoryStream);

        HtmlNodeCollection ratingsTable=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"scroll-table sort-table\"]");
        if(null==ratingsTable||0==ratingsTable.Count) return null;
        return ratingsTable;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=memoryStream) memoryStream.Close();
        if(null!=httpNetResponse) httpNetResponse.Dispose();
      }
    }

// ***************************************************************************************************************************************************************************
//***************************************************************** C I K  C O D E S   - S E C . G O V ***********************************************************************
// ***************************************************************************************************************************************************************************
    public static String GetCIK(String symbol)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany");
        strRequest = sb.ToString();
//        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"companyInfo\"]");  //  yfnc_tablehead1,yfnc_modtitle1,yfnc_tabledata1
        if (null == rows || 1!=rows.Count)
        {
          if (null == rows) MDTrace.WriteLine(LogLevel.DEBUG,"[GetCIK] Received no rows for '" + symbol + "'");
          else MDTrace.WriteLine(LogLevel.DEBUG,"Expected 1 row, got " + rows.Count + "  for '" + symbol + "'");
          return null;
        }
        HtmlNode htmlNodeTitle = rows[0];
        HtmlNodeCollection nodes = htmlNodeTitle.SelectNodes(".//a");
        if (null == nodes || 0==nodes.Count)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,"Expected <a> tag for symbol '" + symbol + "'");
          return null;
        }
        String cik = null;
        String[] strings = nodes[0].InnerText.Split(' ');
        if (strings.Length >= 1) cik = strings[0];
        return cik;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ************************************ S E C   F I L I N G S  -  F O R M  4  &  F O R M 5  - F O R  I N S I D E R  T R A N S A C T I O N S  S E C . G O V     ***************************S
// ***************************************************************************************************************************************************************************************
    public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=5)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse=null;
      String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
      DateTime minFilingDate=DateTime.Now;
      int maxFilings=80;
      bool continuationFlag=true;
      int TIMEOUT_MS_BETWEEN_REQUESTS=1000;

      try
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} dayCount:{1}",symbol,timePeriodDays));
        DateGenerator dateGenerator=new DateGenerator();
        String cik=PricingDA.GetCIKForSymbol(symbol);
        if(null==cik)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: No CIK for symbol {0}",symbol));
          return null;
        }
        minFilingDate=dateGenerator.DaysAddActual(minFilingDate,Math.Abs(timePeriodDays)*-1);
        SECFilings secFilings = new SECFilings();
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
        if (null == tables || tables.Count < 1) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count));
        for (int row = 0; row < rows.Count && continuationFlag; row ++)  
        {
          try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
          HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
          if (null == dataColumns || 5 != dataColumns.Count) continue;
          HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
          if (null == nodes || 0 == nodes.Count) continue;
          String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
          SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
          for (int index = 0; null!=innerCollection &&  index < innerCollection.Count; index++)
          {
            SECFiling innerFiling = innerCollection[index];
            if(innerFiling.FilingDate<minFilingDate)
            {
              continuationFlag=false;
              break;
            }
            if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
            secFilings.Add(innerFiling);
          }
        }
        return InsiderTransactionsParser.GetInstance().Parse(secFilings);
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
    /// <summary>
    /// Get InsiderTransactions for years greater than or equal to specified year.
    /// For example.  If 2023 is specified then get all for years>=2023
    /// </summary>
    /// <param name="symbol">The symbol.</param>
    /// <param name="yearGreaterThanEqualTo">the new y-coordinate.</param>
    public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse=null;
      String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
      int maxFilings=120;
      bool continuationFlag=true;
      int TIMEOUT_MS_BETWEEN_REQUESTS=1000;

      try
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
        DateGenerator dateGenerator=new DateGenerator();
        String cik=PricingDA.GetCIKForSymbol(symbol);
        if(null==cik)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
          return null;
        }
        SECFilings secFilings = new SECFilings();
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
        if (null == tables || tables.Count < 1) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}.  Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
        for (int row = 0; row < rows.Count && continuationFlag; row ++)  
        {
          try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
          HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
          if (null == dataColumns || 5 != dataColumns.Count) continue;
          HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
          if (null == nodes || 0 == nodes.Count) continue;
          String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
          SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
          for (int index = 0; null!=innerCollection &&  index < innerCollection.Count; index++)
          {
            SECFiling innerFiling = innerCollection[index];
            if(innerFiling.FilingDate.Year<yearGreaterThanEqualTo)
            {
              continuationFlag=false;
              break;
            }
            if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
            secFilings.Add(innerFiling);
          }
        }
        return InsiderTransactionsParser.GetInstance().Parse(secFilings);
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ********************************************************************** S E C   F I L I N G S  -  S E C . G O V     ********************************************************************
// ***************************************************************************************************************************************************************************************
    public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
    {
      try
      {
        String cik=PricingDA.GetCIKForSymbol(symbol);
        if(null==cik)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
          return null;
        }
        return GetSECFilings(symbol,cik,maxFilings);
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;      }
    }

    public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
    {
      MemoryStream memoryStream = null;
      HttpNetResponse httpNetResponse=null;
      int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;

      try
      {
        SECFilings secFilings = new SECFilings();
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
        if (null == tables || tables.Count < 1) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
        for (int row = 0; row < rows.Count; row ++)  
        {
          HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
          if (null == dataColumns || 5 != dataColumns.Count) continue;
          HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
          if (null == nodes || 0 == nodes.Count) continue;
          String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
          try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;} 
          SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
          for (int index = 0; null!=innerCollection &&  index < innerCollection.Count; index++)
          {
            SECFiling innerFiling = innerCollection[index];
            secFilings.Add(innerFiling);
          }
        }
        return secFilings;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// Get the filing document in XML format
    private static SECFilings GetSECFilingDocuments(String symbol,DateTime filingDate,String secFilingDocumentUrl)
    {
      MemoryStream memoryStream = null;
      SECFilings secFilings = new SECFilings();
      String secFilingDocument = null;
      HttpNetResponse httpNetResponse=null;
      int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;

      try
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving documents at {0}]",secFilingDocumentUrl));
        WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Request:{0} failed with status {1}]",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"secNum\"]");
        String secAccessionNumber = secNodes[0].InnerText;
        secAccessionNumber = secAccessionNumber.Replace("\n","");
        secAccessionNumber = secAccessionNumber.Trim();
        if (secAccessionNumber.Contains("SEC Accession No.")) secAccessionNumber = secAccessionNumber.Substring(18, secAccessionNumber.Length - 18);
        else return null;
        secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"formName\"]");
        String formName = secNodes[0].InnerText;
        formName = formName.Replace("\n", "");
        formName = formName.Trim();
        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
        if (null == tables || tables.Count < 1) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
        httpNetResponse.Dispose();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments.  Examining form {0} with {1} entries",formName,rows.Count));
        // go through the rows in the table file, searching for the different submission form content
        int sequence = 0;
        for (int row = 0; row < rows.Count; row++)
        {
          HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
          if (null != headerColumns && headerColumns.Count > 0) continue;
          HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
          if (null == dataColumns || 5 != dataColumns.Count) continue;
          String document = dataColumns[2].InnerText.Trim();
          String type = dataColumns[3].InnerText.Trim();
          if(null!=type && type.Equals("GRAPHIC"))continue;
//          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments: SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
          if (document.Contains(".htm"))
          {
            
            HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
            if (null == nodes || 0 == nodes.Count) continue;
            String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments.  Requesting {0}",secFilingDocumentXmlUrl));
            try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
            httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
            if(!httpNetResponse.Success)
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
              continue;
            }
            secFilingDocument = httpNetResponse.ResponseString;
            if (null == secFilingDocument) continue;
            SECFiling secFiling = new SECFiling();
            secFiling.SECAccessionNumber = secAccessionNumber;
            secFiling.Sequence = sequence++;
            secFiling.Symbol = symbol;
            secFiling.Form = type;
            secFiling.Description = formName;
            secFiling.FilingDate = filingDate;
            secFiling.FileNumber = dataColumns[4].InnerText.Trim();
            secFiling.SecFilingUrl = secFilingDocumentXmlUrl;
            secFiling.FormText = secFilingDocument;
            secFiling.FormText = secFiling.FormText.Replace("\n","");
            if (null == secFiling.FormText) continue;
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments.  Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString()));
            secFilings.Add(secFiling);
            httpNetResponse.Dispose();
          }
        }
        return secFilings;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream)
        {
          memoryStream.Close();
          memoryStream = null;
        }
      }
    }
// ***************************************************************************************************************************************************************************************
// ********************************************************************** Y I E L D  C U R V E - T R E A S U R Y . G O V    **************************************************************
// ***************************************************************************************************************************************************************************************
    public static YieldCurve GetYieldCurve(int year)
    {
      YieldCurve yieldCurve = new YieldCurve();
      HttpNetResponse httpNetResponse=null;

      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/pages/xml?data=daily_treasury_yield_curve&field_tdr_date_value=").Append(year);
        strRequest = sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        XElement po = XElement.Parse(httpNetResponse.ResponseString);
        IEnumerable<XElement> childElements =  from el in po.Elements() where el.Name.LocalName.Equals("entry") select el;
        foreach (XElement element in childElements)
        {
          IEnumerable<XElement> childElementsL1 =  from el in element.Elements() select el; 
          foreach (XElement elementL1 in childElementsL1)
          {
            if(elementL1.Name.LocalName.Equals("content"))
            {
              IEnumerable<XElement> elementsL2=elementL1.Elements();
              if(!elementsL2.FirstOrDefault().Name.LocalName.Equals("properties"))continue;
              IEnumerable<XElement> elementsL3=elementsL2.FirstOrDefault().Elements();
              YieldCurveData yieldCurveData = new YieldCurveData();
              yieldCurveData.Date=Utility.Epoch;
              foreach (XElement elementL3 in elementsL3)
              {
                if(elementL3.Name.LocalName.EndsWith("NEW_DATE")) yieldCurveData.Date=Utility.ParseDate(Utility.RemoveAfter(elementL3.Value,'T'));
                else if(elementL3.Name.LocalName.EndsWith("1MONTH"))yieldCurveData.Mo1=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("2MONTH"))yieldCurveData.Mo2=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("3MONTH"))yieldCurveData.Mo3=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("6MONTH"))yieldCurveData.Mo6=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("1YEAR"))yieldCurveData.Yr1=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("2YEAR"))yieldCurveData.Yr2=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("3YEAR"))yieldCurveData.Yr3=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("5YEAR"))yieldCurveData.Yr5=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("7YEAR"))yieldCurveData.Yr7=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("10YEAR"))yieldCurveData.Yr10=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("20YEAR"))yieldCurveData.Yr20=FeedParser.ParseValue(elementL3.Value);
                else if(elementL3.Name.LocalName.EndsWith("30YEAR"))yieldCurveData.Yr30=FeedParser.ParseValue(elementL3.Value);
              }
              if(yieldCurveData.IsValid())yieldCurve.Add(yieldCurveData);
            }
          } 
        }
        yieldCurve=new YieldCurve(yieldCurve.Distinct().ToList());
        return yieldCurve;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return yieldCurve;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// ********************************************************************** E T F  H O L D I N G S  -  Y A H O O  F I N A N C E    *********************************************************
// ***************************************************************************************************************************************************************************************

    public static ETFHoldings GetETFHoldings(String etfSymbol)
    {
      MemoryStream memoryStream = null;
      ETFHoldings etfHoldings = new ETFHoldings();
      HttpNetResponse httpNetResponse=null;
      DateTime modified=DateTime.Now;

      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        etfSymbol = etfSymbol.ToUpper();
        sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings");
        strRequest = sb.ToString();

        WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
//        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS,webProxy);
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }

        List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
        if(null == sections || 1!=sections.Count)
        {
          etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString);
          if(null==etfHoldings)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response.");
            return null;
          }
          return etfHoldings;
        }
        String marker="<table class=\"W(100%) M(0) BdB Bdc($seperatorColor)";
        if(!sections[0].StartsWith(marker))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings: Expected marker {0}",marker));
          return null;
        }

        byte[] streamBytes=Encoding.ASCII.GetBytes(sections[0]);
        memoryStream=new MemoryStream(streamBytes);
        HtmlDocument htmlDocument=new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes(".//td");
        if(null==rows || 0==rows.Count)return null;
        for(int rowIndex=0;rowIndex<rows.Count;rowIndex+=3)
        {
          HtmlNode node = rows[rowIndex];
          ETFHolding etfHolding = new ETFHolding();
          etfHolding.ETFSymbol = etfSymbol;
          etfHolding.HoldingSymbolShareClass = null;
          etfHolding.HoldingCompanyName = rows[rowIndex].InnerText;
          etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = rows[rowIndex+1].InnerText;
          if(null == etfHolding.HoldingSymbol || 
            "N/A".Equals(etfHolding.HoldingSymbol) || 
              "".Equals(etfHolding.HoldingSymbol))continue;
          etfHolding.PercentOfAssets = FeedParser.ParseValue(rows[rowIndex+2].InnerText);
          etfHolding.Modified = modified;
          etfHoldings.Add(etfHolding);
        }
        return etfHoldings;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }

    private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString)
    {
      ETFHoldings etfHoldings = new ETFHoldings();
      try
      {
        DateTime modified = DateTime.Now;
        int groupBy=3;
        List<String> sections = Sections.GetAllItemsInSections(responseString,"section");
        if(null==sections || 0==sections.Count)return null;
        String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault();
        if(String.IsNullOrEmpty(sectionItem))return null;
        List<String> spans =Sections.GetAllItemsInSections(sectionItem,"span"); 
        if(null==spans || 0==spans.Count || spans.Count<groupBy)return null;
        for(int index=0;index<spans.Count;index+=groupBy)
        {
          String strElement = spans[index];
          if(0==index)
          {
            String symbolNameHeading=Utility.BetweenString(spans[index],">","<");
            String companyNameHeading=Utility.BetweenString(spans[index+1],">","<");
            String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<");
            if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets"))
            {
              MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading.");
              return null;
            }
          }
          else
          {
            ETFHolding etfHolding = new ETFHolding();
            etfHolding.ETFSymbol = etfSymbol;
            etfHolding.HoldingSymbolShareClass = null;
            etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<");
            etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<");
            if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue;
            etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<"));
            etfHolding.Modified = modified;
            etfHoldings.Add(etfHolding);
          }
        }
        return 0==etfHoldings.Count?null:etfHoldings;
       }
       catch(Exception exception)
       {
         MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString()));
         return null;
       }
    }
// ******************************************************************************************************************************************************************************
// **************************************************************************C O M P A N Y  P R O F I L E  **********************************************************************
// ***************************************************************** P R O F I L E  :  M O R N I N G S T A R  *******************************************************************
// ***************************************************************** D E S C R I P T I O N  : R E U T E R S  ********************************************************************
// ******************************************************************************************************************************************************************************
    public static CompanyProfile GetCompanyProfile(String symbol)
    {
      String nasdaq = "xnas";
      String nyse = "xnys";
      CompanyProfile companyProfile = null;
      companyProfile = GetCompanyProfileYahoo(symbol);
      if (null==companyProfile)companyProfile=GetCompanyProfileMorningStar(symbol,nasdaq);
      if (null == companyProfile) companyProfile = GetCompanyProfileMorningStar(symbol, nyse);
      return companyProfile;
    }

    // GetCompanyProfile - DataSource Yahoo Finance
    public static CompanyProfile GetCompanyProfileYahoo(String symbol)
    {
      HttpNetResponse httpNetResponse = null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();

        WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo");
        sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile");
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);

//        httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest);

        if (!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }

// Locate Industry
        String strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry", 4);
        if(strIndustry == null)strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry:??", 2);

// Locate Sector
        String strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector", 4);
        if (null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector(s)", 4);
        if(null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector:??", 3);

        if (null != strIndustry && strIndustry.Contains(Constants.CONST_QUESTION)) strIndustry = strIndustry.Replace(Constants.CONST_QUESTION, " - ");
        if (null != strSector && strSector.Contains(Constants.CONST_QUESTION)) strSector = strSector.Replace(Constants.CONST_QUESTION, " - ");

// Locate Description
        String strDescription = Sections.LocateItem(httpNetResponse.ResponseString, "Description", 4);
        if(null == strDescription)
        {
          List<int> indices = Sections.LocateAllOccurrences(httpNetResponse.ResponseString, "Description");
          if(indices.Count>0)
          {
            List<String> sections = Sections.GetSections(httpNetResponse.ResponseString);
            strDescription = Sections.GetFirstNonEmptyItemInSection(sections, indices[0]+1);
          }
        }

        if(null!=strDescription && strDescription.Equals("Description Information Not Available"))
        {
          strDescription = null;
        }

        if(null == strIndustry || null == strSector || null == strDescription)
        {
          return null;
        }

        CompanyProfile companyProfile = new CompanyProfile();
        companyProfile.Symbol = symbol;
        companyProfile.Sector = strSector;
        companyProfile.Industry = strIndustry;
        companyProfile.Description = strDescription;
        if (null != companyProfile.Sector) companyProfile.Sector = companyProfile.Sector.Trim();
        if (null != companyProfile.Industry) companyProfile.Industry = companyProfile.Industry.Trim();
        if (null != companyProfile.Description) companyProfile.Description = Utility.RemoveHtml(companyProfile.Description.Trim());
        return companyProfile;
      } 
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG, exception);
        return null;
      } 
      finally
      {
        if (null != httpNetResponse) httpNetResponse.Dispose();
      }
    }

    public static CompanyProfile GetCompanyProfileMorningStar(String symbol,String exchange)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyProfileMorningStar");
        sb.Append(String.Format("https://www.morningstar.com/stocks/{0}/{1}/quote",exchange,symbol));
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
//        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy);
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy);

        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }

        String strSector=null;
        String strIndustry=null;
        String strDescription=null;
        int itemIndex=0;
        List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);

        bool found=Sections.FindInSections(sections,"Sector",0,ref itemIndex);
        if(found)strSector=sections[itemIndex+3];

        found=Sections.FindInSections(sections,"Industry",itemIndex,ref itemIndex);
        if(found)strIndustry=sections[itemIndex+3];

        found=Sections.FindInSections(sections,"Business Description",itemIndex,ref itemIndex);
        if(found)strDescription=sections[itemIndex+2];
        else
        {
          itemIndex=0;
          found=Sections.FindInSections(sections,"Company Profile",itemIndex,ref itemIndex);
          if(found)strDescription = Sections.GetFirstNonEmptyItemInSection(sections, itemIndex+1);
        }

        if(null==strIndustry && null==strSector)return null;

        strSector=strSector.Trim();
        strIndustry=strIndustry.Trim();
        CompanyProfile companyProfile = new CompanyProfile();
        companyProfile.Symbol = symbol;
        companyProfile.Sector = strSector;
        companyProfile.Industry = strIndustry;
        companyProfile.Description=strDescription;
        if(null!=companyProfile.Sector)companyProfile.Sector=companyProfile.Sector.Trim();
        if(null!=companyProfile.Industry)companyProfile.Industry=companyProfile.Industry.Trim();
        if(null!=strDescription)companyProfile.Description=companyProfile.Description.Trim();

        if(null!=companyProfile.Industry && companyProfile.Industry.Contains(Constants.CONST_QUESTION))
        {
          companyProfile.Industry=companyProfile.Industry.Replace(Constants.CONST_QUESTION," - ");
        }
        if(null!=companyProfile.Sector && companyProfile.Sector.Contains(Constants.CONST_QUESTION))
        {
          companyProfile.Sector=companyProfile.Sector.Replace(Constants.CONST_QUESTION," - ");
        }
        if((null!=companyProfile.Sector && companyProfile.Sector.Equals(Constants.CONST_QUESTION)) 
          || (null!=companyProfile.Industry && companyProfile.Industry.Equals(Constants.CONST_QUESTION)) 
          || (null!=companyProfile.Description && companyProfile.Description.Equals(Constants.CONST_QUESTION)))return null;
        return companyProfile;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }

// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S  - S E E K I N G   A L P H A ***********************************************************
// ***************************************************************************************************************************************************************************
    public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol)
    {
      Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol);
      //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol);
      //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol);
      return headlines;
    }

    //public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
    //{
    //  HttpNetResponse httpNetResponse=null;
    //  Headlines headlines=new Headlines();
    //  try
    //  {
    //    StringBuilder sb = new StringBuilder();
    //    String strRequest;
    //    symbol = symbol.ToUpper();

    //    #region Create Cookies

    //    StringBuilder lastVisitedPage = new StringBuilder();
    //    lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
    //    lastVisitedPage.Append(symbol);
    //    lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");

    //    String domain = "seekingalpha.com";
    //    CookieCollection cookieCollection = new CookieCollection();
    //    cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
    //    cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});

    //    #endregion

    //    sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
    //    strRequest = sb.ToString();
    //    MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
    //    WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
    //    httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
    //    if(!httpNetResponse.Success)
    //    {
    //      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
    //      return null;
    //    }
    //    byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
    //    List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
    //    if(null==keyValuePairs)return null;

    //    foreach (KeyValue keyValue in keyValuePairs)
    //    {
    //      Headline headline=null;
    //      if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
    //      else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
    //      headline.Entry=headline.Entry.Replace("\\"," ");
    //      headline.Entry=headline.Entry.Trim();
    //      headline.Entry=Uri.UnescapeDataString(headline.Entry);
    //      headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
    //      headline.Entry=Utility.RemoveHtml(headline.Entry);
    //      headline.Source = "Seeking Alpha";
    //      headlines.Add(headline);
    //    }
    //    return headlines;
    //  }
    //  catch (Exception exception)
    //  {
    //    MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
    //    return null;
    //  }
    //  finally
    //  {
    //    if(null!=httpNetResponse)httpNetResponse.Dispose();
    //  }
    //}

    //public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
    //{
    //  HttpNetResponse httpNetResponse=null;
    //  Headlines headlines=new Headlines();

    //  try
    //  {
    //    StringBuilder sb = new StringBuilder();
    //    String strRequest;
    //    symbol = symbol.ToUpper();

    //    DateTime marketDate=PremarketDA.GetLatestMarketDate();
    //    if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;

    //    #region Create Cookies

    //    StringBuilder lastVisitedPage = new StringBuilder();
    //    lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
    //    lastVisitedPage.Append(symbol);
    //    lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");

    //    String domain = "seekingalpha.com";
    //    CookieCollection cookieCollection = new CookieCollection();
    //    cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
    //    cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});

    //    #endregion

    //    sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
    //    sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
    //    sb.Append("T04%3A00%3A00.000Z&to=");
    //    sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
    //    sb.Append("T14%3A20%3A34.999Z");
    //    strRequest = sb.ToString();
    //    MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
    //    WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
    //    httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
    //    if(!httpNetResponse.Success)
    //    {
    //      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
    //      return null;
    //    }
    //    byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
    //    List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
    //    if(null==keyValuePairs)return null;

    //    foreach (KeyValue keyValue in keyValuePairs)
    //    {
    //      Headline headline=null;
    //      if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
    //      else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
    //      headline.Entry=headline.Entry.Replace("\\"," ");
    //      headline.Entry=headline.Entry.Trim();
    //      headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
    //      headline.Entry=Uri.UnescapeDataString(headline.Entry);
    //      headline.Source = "Seeking Alpha";
    //      headlines.Add(headline);
    //    }
    //    return headlines;
    //  }
    //  catch (Exception exception)
    //  {
    //    MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
    //    return null;
    //  }
    //  finally
    //  {
    //    if(null!=httpNetResponse)httpNetResponse.Dispose();
    //  }
    //}

    /// <summary>
    /// Retrieves headlines from seeking alpha.  If ignoreMarketDate is TRUE then the results will not be 
    /// filtered by the current market date .
    /// </summary>
    /// <param name="symbol"></param>
    /// <param name="ignoreMarketDate"></param>
    /// <returns></returns>
    public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false)
    {
      HttpNetResponse httpNetResponse=null;
      Headlines headlines=new Headlines();

      try
      {
        String domain = "seekingalpha.com";
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();

// Visit the home page
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV3");
        sb.Append("https://").Append(domain);

        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(sb.ToString(),domain,30000,webProxy,false);
        if(!httpNetResponse.Success)return null;
        CookieCollection cookieCollection = httpNetResponse.CookieCollection;

        sb=new StringBuilder();
        DateTime marketDate=PremarketDA.GetLatestMarketDate();
        if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;

        #region Create Cookies

        StringBuilder lastVisitedPage = new StringBuilder();
        lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
        lastVisitedPage.Append(symbol);
        lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");

        cookieCollection.Add(new Cookie("LAST_VISITED_PAGE", lastVisitedPage.ToString()) { Domain = domain });
        cookieCollection.Add(new Cookie("_ga", "GA1.1.1862228764.1739274352") { Domain = domain });
        cookieCollection.Add(new Cookie("_ga_KGRFF2R2C5","GS1.1.1739274352.1.1.1739274637.60.0.0"){ Domain = domain });
        cookieCollection.Add(new Cookie("_gcl_au","1.1.1611738349.1739274352"){ Domain = domain });
        cookieCollection.Add(new Cookie("_pxvid","bfabea2f-e86d-11ef-832d-24c9f227f821"){ Domain = domain });
        cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain });
        cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain });

        #endregion
        sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]=");
//        sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2));
//        sb.Append("T04%3A00%3A00.000Z");
        sb.Append("0");
        sb.Append("&filter[until]=");
//        sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2));
//        sb.Append("T14%3A20%3A34.999Z");
        sb.Append("0");
        sb.Append("&id=").Append(symbol);
        sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1");
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
        
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection);

        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
        if(null==keyValuePairs)return null;

        foreach (KeyValue keyValue in keyValuePairs)
        {
          Headline headline=null;
          if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
          else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
          headline.Entry=headline.Entry.Replace("\\"," ");
          headline.Entry=headline.Entry.Trim();
          headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
          headline.Entry=Uri.UnescapeDataString(headline.Entry);
          headline.Source = "Seeking Alpha";
          headlines.Add(headline);
        }

        if(!ignoreMarketDate)
        {
          headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
        }

        return headlines;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }

// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S  - M A R K E T  W A T C H  *************************************************************
// ***************************************************************************************************************************************************************************

    public static Headlines GetCompanyHeadlinesMarketWatch(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      Headlines headlines=new Headlines();
      DateTime modified=DateTime.Now;

      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();
        MemoryStream memoryStream=null;
        CookieCollection cookieCollection=new CookieCollection();
        httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.marketwatch.com",cookieCollection);
        Thread.Sleep(500);
        sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol).Append("?mod=search_symbol");
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesMarketWatch");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,DEFAULT_TIMEOUT_MS,webProxy,true,cookieCollection);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream=new MemoryStream(streamBytes);
        HtmlDocument htmlDocument=new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection divSections=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"article__content\"]");

        if(null == divSections)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for request {0} ",strRequest));
          return headlines;
        }
        CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);


        for(int nodeIndex=0;nodeIndex<divSections.Count;nodeIndex++)
        {
          HtmlNode node=divSections[nodeIndex];

          byte[] bytes=Encoding.ASCII.GetBytes(node.InnerHtml);
          memoryStream=new MemoryStream(bytes);
          HtmlDocument document=new HtmlDocument();
          document.Load(memoryStream);
          HtmlNode htmlNodeHeadline = document.DocumentNode.SelectSingleNode("//*[@class=\"article__headline\"]");
          HtmlNode htmlNodeTimestamp = document.DocumentNode.SelectSingleNode("//*[@class=\"article__timestamp\"]");
          if(null==htmlNodeHeadline || null==htmlNodeTimestamp)
          {
            continue;
          }

          List<String> sections=Sections.GetSections(htmlNodeHeadline.InnerHtml);
          if(null == sections || 0==sections.Count)continue;
          String strHeadline = Utility.RemoveHtml(sections[sections.Count-1]);
          String strTimestamp = htmlNodeTimestamp.InnerText;

          if(String.IsNullOrEmpty(strHeadline) || String.IsNullOrEmpty(strTimestamp))
          {
            continue;
          }

          Headline headline= new Headline();
          headline.Symbol=symbol;
          headline.Date=FeedParser.ParseValueDateTimeMonthFormatTZ(strTimestamp);
          headline.Entry=strHeadline.Trim();
          headline.Source="MarketWatch";
          headline.Modified=modified;
          headline.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;

          if(Utility.IsEpoch(headline.Date))
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Invalid date for symbol {0}",symbol));
            continue;
          }
          headlines.Add(headline);
        }
        headlines=new Headlines(headlines.Distinct().ToList());
        return headlines;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************
//************************************************************************ H E A D L I N E S  - N A S D A Q ******************************************************************
// ***************************************************************************************************************************************************************************
    public static Headlines GetCompanyHeadlinesNASDAQ(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      Headlines headlines=new Headlines();
      try
      {
        StringBuilder sb=new StringBuilder();
        String strRequest;
        symbol=symbol.ToUpper();
        sb.Append("https://api.nasdaq.com/api/news/topic/articlebysymbol?q=").Append(symbol).Append("|etf&offset=0&limit=8&fallback=false");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }

//	      List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
	      List<List<String>> items=LocateJSONItemsTags(httpNetResponse.ResponseString);
				if(null==items || items.Count<7)return null;
				int index=0;
				for(;index<items.Count;index++)
				{
				  if((items[index])[0].Equals("title"))break;
				}
				for(;index<items.Count;index+=7)
				{
				  if(!(items[index])[0].Equals("title"))continue;

          DateTime referenceDate = Utility.Epoch;
          if((items[index+3])[0].Equals("ago"))
          {
            referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
          }
          else if((items[index+2])[0].Equals("ago"))
          {
            referenceDate=ProcessNASDAQRelativeDate((items[index+2])[1]);
          }
          else continue;

//					if(!(items[index+3])[0].Equals("ago"))continue;
//					DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
					if(Utility.IsEpoch(referenceDate))continue;
					Headline headline=new Headline();
					headline.Source="NASDAQ";
					headline.Symbol=symbol;
					headline.Date=referenceDate;
					headline.Entry=Uri.UnescapeDataString((items[index])[1]);
          headline.Entry=Utility.RemoveHtml(headline.Entry);
					headlines.Add(headline);
				}
        return headlines;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse) httpNetResponse.Dispose();
        if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
      }
    }
// ***************************************************************************************************************************************************************************
// ************************************ C O N S U M E R  P R I C E  I N D E X  -  B U R E A U  O F  L A B O R  S T I S T I C S ************************************************
// ***************************************************************************************************************************************************************************

		public static PriceIndices GetConsumerPriceIndices()
		{
		  String indexCode=null;
			String indexName=null;
			String indexSource="BLS";
		  HttpNetResponse httpNetResponse=null;
			PriceIndices priceIndices=null;
			MemoryStream memoryStream=null;

			try
			{
			  StringBuilder sb=new StringBuilder();
				sb.Append("https://www.bls.gov/regions/mid-atlantic/data/consumerpriceindexhistorical_us_table.htm");
				String strRequest=sb.ToString();

        WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
//        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy);
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,300000,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
				byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
				memoryStream = new MemoryStream(streamBytes);
				HtmlDocument htmlDocument = new HtmlDocument();
				htmlDocument.Load(memoryStream);
				HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"ro3fxusu_cms\"]");
				HtmlNodeCollection rowData=table[0].SelectNodes(".//tr");

				HtmlNodeCollection titles=table[0].SelectNodes("//*[@class=\"tableTitle\"]");
				if(titles.Count<1)
				{
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate title."));
				  return null;
				}
				String titleInfo=titles[0].InnerText;
				indexName=Utility.BetweenString(titleInfo,null,"(").Trim();
				indexCode=Utility.BetweenString(titleInfo,"(",")").Trim();
				if(null==indexName || null==indexCode)
				{
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate indexName/indexCode."));
				  return null;
				}
				if(rowData.Count<3)
				{
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, insufficient row data."));
				  return null;
				}
				HtmlNode htmlNode=rowData[2].SelectNodes(".//th").FirstOrDefault();
				if(null==htmlNode)
				{
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate proper header."));
				  return null;
				}
				if(!"Consumer Price Index".Equals(htmlNode.InnerText))
				{
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate consumer price index."));
				  return null;
				}

				priceIndices=new PriceIndices();
				for(int index=3;index<rowData.Count;index++)
        {
				  HtmlNodeCollection headerElements=rowData[index].SelectNodes(".//th");
					if(null==headerElements || 0==headerElements.Count)break;
				  HtmlNode headerNode=headerElements.FirstOrDefault();
					if(null==headerNode ||"".Equals(headerNode.InnerText))break;
					int year=int.Parse(headerNode.InnerText);
					HtmlNodeCollection monthsDataCollection=rowData[index].SelectNodes(".//td");
					for(int monthIndex=0;monthIndex<monthsDataCollection.Count;monthIndex++)
					{
					  String strValue=monthsDataCollection[monthIndex].InnerText;
						if(null==strValue || "".Equals(strValue))break;
						double value=FeedParser.ParseValue(strValue);
						if(double.IsNaN(value))break;
						PriceIndex priceIndex=new PriceIndex();
						priceIndex.AsOf=new DateTime(year,monthIndex+1,1);
						priceIndex.Code=indexCode;
						priceIndex.Name=indexName;
						priceIndex.Value=value;
						priceIndex.Source=indexSource;
						priceIndices.Add(priceIndex);
					}
				}
				return priceIndices;
			}
			catch(Exception exception)
			{
			  MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
				return null;
			}
			finally
			{
			  if(null!=httpNetResponse)
				{
				  httpNetResponse.Dispose();
				}
				if(null!=memoryStream)
				{
				  memoryStream.Close();
					memoryStream.Dispose();
					memoryStream=null;
				}
			}
		}
// ***************************************************************************************************************************************************************************
// ******************************************************** A N A L Y S T  P R I C E T A R G E T  -  M A R K E T  B E A T  ***************************************************
// ***************************************************************************************************************************************************************************
    public static AnalystPriceTarget GetAnalystPriceTarget(String symbol)
    {
      //AnalystPriceTarget analystPriceTarget = GetAnalystPriceTargetYahoo(symbol);
      AnalystPriceTarget analystPriceTarget = GetAnalystPriceTargetMarketBeat(symbol);
      return analystPriceTarget;
    }

    //public static AnalystPriceTarget GetAnalystPriceTargetMarketBeat(String symbol)
    //{
    //  AnalystPriceTarget analystPriceTarget = null;
    //  String nyse="NYSE";
    //  String nasdaq="NASDAQ";

    //  try
    //  {
    //    HtmlNodeCollection table = GetMarketBeatAnalystPriceTargetTable(symbol, nasdaq);
    //    if(null==table)table = GetMarketBeatAnalystPriceTargetTable(symbol, nyse);
    //    if(null==table)return null;
    //    HtmlNodeCollection headerColumns=table[0].SelectNodes(".//th");
    //    HtmlNodeCollection dataColumns=table[0].SelectNodes(".//td");


    //    analystPriceTarget=new AnalystPriceTarget();
    //    analystPriceTarget.Symbol=symbol;
    //    analystPriceTarget.Date=DateTime.Now.Date;
    //    for(int index=0;index<headerColumns.Count;index++)
    //    {
    //      String forecastType = headerColumns[index].InnerText;
    //      String forecastData = dataColumns[index].InnerText;

    //      if (forecastType.Equals("High Forecast"))
    //      {
    //        analystPriceTarget.HighTargetPrice = FeedParser.ParseValue(forecastData);
    //      } 
    //      else if (forecastType.Equals("Average Forecast"))
    //      {
    //        analystPriceTarget.MeanTargetPrice = FeedParser.ParseValue(forecastData);
    //        analystPriceTarget.MedianTargetPrice = analystPriceTarget.MeanTargetPrice;
    //      } 
    //      else if (forecastType.Equals("Low Forecast"))
    //      {
    //        analystPriceTarget.LowTargetPrice = FeedParser.ParseValue(forecastData);
    //      }
    //    }

    //    if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
    //       double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
    //       double.IsNaN(analystPriceTarget.LowTargetPrice)&&
    //       double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
    //    return analystPriceTarget;
    //  }
    //  catch (Exception exception)
    //  {
    //    MDTrace.WriteLine(LogLevel.DEBUG,exception);
    //    return null;
    //  }
    //}
    public static AnalystPriceTarget GetAnalystPriceTargetMarketBeat(String symbol)
    {
      AnalystPriceTarget analystPriceTarget = null;
      String nyse="NYSE";
      String nasdaq="NASDAQ";

      try
      {
        HtmlNodeCollection table = GetMarketBeatAnalystPriceTargetTable(symbol, nasdaq);
        if(null==table)table = GetMarketBeatAnalystPriceTargetTable(symbol, nyse);
        if(null==table)return null;

        HtmlNodeCollection rows=table[0].SelectNodes(".//tr");

        if(null == rows || 0==rows.Count)return null;

        analystPriceTarget=new AnalystPriceTarget();
        analystPriceTarget.Symbol=symbol;
        analystPriceTarget.Date=DateTime.Now.Date;

        for(int rowIndex=0;rowIndex<rows.Count;rowIndex++)
        {
          HtmlNodeCollection data = rows[rowIndex].SelectNodes(".//td");

          if(null == data || data.Count ==0 || !data[0].InnerText.Equals("Consensus Price Target"))continue;

          analystPriceTarget.HighTargetPrice=
            analystPriceTarget.LowTargetPrice=
            analystPriceTarget.MeanTargetPrice=
            analystPriceTarget.MedianTargetPrice=
            FeedParser.ParseValue(data[1].InnerText);

          break;
        }
        if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
           double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
           double.IsNaN(analystPriceTarget.LowTargetPrice)&&
           double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
        return analystPriceTarget;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
    }

    private static HtmlNodeCollection GetMarketBeatAnalystPriceTargetTable(String symbol, String exchange)
    {
      MemoryStream memoryStream=null;
      HttpNetResponse httpNetResponse=null;
      try
      {
        
        StringBuilder sb = new StringBuilder();
        String strRequest;
        symbol = symbol.ToUpper();
        sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target");
        WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetMarketBeat");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetMarketBeat: {0}", strRequest));
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
        if(!httpNetResponse.Success)
        {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
            return null;
        }

        memoryStream = null;
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
//        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"w-100 my-2\"]");
        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"consensus-table\"]");
        if(null==table || 1!=table.Count)
        {
          return null;
        }
        return table;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
        if(null!=memoryStream)
        {
          memoryStream.Close();
          memoryStream.Dispose();
          memoryStream=null;
        }
      }
    }

// ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R  H I S T O R I C A L  D A T A  V 2  B E G I N *******************************************
// ****************************************************************************************************************************************************************************
// If this breaks then it may be necessary to run the developer tools and point to the operatingPerformance page on MorningStar website.
// Set a filter in the network tab in the developer tools to "operatingPerformance".  You should see the sal api information in the filter list.
// Match the salVersion to the latest sal version in the actual request.  
// Also... Examine the GetRequestNoEncodingMStar(...) method.  It may be necesary to update the RequestId and maybe some other information in there.
// Not sure how long the RequestId lives for.  I updated it on 03/12/2024
// 02-07-2025 : Morningstar moved the ROA and the ROIC so I had to point to a new url to source that information
    public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
    {
      Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
      HttpNetResponse httpNetResponse=null;
      String nasdaq = "xnas";
      String nyse = "xnyse";
      String nys="xnys";
      String salVersion="4.30.0";
      String salVerson45100="4.51.0";
      String strDefinition = default;
      String strColumnDefinition = default;
      int TIMEOUT_BETWEEN_REQUESTS_MS=500;

      String exchange=nasdaq;
      String url="https://www.morningstar.com/api/v2/stocks/";

      try
      {
        StringBuilder sb = new StringBuilder();
        StringBuilder strReferer = new StringBuilder();
        String strRequest;
        sb=new StringBuilder();
        sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        WebProxy webProxy=HttpNetRequest.GetProxy("GetHistoricalValues");

        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
        if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
        {
          sb=new StringBuilder();
          exchange=nys;
          sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
          strRequest=sb.ToString();
          MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
          httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
          if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
          {
            sb=new StringBuilder();
            exchange=nyse;
            sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
            strRequest=sb.ToString();
            MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
            httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
            if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
            {
              MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
              return null;
            }
          }
        }
// This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier.
        String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
        if(null==securityId)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
          return null;
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));

// PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(salVerson45100);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        List<List<String>> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString);
        Dictionary<String,MStarDataSet> dataSetsProfitabilityAndEfficiency=GetData("fiscalPeriodYear", profitabilityItems, httpNetResponse.ResponseString);
        //Dictionary<String,MStarDataSet> dataSetsProfitabilityAndEfficiency=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

// KETSTATS - FINANCIALHEALTH
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
        Dictionary<String,MStarDataSet> dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString);
        httpNetResponse.Dispose();

// NEWFINANCIALS - ANNUAL/SUMMARY
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

// NEWFINANCIALS - INCOMESTATEMENT
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

// NEWFINANCIALS - CASHFLOW
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

//NEWFINANCIALS - BALANCESHEET
        sb=new StringBuilder();
        sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
          return null;
        }
        Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

// RETURN ON ASSETS
        strDefinition="roa";
        strColumnDefinition="columnDefs";
        if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
        {
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// RETURN ON INVESTED CAPITAL
        strDefinition="roic";
        strColumnDefinition="columnDefs";
        if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
        {
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// BOOK VALUE PER SHARE
        strDefinition="bookValuePerShare";
        strColumnDefinition="columnDefs";
        if(dataSetsFinancialHealth.ContainsKey(strDefinition))
        {
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.BVPS;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsFinancialHealth[strColumnDefinition],dataSetsFinancialHealth[strDefinition],elementType);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// INVENTORIES
        strDefinition="Inventories";
        strColumnDefinition="columnDefs";
        if(dataSetsBalanceSheet.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Inventory;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// ACCOUNTS RECEIVABLES
        strDefinition="Trade/Accounts Receivable, Current";
        strColumnDefinition="columnDefs";
        if(dataSetsBalanceSheet.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.AccountsReceivable;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// COST OF GOOD AND SERVICES
        strDefinition="Cost of Goods and Services";
        strColumnDefinition="columnDefs";
        if(dataSetsIncomeStatement.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.COGS;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// TOTAL OPERATING PROFIT/LOSS
        strDefinition="Total Operating Profit/Loss";
        strColumnDefinition="columnDefs";
        if(dataSetsIncomeStatement.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingIncome;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// INTEREST EXPENSE
        strDefinition="Interest Expense Net of Capitalized Interest";
        strColumnDefinition="columnDefs";
        if(dataSetsIncomeStatement.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.InterestExpense;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// REPORTED EFFECTIVE TAX RATE
        strDefinition="Reported Effective Tax Rate";
        strColumnDefinition="columnDefs";
        if(dataSetsIncomeStatement.ContainsKey(strDefinition))
        {
          double multiplier=100.00;
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.TaxRate;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// TOTAL REVENUE
        strDefinition="Total Revenue";
        strColumnDefinition="columnDefs";
        if(dataSetsAnnuals.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Revenue;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// NET INCOME AVAILABLE TO COMMON STOCKHOLDERS
        strDefinition="Net Income Available to Common Stockholders";
        strColumnDefinition="columnDefs";
        if(dataSetsAnnuals.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
          multiplier/=1000000;
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.NetIncomeAvailableToCommonShareholders;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// DILUTED EPS
        strDefinition="Diluted EPS";
        strColumnDefinition="columnDefs";
        if(dataSetsAnnuals.ContainsKey(strDefinition))
        {
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.EPS;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// WORKING CAPITAL
        strDefinition="Working Capital";
        strColumnDefinition="columnDefs";
        if(dataSetsAnnuals.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.WorkingCapital;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString());
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// FREE CASHFLOW
        strDefinition="Free Cash Flow";
        strColumnDefinition="columnDefs";
        if(dataSetsAnnuals.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.FreeCashflow;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

// OPERATING CASHFLOW
        strDefinition="Cash Generated from Operating Activities";
        strColumnDefinition="columnDefs";
        if(dataSetsCashflowStatement.ContainsKey(strDefinition))
        {
          double multiplier=dataSetsCashflowStatement["multiplier"].GetItem<double>(0);
          TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingCashflow;
          TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsCashflowStatement[strColumnDefinition],dataSetsCashflowStatement[strDefinition],elementType,multiplier);
          if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
          {
            values.Add(elementType, timeSeriesCollection);
            String strMessage=String.Format("got {0} records for '{1}' for {2}.  Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
            MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
          }
        }

        return values;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception);
        return null;
      }
    }

    internal class MStarDataSet
    {
      private String name;
      private List<String> values;
      public MStarDataSet()
      {
        values=new List<String>();
      }
      public MStarDataSet(String name,List<String> values)
      {
        this.name=name;
        this.values=values;
      }
      public MStarDataSet(String name,String value)
      {
        values=new List<String>();
        this.name=name;
        this.values.Add(value);
      }
      public String Name
      {
        get{return name;}
        set{name=value;}
      }
      public List<String> Values
      {
        get{return values;}
        private set{;}
      }
      public T GetItem<T>(int index=0)
      {
        T result=default(T);
        try {result = (T)Convert.ChangeType(values[index], typeof(T));}
        catch {result = default(T);}
        return result;
      } 
    }

// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
//\"orderOfMagnitude\":\"Million\",
    private static double GetMultiplierForDataSet(String responseString)
    {
      double multiplier=1.00;
      try
      {
        string token="\"indexName\"";
        int index=responseString.IndexOf(token);
        if(-1==index)
        {
          token="\"orderOfMagnitude\"";
          index=responseString.IndexOf(token);
          if(-1==index)return multiplier;
          string strSubString=responseString.Substring(index+token.Length);
          strSubString=Utility.BetweenString(strSubString,"\"","\"");
          string[] parts=strSubString.Split(' ');
          if(1!=parts.Length)return multiplier;
          if(parts[0].Equals("Million"))multiplier=1000000;
          else if(parts[0].Equals("Billion"))multiplier=1000000000;
          else if(parts[0].Equals("Trillion"))multiplier=1000000000000;
          else multiplier=1.00;
          return multiplier;
        }
        else
        {
          string strSubString=responseString.Substring(index+token.Length);
          strSubString=Utility.BetweenString(strSubString,"\"","\"");
          string[] parts=strSubString.Split(' ');
          if(2>parts.Length)return multiplier;
          if(parts[parts.Length-2].Equals("TR"))multiplier=1000000000;
          else 
          {
            multiplier=1;
          }
          return multiplier;
        }
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMultiplierForDataSet encountered an exception:{0}",exception.ToString()));
        return multiplier;
      }
    }

// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
//\"orderOfMagnitude\":\"Million\",
// {"currency":"USD","currencySymbol":"$","orderOfMagnitude":"Billion","fiscalYearEndDate":"12-31"},"userType":"Free"}
    private static String GetCurrencyForDataSet(String responseString)
    {
      String currency=null;
      try
      {
        string token="\"indexName\"";
        int index=responseString.IndexOf(token);
        if(-1==index)
        {
          token="\"currency\"";
          index=responseString.IndexOf(token);
          if(-1==index)return currency;
          string strSubString=responseString.Substring(index+token.Length);
          strSubString=Utility.BetweenString(strSubString,"\"","\"");
          string[] parts=strSubString.Split(' ');
          if(1!=parts.Length)return currency;
          currency=parts[0];
          return currency;
        }
        else
        {
          string strSubString=responseString.Substring(index+token.Length);
          strSubString=Utility.BetweenString(strSubString,"\"","\"");
          string[] parts=strSubString.Split(' ');
          if(2>parts.Length)return currency;
          currency=parts[parts.Length-1];
          return currency;
        }
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCurrencyForDataSet encountered an exception:{0}",exception.ToString()));
        return currency;
      }
    }

    public static String GetMStarSecurityId(String symbol,String responseString)
    {
      String securityIdentifier=null;
      if(MStarSecurityDA.HasSecurityId(symbol))
      {
        securityIdentifier=MStarSecurityDA.GetSecurityId(symbol);
        return securityIdentifier; 
      }
      securityIdentifier=GetMStarSecurityId_1(responseString);
      if(null==securityIdentifier)
      {
         securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
         if(null==securityIdentifier)
         {
            securityIdentifier=GetMStarSecurityId_3(responseString);
         }
      }
      if(null==securityIdentifier)return null;
      MStarSecurityDA.PutSecurityId(symbol,securityIdentifier);
      return securityIdentifier;
    }

//byId:{\"0P000003MU\":$}
// AAPL:0P000000GY

// 5","0P000003MU","MIDD",60
    private static String GetMStarSecurityId_2(String symbol,String responseString)
    {
      try
      {
        String searchString="\""+symbol+"\"";
        int index=responseString.LastIndexOf(searchString);
        if(-1==index)return null;
        int commaCount=0;
        while(index>=0 && commaCount!=2)
        {
          char ch=responseString[index--];
          if(ch.Equals(','))commaCount++;
        }
        if(index<0)return null;
        responseString=responseString.Substring(index+1);
        String securityIdentifier=Utility.BetweenString(responseString,"\"","\"");
        if(securityIdentifier.Equals("en-us")||securityIdentifier.StartsWith("blt"))return null;
        return securityIdentifier;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_2 encountered an exception:{0}",exception.ToString()));
        return null;
      }
    }

// us","AAON",401,"subscription","blt8b91befd99376df6","bltc8ff21a187c6e079",2,"image\u002Fsvg+xml","blt187f65f0e94c77b5","0P00000012",60,80,"AAON In
    private static String GetMStarSecurityId_3(String responseString)
    {
      try
      {
        string token="0P";
        int index=responseString.IndexOf(token);
        if(-1==index)return null;
        string securityIdentifier=responseString.Substring(index-token.Length);
        securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
        return securityIdentifier;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_3 encountered an exception:{0}",exception.ToString()));
        return null;
      }
    }

//","performanceID":"0P000000GY","securityID":"0P000000GY","ticker":"AAPL","exc
    private static String GetMStarSecurityId_1(String responseString)
    {
      try
      {
        string token="\"securityID\"";
        int index=responseString.IndexOf(token);
        if(-1==index)return null;
        string securityIdentifier=responseString.Substring(index+token.Length);
        securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
        return securityIdentifier;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_1 encountered an exception:{0}",exception.ToString()));
        return null;
      }
    }

    private static Dictionary<String,MStarDataSet> GetData(String responseString)
    {
      try
      {
        Dictionary<String,String> uniqueDataLabels=new Dictionary<String,String>();
        String label="\"label\"";
        String datum="\"datum\"";
        String columnDefs="\"columnDefs\"";
        String strData=null;
        Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
        if(null==responseString)return data;
        int index=responseString.IndexOf(columnDefs);
        if(-1==index)return null;
        strData=responseString.Substring(index+columnDefs.Length);
        strData=Utility.BetweenString(strData,"[","]");
        List<String> items = Utility.ToList(strData,',');
        for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
        {
          String strItem=items[itemIndex];
          strItem=Utility.RemoveQuotes(strItem);
          items[itemIndex]=strItem;
        }
        String labelName=Utility.RemoveQuotes(columnDefs);
        data.Add(labelName,new MStarDataSet(labelName,items));

        int dataIndex=0;
        for(index=responseString.IndexOf(label);index!=-1;index=responseString.IndexOf(label,dataIndex))
        {

          dataIndex=responseString.IndexOf(datum,index);
          String strLabel=responseString.Substring(index+label.Length);
          strLabel=Utility.BetweenString(strLabel,"\"","\"");
          if(uniqueDataLabels.ContainsKey(strLabel))break;
          uniqueDataLabels.Add(strLabel,strLabel);
          strData=responseString.Substring(dataIndex+datum.Length);
          strData=Utility.BetweenString(strData,"[","]");
          items=Utility.ToList(strData,',');
          for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
          {
            String strItem=items[itemIndex];
            strItem=Utility.RemoveQuotes(strItem);
            items[itemIndex]=strItem;
          }
          data.Add(strLabel,new MStarDataSet(strLabel,items));
        }
        double multiplier=GetMultiplierForDataSet(responseString);
        String currency=GetCurrencyForDataSet(responseString);
        data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString()));
        data.Add("currency",new MStarDataSet("currency",currency));
        return data;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String responseString) encountered an exception:{0}",exception.ToString()));
        return new Dictionary<String,MStarDataSet>();
      }
    }
    private static Dictionary<String,MStarDataSet> GetData(String mainHeading, List<List<String>> items,String responseString)
    {
      try
      {
        Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
        Dictionary<String,Dictionary<String,String>> dataDictionary=new Dictionary<String,Dictionary<String,String>>();
        Dictionary<String,String> uniqueElementNamesDictionary=new Dictionary<String,String>();

        if(null==mainHeading || null==items || null==responseString)
        {
          return data;
        }
        for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
        {
          List<String> subItems=items[itemIndex];
          if(subItems[0].Equals(mainHeading))
          {
            String strDate=subItems[1];
            dataDictionary.Add(strDate,new Dictionary<String,String>());  
            for(++itemIndex;itemIndex<items.Count;itemIndex++)
            {
              List<String> subsubItems=items[itemIndex];
              if(subsubItems[0].Equals(mainHeading))
              {
                itemIndex--;
                break;
              }
              Dictionary<String,String> datedItems=dataDictionary[strDate];
              String subItemElementName=subsubItems[0];
              if(!datedItems.ContainsKey(subItemElementName))
              {
                if(!uniqueElementNamesDictionary.ContainsKey(subItemElementName))uniqueElementNamesDictionary.Add(subItemElementName,subItemElementName);
                datedItems.Add(subItemElementName,subsubItems[1]);
              }
            }
          }
        }
        List<String> strDates=new List<String>(dataDictionary.Keys);
        List<String> uniqueElementNames=new List<String>(uniqueElementNamesDictionary.Keys);
        data.Add("columnDefs",new MStarDataSet("columnDefs",strDates));

        foreach(String elementName in uniqueElementNames)
        {
          List<String> elementValues=new List<String>();
          foreach(String strDate in strDates)
          {
            Dictionary<String,String> inner=dataDictionary[strDate];
            if(!inner.ContainsKey(elementName))
            {
              continue;
            }
            elementValues.Add(inner[elementName]);
          }
          data.Add(elementName,new MStarDataSet(elementName,elementValues));
        }
        double multiplier=GetMultiplierForDataSet(responseString);
        if(double.IsNaN(multiplier))multiplier=1.00;                                 // This GetData method is only used to retrieve ratios so far so the data sets do not have a multiplier section.
        data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString())); // If this method is expanded to retrieve data points other than non-ratios then you will have to come up with method for retrieving the multiplier.
        return data;                                                                 // so that we know if the data is in trillions, billions, millions and what not.
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String mainHeading, List<List<String>> items) encountered an exception {0}",exception.ToString()));
        return new Dictionary<String,MStarDataSet>();
      }
    }

    private static TimeSeriesCollection CreateTimeSeriesCollection(String symbol, MStarDataSet colDefData, MStarDataSet dataElements, TimeSeriesElement.ElementType elementType,double multiplier=1.00)
    {
      TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection();
      Dictionary<DateTime,DateTime> dates = new Dictionary<DateTime,DateTime>();
      DateTime today=DateTime.Now;
      try
      {
        if(null==colDefData || null==dataElements || null==colDefData.Values || null==dataElements.Values || dataElements.Values.Count()!=colDefData.Values.Count())
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot create time series for symbol {0} for element type {1}",symbol,TimeSeriesElement.StringForType(elementType)));
          return timeSeriesCollection;
        }    
        for (int index = 0; index < colDefData.Values.Count;index++)
        {
          DateTime reportDate=DateTime.Now;
          String strDateElement=colDefData.Values[index];
          String strDataElement=dataElements.Values[index];

          if("_PO_".Equals(strDataElement))
          {
            continue;
          }
          if("null".Equals(strDataElement))
          {
            continue;
          }

          if(strDateElement.Length.Equals(4))
          {
            bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
            if(!result)continue;
            if(reportDate.Year.Equals(today.Year))
            {
              reportDate=new DateTime(reportDate.Year,today.Month,1);
            }
            else
            {
             reportDate=new DateTime(reportDate.Year,12,31);
            }
          }
          else
          {
            bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy-MM"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
            if(!result)continue;
          }
          TimeSeriesElement timeSeriesElement = new TimeSeriesElement();
          timeSeriesElement.Type = elementType;
          timeSeriesElement.Symbol = symbol;
          timeSeriesElement.AsOf =reportDate;
          timeSeriesElement.Value = FeedParser.ParseValue(dataElements.Values[index])*multiplier;
          if (double.IsNaN(timeSeriesElement.Value)) continue;
          if (!dates.ContainsKey(timeSeriesElement.AsOf))
          {
            dates.Add(timeSeriesElement.AsOf, timeSeriesElement.AsOf);
            timeSeriesCollection.Add(timeSeriesElement);
          }
        }
        return timeSeriesCollection;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateTimeSeriesCollection encountered an exception processing symbol '{0}' {1}",symbol,exception.ToString()));
        return new TimeSeriesCollection();
      }
    }
// ****************************************************************************************************************************************************************************
// ************************************************************ M O R N I N G S T A R  H I S T O R I C A L  D A T A  V 2  E N D ***********************************************
// ****************************************************************************************************************************************************************************

// *****************************************************************************************************************************************************************************
// ************************************************************ C A S H F L O W  S T A T E M E N T  - M O R N I N G S T A R   V 2 **********************************************
// *****************************************************************************************************************************************************************************
    public static List<CashflowStatement> GetCashflowStatement(String symbol,CashflowStatement.PeriodType periodType)
    {
      List<CashflowStatement> cashflowStatements=null;
      HttpNetResponse httpNetResponse=null;
      String nasdaq = "xnas";
      String nyse = "xnyse";
      String nys="xnys";

      try
      {
        if(!CashflowStatement.PeriodType.Annual.Equals(periodType))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unsupported CashflowStatement.PeriodType Quarterly.  Only Annual supported from M*"));
          return null;
        }
        StringBuilder sb = new StringBuilder();
        StringBuilder strReferer = new StringBuilder();
        String strRequest;
        sb=new StringBuilder();
        sb.Append("https://www.morningstar.com/stocks/").Append(nasdaq).Append("/").Append(symbol).Append("/performance");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement");
        httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
        if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
        {
          sb=new StringBuilder();
          sb.Append("https://www.morningstar.com/stocks/").Append(nys).Append("/").Append(symbol).Append("/performance");
          strRequest=sb.ToString();
          MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
          httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
          if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
          {
            sb=new StringBuilder();
            sb.Append("https://www.morningstar.com/stocks/").Append(nyse).Append("/").Append(symbol).Append("/performance");
            strRequest=sb.ToString();
            MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
            httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
            if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
            {
              MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
              return null;
            }
          }
        }
        String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
        if(null==securityId)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
          return null;
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));  
        try{Thread.Sleep(250);}finally{;}  

// Fetch the equity financial details.  Success often requires several attempts.
        httpNetResponse=GetMStarEquityFinancialDetails(securityId);
        if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to retrieve MorningStar EquityFinancialDetails."));
        }
        Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString); 
        httpNetResponse.Dispose();

        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} cashflow statement items for {1}",dataSetsCashflowStatement.Count,symbol));

        cashflowStatements=CreateCashflowStatements(symbol, dataSetsCashflowStatement); 
        return cashflowStatements;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCashflowStatement encountered an exception {0}",exception.ToString()));
        return null;
      }
    }

    private static HttpNetResponse GetMStarEquityFinancialDetails(String securityId)
    {
      HttpNetResponse httpNetResponse = new HttpNetResponse();
      int maxRetries=10;
      int retry=0;
      int waitBetweenAttemptsMS=500;

      StringBuilder sb=new StringBuilder();
      sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=3.74.0");
      String strRequest=sb.ToString();
      try
      {
        MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
        WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement");
        while(retry<maxRetries && false==httpNetResponse.Success)
        {
          httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
          if(httpNetResponse.Success && !String.IsNullOrEmpty(httpNetResponse.ResponseString))return httpNetResponse;
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarEquityFinancialDetails Attempt {0} of {1}",retry+1,maxRetries));
          try{Thread.Sleep(waitBetweenAttemptsMS);}catch(Exception){;}
          retry++;
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
        return httpNetResponse;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception running request: {0}, Exception:{1}",strRequest,exception.ToString()));
        return httpNetResponse;
      }
    }
     
    private static List<CashflowStatement> CreateCashflowStatements(String symbol,Dictionary<String,MStarDataSet> cashflowDataElements) 
    {
      DateTime today=DateTime.Now;
      List<CashflowStatement> cashflowStatements=new List<CashflowStatement>();
      String currencyCashflow=cashflowDataElements["currency"].Values[0];
      double cashflowStatementMultiplier=double.Parse(cashflowDataElements["multiplier"].Values[0]);
      Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
      String baseCurrency="USD";

      try
      {
        Dictionary<DateTime,Dictionary<String,String>> cashflowDictionaryByDate=CreateDictionaryByDate(cashflowDataElements);
        List<DateTime> cashflowDateKeys=new List<DateTime>(cashflowDictionaryByDate.Keys);
        if(0==cashflowDateKeys.Count)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: No dates in cashflowDataElement."));
          return null;
        }

        foreach(DateTime cashflowDate in cashflowDateKeys)
        {
          Dictionary<String,String> cashflowDatedElements=cashflowDictionaryByDate[cashflowDate];

          CashflowStatement cashflowStatement=new CashflowStatement();
          cashflowStatement.Symbol=symbol;
          cashflowStatement.AsOf=cashflowDate;
          cashflowStatement.Modified=DateTime.Now;
          cashflowStatement.Period=CashflowStatement.PeriodType.Annual;

          if(!baseCurrency.Equals(currencyCashflow))
          {
            if(null==CurrencyConversionDA.GetMaxDateForCurrency(currencyCashflow,baseCurrency,cashflowStatement.AsOf))
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currencyCashflow,cashflowStatement.AsOf.ToShortDateString()));
              CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currencyCashflow,cashflowStatement.AsOf);
              if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
              {
                MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString()));
                return null;
              }
            }
            CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currencyCashflow,baseCurrency,cashflowStatement.AsOf);
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
            conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement);
          }

          double currencyMultiplier=1.00; 
          currencyMultiplier=conversionRates.ContainsKey(cashflowStatement.AsOf)?conversionRates[cashflowStatement.AsOf].UnitsPerSource:currencyMultiplier;

          if(cashflowDatedElements.ContainsKey("Depreciation and Amortization, Non-Cash Adjustment"))
          {
            String value = cashflowDatedElements["Depreciation and Amortization, Non-Cash Adjustment"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.DepreciationAndAmortization=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.DepreciationAndAmortization=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Deferred Taxes, Non-Cash Adjustment"))
          {
            String value = cashflowDatedElements["Deferred Taxes, Non-Cash Adjustment"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.DeferredIncomeTaxes=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.DeferredIncomeTaxes=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Stock-Based Compensation, Non-Cash Adjustment"))
          {
            String value = cashflowDatedElements["Stock-Based Compensation, Non-Cash Adjustment"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.StockBasedCompensation=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.StockBasedCompensation=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Change in Trade/Accounts Receivable"))
          {
            String value = cashflowDatedElements["Change in Trade/Accounts Receivable"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.AccountsReceivable=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.AccountsReceivable=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Change in Accrued Expenses"))
          {
            String value = cashflowDatedElements["Change in Accrued Expenses"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.AccruedLiabilities=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.AccruedLiabilities=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Cash Generated from Operating Activities"))
          {
            String value = cashflowDatedElements["Cash Generated from Operating Activities"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value))
            {
              cashflowStatement.OperatingCashflow=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
            }
            else cashflowStatement.OperatingCashflow=double.NaN;
          }

          if(cashflowDatedElements.ContainsKey("Purchase/Sale and Disposal of Property, Plant and Equipment, Net"))
          {
            String value = cashflowDatedElements["Purchase/Sale and Disposal of Property, Plant and Equipment, Net"];
            if(!"_PO_".Equals(value)&&!"null".Equals(value)&& !double.IsNaN(cashflowStatement.OperatingCashflow))
            {
              cashflowStatement.FreeCashflow=(cashflowStatement.OperatingCashflow+(double.Parse(value)*cashflowStatementMultiplier))*currencyMultiplier;
            }
            else cashflowStatement.FreeCashflow=double.NaN;
          }

          if(cashflowStatement.IsValid())
          {
            cashflowStatements.Add(cashflowStatement);
          }
        }

        return cashflowStatements;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: Exception encountered {0}",exception.ToString()));
        return null;
      }
    }

    private static Dictionary<DateTime,Dictionary<String,String>> CreateDictionaryByDate(Dictionary<String,MStarDataSet> dataSets)
    {
      try
      {
        DateTime today=DateTime.Now;
        Dictionary<DateTime,Dictionary<String,String>> dictionaryByDate=new Dictionary<DateTime,Dictionary<String,String>>();
        MStarDataSet columnDefs=dataSets["columnDefs"];
        List<String> itemKeys=new List<String>(dataSets.Keys);

        itemKeys.RemoveAt(itemKeys.IndexOf("columnDefs"));
        itemKeys.RemoveAt(itemKeys.IndexOf("multiplier"));
        itemKeys.RemoveAt(itemKeys.IndexOf("currency"));
        for(int dateIndex=0;dateIndex<columnDefs.Values.Count;dateIndex++)
        {
          String strDateElement=columnDefs.Values[dateIndex];
          DateTime reportDate=Utility.Epoch;
          if(strDateElement.Length.Equals(4))
          {
            bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
            if(!result)continue;
            if(reportDate.Year.Equals(today.Year))
            {
              reportDate=new DateTime(reportDate.Year,today.Month,1);
            }
            else
            {
              reportDate=new DateTime(reportDate.Year,12,31);
            }
          }
          else
          {
            bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy-MM"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
            if(!result)continue;
          }

          foreach(String itemKey in itemKeys)
          {
            MStarDataSet dataSet=dataSets[itemKey];
            String value=null;
            value=dataSet.Values[dateIndex];
            if(!dictionaryByDate.ContainsKey(reportDate))
            {
              dictionaryByDate.Add(reportDate,new Dictionary<String,String>());
            }
            Dictionary<String,String> datedCollection=dictionaryByDate[reportDate];

              if(!datedCollection.ContainsKey(itemKey))
              {
                datedCollection.Add(itemKey,value);
              }
              else datedCollection[itemKey]=value;
            }
        }
        return dictionaryByDate; 
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateDictionaryByDate: Exception encountered {0}",exception.ToString()));
        return null;
      }
    }

// *****************************************************************************************************************************************************************************
// ************************************************************ C A S H F L O W  S T A T E M E N T  - M O R N I N G S T A R   V 2   E N D   ************************************
// *****************************************************************************************************************************************************************************

// *****************************************************************************************************************************************************************************
// ******************************************************************* C A S H F L O W  S T A T E M E N T  - M O R N I N G S T A R  ********************************************
// *****************************************************************************************************************************************************************************
    public static List<CashflowStatement> GetCashflowStatement_Obsolete(String symbol,CashflowStatement.PeriodType periodType)
    {
      List<CashflowStatement> cashflowStatements=new List<CashflowStatement>();
      DateGenerator dateGenerator=new DateGenerator();
      Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
      DateTime currentDate=DateTime.Now;
      String baseCurrency="USD";
      String nasdaq="XNAS";
      String nyse="XNYS";

      try
      {
        CSVFileExt csvFile=null;
        csvFile=GetCashflowStatementCSV(symbol,periodType,nasdaq);
        if(null==csvFile)csvFile=GetCashflowStatementCSV(symbol,periodType,nyse);
        if(null==csvFile)return null;
        String dateMarker=csvFile.GetRowData(0,0);
        if(null==dateMarker||!dateMarker.StartsWith("Fiscal"))return null;
        String currency=Utility.BetweenString(dateMarker,".",".");
        if(null!=currency)currency=currency.Trim().ToUpper();
        else currency=baseCurrency;     // if we can't find the currency in the header then assume it's USD

// build the list of asof dates for the cashflow records
        List<String> rowData=csvFile.GetRow(0);
        for(int index=0;index<rowData.Count-2;index++)
        {
          CashflowStatement cashflowStatement=new CashflowStatement();
          cashflowStatement.Symbol=symbol;
          cashflowStatement.Period=periodType;
          cashflowStatement.AsOf=dateGenerator.GetNextBusinessDay(Utility.ParseDate(rowData[index+1])); //     Utility.ParseDate(rowData[index+1]);  // dateGenerator.GetNextBusinessDay(Utility.ParseDate(rowData[index+1]));
          cashflowStatement.Modified=currentDate;
          cashflowStatements.Add(cashflowStatement);
          if(!baseCurrency.Equals(currency))
          {
            if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,cashflowStatement.AsOf))
            {
              MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,cashflowStatement.AsOf.ToShortDateString()));
              CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,cashflowStatement.AsOf);
              if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
              {
                MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,cashflowStatement.AsOf.ToShortDateString()));
                return null;
              }
            }
            CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,cashflowStatement.AsOf);
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
            conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement);
          }
        }
        for(int rowIndex=1;rowIndex<csvFile.RowCount;rowIndex++)
        {
          String itemHeading=csvFile.GetRowData(rowIndex,0);
          if(null==itemHeading)continue;
          itemHeading=itemHeading.ToLower();
          if(itemHeading.Equals("Depreciation & amortization".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].DepreciationAndAmortization=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Deferred income taxes".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].DeferredIncomeTaxes=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Stock based compensation".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].StockBasedCompensation=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Accounts receivable".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].AccountsReceivable=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Accounts payable".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].AccountsPayable=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Inventory".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].Inventory=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Accrued liabilities".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].AccruedLiabilities=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Operating cash flow".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].OperatingCashflow=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
          else if(itemHeading.Equals("Free Cash Flow".ToLower()))
          {
            for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
            {
              double multiplier=1.00;
              multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
              cashflowStatements[colIndex-1].FreeCashflow=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
            }
          }
        }
        return cashflowStatements;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
      }
    }
    private static CSVFileExt GetCashflowStatementCSV(String symbol,CashflowStatement.PeriodType periodType,String exchange)
    {
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;

      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        String strReferer;
        sb.Append("http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=").Append(exchange).Append(":");
        sb.Append(symbol).Append("&region=usa&culture=en-US&version=SAL&cur=&reportType=cf&period=");
        sb.Append(periodType.Equals(CashflowStatement.PeriodType.Quarterly) ? 3 : 12).Append("&dataType=A&order=asc&columnYear=5&curYearPart=1st5year&rounding=1&view=raw&r=310540&denominatorView=raw&number=1");
        strRequest = sb.ToString();
        sb=new StringBuilder();
        sb.Append("http://financials.morningstar.com/cash-flow/cf.html?t=").Append(symbol).Append("&region=usa&culture=en-US");
        strReferer=sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestCSV(strRequest,strReferer);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        memoryStream=(MemoryStream)httpNetResponse.ResponseStringToStream();
        CSVFileExt csvFile=new CSVFileExt();
        csvFile.LoadCSVFileNoHeader(new StreamReader(memoryStream),1);
        if(0==csvFile.RowCount)return null;
        return csvFile;
      }
      catch(Exception)
      {
        return null;
      }
      finally
      {
        if (null != memoryStream) memoryStream.Close();
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }

// *****************************************************************************************************************************************************************************
// ******************************************************************* C A S H F L O W  S T A T E M E N T  - M O R N I N G S T A R  E N D   ************************************
// *****************************************************************************************************************************************************************************

// ****************************************************************************************************************************************************************************
// ************************************************************ I N C O M E  S T A T E M E N T  -  N A S D A Q / F I N V I Z **************************************************
// ****************************************************************************************************************************************************************************
    public static List<IncomeStatement> GetIncomeStatement(String symbol,IncomeStatement.PeriodType periodType)
    {
		  List<IncomeStatement> incomeStatements=GetIncomeStatementNASDAQ(symbol,periodType);
			if(null!=incomeStatements && incomeStatements.Count>0)return incomeStatements;
			incomeStatements=GetIncomeStatementFinViz(symbol,periodType);
			return incomeStatements;
    }


//{"currency":"USD","data":{
//"Period End Date":["TTM","1/2/2021","12/28/2019","12/29/2018","12/30/2017","12/31/2016","1/2/2016","1/3/2015"],
//"Total Revenue":["2,930.65","2,513.26","2,959.45","2,722.93","2,335.54","2,267.85","1,826.60","1,636.54"],
//"Cost of Revenue":["1,872.32","1,631.21","1,855.95","1,718.79","1,422.80","1,366.67","1,120.09","995.95"],
//"Selling, General and Administrative":["596.80","531.90","583.81","538.84","468.22","471.64","344.55","320.37"],
//"Gross Profit":["1,058.33","882.05","1,103.50","1,004.14","912.74","901.18","706.50","640.59"],
//"Total Operating Expense":["2,492.86","2,188.83","2,445.40","2,276.97","1,956.93","1,848.83","1,523.99","1,336.11"],

//"Research and Development":["","35.30","41.20","35.30","29.10","26.30","22.40","22.60"],
//"Net Income":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
//"Income Avail. to Common Incl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],

//"Interest Income Net":["-71.44","-81.52","-83.51","-56.21","-23.64","-21.98","-10.17","-11.99"],
//"Period Length":["12 Months","53 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","53 Weeks"],
//"Unusual Expense/Income":["23.74","25.72","5.64","19.33","65.91","10.52","59.35","19.78"],
//"Operating Income":["437.79","324.43","514.04","445.97","378.61","419.02","302.60","300.43"],
//"Other Income Net":["31.16","25.14","32.09","33.76","28.55","24.27","-11.27","-7.65"],
//"Net Income Before Taxes":["397.51","268.06","462.62","423.51","383.53","421.31","281.17","280.79"],
//"Provision for Income Taxes":["75.31","60.76","110.38","106.36","85.40","137.09","89.56","87.48"],
//"Income Avail. to Common Excl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
//"Diluted Average Shares":["55.71","55.14","55.66","55.60","56.72","57.09","56.97","56.78"],
//"Diluted EPS Excl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"],
//"Diluted EPS Incl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"]}}
		public static List<IncomeStatement> GetIncomeStatementFinViz(String symbol,IncomeStatement.PeriodType periodType)
		{
			HttpNetResponse httpNetResponse=null;
			String baseCurrency="USD";
			try
			{
				List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
        Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();

				StringBuilder sb = new StringBuilder();
				String strRequest;
				sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
				if(IncomeStatement.PeriodType.Annual.Equals(periodType))sb.Append("&s=IA");
				else sb.Append("&s=IQ");
				strRequest = sb.ToString();
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz {0}",strRequest));
				httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
				if(!httpNetResponse.Success)
				{
					MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
					return null;
				}
				String currency="USD";
        JObject json=JObject.Parse(httpNetResponse.ResponseString);
        JToken jsonToken=null;

        if(null!=(jsonToken=json.SelectToken("currency")))	
				{
				  currency=jsonToken.ToObject<String>();
				}

        if(null==(jsonToken=json.SelectToken("data")))	
				{
				  MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
				  return null;  
				}

				List<String> strDateItems=null;
				List<String> strTotalRevenueItems=null;
				List<String> strCostOfRevenueItems=null;
				List<String> strSGAItems=null;
				List<String> strGrossProfitItems=null;
				List<String> strTotalOperatingExpense=null;
				List<String> strResearchAndDevelopmentItems=null;
				List<String> strIncomeAvailableToCommonShareHolders=null;

				JToken token=jsonToken["Period End Date"];
				if(null==token)
				{
				  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No IncomeStatemnt dates returned for {0}",symbol));
					return null;
				}
				strDateItems=token.ToObject<List<String>>();

				token=jsonToken["Total Revenue"];
				if(null!=token)strTotalRevenueItems=token.ToObject<List<String>>();

				token=jsonToken["Cost of Revenue"];
				if(null!=token)strCostOfRevenueItems=token.ToObject<List<String>>();

				token=jsonToken["Selling, General and Administrative"];
				if(null!=token)strSGAItems=token.ToObject<List<String>>();

				token=jsonToken["Gross Profit"];
				if(null!=token)strGrossProfitItems=token.ToObject<List<String>>();

				token=jsonToken["Total Operating Expense"];
				if(null!=token)strTotalOperatingExpense=token.ToObject<List<String>>();

				token=jsonToken["Research and Development"];
				if(null!=token)strResearchAndDevelopmentItems=token.ToObject<List<String>>();

				token=jsonToken["Income Avail. to Common Incl. Extraord."];
				if(null!=token)strIncomeAvailableToCommonShareHolders=token.ToObject<List<String>>();

				for(int index=0;index<strDateItems.Count;index++)
				{
				  String strDateItem=strDateItems[index];
					if("TTM".Equals(strDateItem,StringComparison.CurrentCultureIgnoreCase))continue;
				  IncomeStatement incomeStatement=new IncomeStatement();
					incomeStatement.Symbol=symbol;
					incomeStatement.Period=periodType;
					incomeStatement.AsOf=Utility.ParseDate(strDateItems[index]);
					incomeStatement.Modified=DateTime.Now;
          double multiplier=1.00;

// perform currency conversion
					if(!baseCurrency.Equals(currency))
					{
						if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,incomeStatement.AsOf))
						{
							MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,incomeStatement.AsOf.ToShortDateString()));
							CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,incomeStatement.AsOf);
							if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
							{
								MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString()));
								continue;
							}
						}
						CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,incomeStatement.AsOf);
						MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for IncomeStatement : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
						conversionRates.Add(incomeStatement.AsOf,currencyConversionElement);
					}
          multiplier=conversionRates.ContainsKey(incomeStatement.AsOf)?conversionRates[incomeStatement.AsOf].UnitsPerSource:multiplier;

					if(null!=strTotalRevenueItems && strTotalRevenueItems.Count>index)incomeStatement.TotalRevenue=FeedParser.ParseValue(strTotalRevenueItems[index])*1000000.00*multiplier;
					if(null!=strCostOfRevenueItems && strCostOfRevenueItems.Count>index)incomeStatement.CostOfRevenue=FeedParser.ParseValue(strCostOfRevenueItems[index])*1000000.00*multiplier;
					if(null!=strSGAItems && strSGAItems.Count>index)incomeStatement.SGA=FeedParser.ParseValue(strSGAItems[index])*1000000.00*multiplier;
					if(null!=strGrossProfitItems && strGrossProfitItems.Count>index)incomeStatement.GrossProfit=FeedParser.ParseValue(strGrossProfitItems[index])*1000000.00*multiplier;
					if(null!=strTotalOperatingExpense && strTotalOperatingExpense.Count>index)incomeStatement.OperatingExpenses=FeedParser.ParseValue(strTotalOperatingExpense[index])*1000000.00*multiplier;
					if(null!=strResearchAndDevelopmentItems && strResearchAndDevelopmentItems.Count>index)incomeStatement.ResearchAndDevelopment=FeedParser.ParseValue(strResearchAndDevelopmentItems[index])*1000000.00*multiplier;
					if(null!=strIncomeAvailableToCommonShareHolders && strIncomeAvailableToCommonShareHolders.Count>index)incomeStatement.NetIncomeApplicableToCommonShares=FeedParser.ParseValue(strIncomeAvailableToCommonShareHolders[index])*1000000.00*multiplier;
					incomeStatements.Add(incomeStatement);
				}
				return incomeStatements;
			}
			catch(Exception exception)
			{
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
				return null;
			}
		}
    public static List<IncomeStatement> GetIncomeStatementNASDAQ(String symbol,IncomeStatement.PeriodType periodType)
    {
      List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      double multiplier=1000.00;
      try
      {
        StringBuilder sb=new StringBuilder();
        String strRequest;
        sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
        if(periodType.Equals(IncomeStatement.PeriodType.Annual))sb.Append(""); 
        else sb.Append("?frequency=2");

        strRequest=sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success) return incomeStatements;
        byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        JObject json=JObject.Parse(httpNetResponse.ResponseString);
        JToken jsonToken=null;
        if(null==(jsonToken=json.SelectToken("data.incomeStatementTable.headers")))
        {
         String reason="unknown";
         jsonToken=json.SelectToken("message");
         if(null!=jsonToken)reason=jsonToken.ToString();
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
          return incomeStatements;
        }
        JObject headerObject=(JObject)jsonToken;
        int elements=headerObject.Count-1;
        String headerElement=headerObject["value1"].ToString();
        if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
          return incomeStatements;
        }

        for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
        {
          IncomeStatement incomeStatement=new IncomeStatement();
          String itemIndex="value"+headerIndex;
          jsonToken=json.SelectToken("data.incomeStatementTable.headers."+itemIndex);
          if(null==jsonToken)continue;
          String strDate=jsonToken.ToString();
					if(string.IsNullOrEmpty(strDate))continue;
          incomeStatement.AsOf=Utility.ParseDate(strDate);
          incomeStatement.Symbol=symbol.ToUpper();
          incomeStatement.Period=periodType;
          incomeStatements.Add(incomeStatement);
        }

        jsonToken=json.SelectToken("data.incomeStatementTable.rows");
        if(null==jsonToken)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
          return incomeStatements;
        }

        JArray collection=(JArray)jsonToken;
        for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
        {
          JObject rowItem=(JObject)collection[rowIndex];
          String strHeader=rowItem.SelectToken("value1").ToString();

          if(null==strHeader)continue;
          if("Total Revenue".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].TotalRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Cost of Revenue".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].CostOfRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Gross Profit".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].GrossProfit=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Operating Expenses".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Research and Development".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].ResearchAndDevelopment=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Sales, General and Admin.".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].SGA=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Non-Recurring Items".Equals(strHeader))
          {
            continue;
          }
          else if("Other Operating Items".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Operating Income".Equals(strHeader))
          {
            continue;
          }
          else if("Add'l income/expense items".Equals(strHeader))
          {
            continue;
          }
          else if("Earnings Before Interest and Tax".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].EBIT=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Interest Expense".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].InterestExpense=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Earnings Before Tax".Equals(strHeader))
          {
            continue;
          }
          else if("Income Tax".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].IncomeTaxExpense=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Minority Interest".Equals(strHeader))
          {
            continue;
          }
          else if("Equity Earnings/Loss Unconsolidated Subsidiary".Equals(strHeader))
          {
            continue;
          }
          else if("Net Income-Cont. Operations".Equals(strHeader))
          {
            continue;
          }
          else if("Net Income".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].NetIncome=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Net Income Applicable to Common Shareholders".Equals(strHeader))
          {
            for(int index=0;index<incomeStatements.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              incomeStatements[index].NetIncomeApplicableToCommonShares=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
        }
        return incomeStatements;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse) httpNetResponse.Dispose();
        if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
      }
    }

// *******************************************************************************************************************************************************************
// ************************************************************B A L A N C E  S H E E T - N A S D A Q / F I N V I Z **************************************************
// *******************************************************************************************************************************************************************
    public static List<BalanceSheet> GetBalanceSheet(String symbol,BalanceSheet.PeriodType periodType)
    {
		  List<BalanceSheet> balanceSheet=null;
      balanceSheet=GetBalanceSheetNASDAQ(symbol,periodType);
			if(null!=balanceSheet && balanceSheet.Count>0)return balanceSheet;
			balanceSheet=GetBalanceSheetFinViz(symbol,periodType);
			return balanceSheet;
    }
//https://finviz.com/api/statement.ashx?t=aapl&s=BA
//https://finviz.com/api/statement.ashx?t=aapl&s=BQ
//{"currency":"USD",
//"data":{
//"Period End Date":["9/26/2020","9/28/2019","9/29/2018","9/30/2017","9/24/2016","9/26/2015","9/27/2014","9/28/2013"],
//"Cash and Equivalents":["20,243.00","36,640.00","14,338.00","12,307.00","11,883.00","9,731.00","3,612.00","5,554.00"],
//"Short Term Investments":["52,927.00","51,713.00","40,388.00","53,892.00","46,671.00","20,481.00","11,233.00","26,287.00"],
//"Cash and Short Term Investments":["90,943.00","100,557.00","66,301.00","74,181.00","67,155.00","41,601.00","25,077.00","40,546.00"],
//"Accounts Receivable - Trade, Net":["16,120.00","22,926.00","23,186.00","17,874.00","15,754.00","16,849.00","17,460.00","13,102.00"],
//"Total Receivables, Net":["37,445.00","45,804.00","48,995.00","35,673.00","29,299.00","30,343.00","27,219.00","20,641.00"],
//"Total Inventory":["4,061.00","4,106.00","3,956.00","4,855.00","2,132.00","2,349.00","2,111.00","1,764.00"],
//"Total Current Assets":["143,713.00","162,819.00","131,339.00","128,645.00","106,869.00","89,378.00","68,531.00","73,286.00"],
//"Property, Plant And Equipment - Gross":["112,096.00","95,957.00","90,403.00","75,076.00","61,245.00","49,257.00","39,015.00","28,519.00"],
//"Property, Plant And Equipment - Net":["45,336.00","37,378.00","41,304.00","33,783.00","27,010.00","22,471.00","20,624.00","16,597.00"],
//"Goodwill, Net":["","","","","5,414.00","5,116.00","4,616.00","1,577.00"],
//"Intangibles, Net":["","","","","3,206.00","3,893.00","4,142.00","4,179.00"],
//"Long Term Investments":["100,887.00","105,341.00","170,799.00","194,714.00","170,430.00","164,065.00","130,162.00","106,215.00"],
//"Other Long Term Assets":["33,952.00","32,978.00","22,283.00","18,177.00","8,757.00","5,422.00","3,764.00","5,146.00"],
//"Total Assets":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
//"Accounts Payable":["42,296.00","46,236.00","55,888.00","44,242.00","37,294.00","35,490.00","30,196.00","22,367.00"],
//"Accrued Expenses":["1,436.00","","","","20,951.00","24,169.00","7,689.00","4,782.00"],
//"Notes Payable/Short Term Debt":["4,996.00","5,980.00","11,964.00","11,977.00","8,105.00","8,499.00","6,308.00","0.00"],
//"Current Port. of LT Debt/Capital Leases":["8,797.00","10,260.00","8,784.00","6,496.00","3,500.00","2,500.00","",""],
//"Other Current liabilities":["47,867.00","43,242.00","39,293.00","38,099.00","9,156.00","9,952.00","19,255.00","16,509.00"],
//"Total Current Liabilities":["105,392.00","105,718.00","115,929.00","100,814.00","79,006.00","80,610.00","63,448.00","43,658.00"],
//"Long Term Debt":["98,667.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
//"Capital Lease Obligations":["637.00","","","","","","",""],
//"Total Long Term Debt":["99,304.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
//"Total Debt":["113,097.00","108,047.00","114,483.00","115,680.00","87,032.00","64,328.00","35,295.00","16,960.00"],
//"Other Liabilities":["53,853.00","50,503.00","48,914.00","11,747.00","12,985.00","12,989.00","7,598.00","6,344.00"],
//"Total Liabilities":["258,549.00","248,028.00","258,578.00","241,272.00","193,437.00","170,990.00","120,292.00","83,451.00"],
//"Common Stock":["50,779.00","45,174.00","40,201.00","35,867.00","31,251.00","27,416.00","23,313.00","19,764.00"],
//"Additional Paid-In Capital":["","","","","","","",""],
//"Retained Earnings":["14,966.00","45,898.00","70,400.00","98,330.00","96,364.00","92,284.00","87,152.00","104,256.00"],
//"Unrealized Gain (Loss)":["1,846.00","707.00","-3,209.00","328.00","1,174.00","-653.00","-242.00","-105.00"],
//"Other Equity":["-2,252.00","-1,291.00","-245.00","-478.00","-540.00","308.00","1,324.00","-366.00"],
//"Total Equity":["65,339.00","90,488.00","107,147.00","134,047.00","128,249.00","119,355.00","111,547.00","123,549.00"],
//"Total Liabilities and Equity":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
//"Total Common Shares Outstanding":["16,976.76","17,772.94","19,019.94","20,504.80","21,344.66","22,315.01","23,464.64","25,177.46"],
//"Full-Time Employees":["147,000","137,000","132,000","123,000","116,000","110,000","92,600","80,300"],
//"Number of Common Shareholders":["22,797","23,233","23,712","25,333","25,641","25,924","26,112","24,710"]}}
		public static List<BalanceSheet> GetBalanceSheetFinViz(String symbol,BalanceSheet.PeriodType periodType)
		{
			HttpNetResponse httpNetResponse=null;
			String baseCurrency="USD";
			try
			{
				List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
        Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();

				StringBuilder sb = new StringBuilder();
				String strRequest;
				sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
				if(BalanceSheet.PeriodType.Annual.Equals(periodType))sb.Append("&s=BA");
				else sb.Append("&s=BQ");
				strRequest = sb.ToString();
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz {0}",strRequest));
				httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
				if(!httpNetResponse.Success)
				{
					MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
					return null;
				}
				String currency="USD";
        JObject json=JObject.Parse(httpNetResponse.ResponseString);
        JToken jsonToken=null;

        if(null!=(jsonToken=json.SelectToken("currency")))	
				{
				  currency=jsonToken.ToObject<String>();
				}

        if(null==(jsonToken=json.SelectToken("data")))	
				{
				  MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
				  return null;  
				}

				List<String> strDateItems=null;
				List<String> strLongTermDebtItems=null;
				List<String> strTotalCurrentAssetItems=null;
				List<String> strTotalLiabilityItems=null;
				List<String> strTotalAssetItems=null;
				List<String> strGoodwillItems=null;
				List<String> strPPEItems=null;
				List<String> strCashAndEquivalentsItems=null;
				List<String> strTotalInventoryItems=null;
				List<String> strIntangiblesItems=null;
				List<String> strTotalCurrentLiabilityItems=null;
				List<String> strOtherLiabilityItems=null;
				List<String> strTotalEquityItems=null;

				JToken token=jsonToken["Period End Date"];
				if(null==token)
				{
				  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No BalanceSheet dates returned for {0}",symbol));
					return null;
				}
				strDateItems=token.ToObject<List<String>>();

        token=jsonToken["Long Term Debt"];
				if(null!=token)strLongTermDebtItems=token.ToObject<List<String>>();

				token=jsonToken["Total Current Assets"];
				if(null!=token)strTotalCurrentAssetItems=token.ToObject<List<String>>();

				token=jsonToken["Total Liabilities"];
				if(null!=token)strTotalLiabilityItems=token.ToObject<List<String>>();

				token=jsonToken["Total Assets"];
				if(null!=token)strTotalAssetItems=token.ToObject<List<String>>();

				token=jsonToken["Goodwill, Net"];
				if(null!=token)strGoodwillItems=token.ToObject<List<String>>();

				token=jsonToken["Property, Plant And Equipment - Net"];
				if(null!=token)strPPEItems=token.ToObject<List<String>>();

				token=jsonToken["Cash and Equivalents"];
				if(null!=token)strCashAndEquivalentsItems=token.ToObject<List<String>>();

				token=jsonToken["Total Inventory"];
				if(null!=token)strTotalInventoryItems=token.ToObject<List<String>>();

				token=jsonToken["Intangibles, Net"];
				if(null!=token)strIntangiblesItems=token.ToObject<List<String>>();

				token=jsonToken["Total Current Liabilities"];
				if(null!=token)strTotalCurrentLiabilityItems=token.ToObject<List<String>>();

				token=jsonToken["Other Liabilities"];
				if(null!=token)strOtherLiabilityItems=token.ToObject<List<String>>();

				token=jsonToken["Total Equity"];
				if(null!=token)strTotalEquityItems=token.ToObject<List<String>>();

				for(int index=0;index<strDateItems.Count;index++)
				{
				  BalanceSheet balanceSheet=new BalanceSheet();

					balanceSheet.Symbol=symbol;
					balanceSheet.Period=periodType;
					balanceSheet.AsOf=Utility.ParseDate(strDateItems[index]);
					balanceSheet.Modified=DateTime.Now;
          double multiplier=1.00;

// perform currency conversion
					if(!baseCurrency.Equals(currency))
					{
						if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,balanceSheet.AsOf))
						{
							MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,balanceSheet.AsOf.ToShortDateString()));
							CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,balanceSheet.AsOf);
							if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
							{
								MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString()));
								continue;
							}
						}
						CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,balanceSheet.AsOf);
						MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for BalanceSheet : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
						conversionRates.Add(balanceSheet.AsOf,currencyConversionElement);
					}
          multiplier=conversionRates.ContainsKey(balanceSheet.AsOf)?conversionRates[balanceSheet.AsOf].UnitsPerSource:multiplier;

					if(null!=strLongTermDebtItems && strLongTermDebtItems.Count>index)balanceSheet.LongTermDebt=FeedParser.ParseValue(strLongTermDebtItems[index])*1000000.00*multiplier;
					if(null!=strTotalCurrentAssetItems && strTotalCurrentAssetItems.Count>index)balanceSheet.TotalCurrentAssets=FeedParser.ParseValue(strTotalCurrentAssetItems[index])*1000000.00*multiplier;
					if(null!=strTotalLiabilityItems && strTotalLiabilityItems.Count>index)balanceSheet.TotalLiabilities=FeedParser.ParseValue(strTotalLiabilityItems[index])*1000000.00*multiplier;
					if(null!=strTotalAssetItems && strTotalAssetItems.Count>index)balanceSheet.TotalAssets=FeedParser.ParseValue(strTotalAssetItems[index])*1000000.00*multiplier;
					if(null!=strGoodwillItems && strGoodwillItems.Count>index)balanceSheet.Goodwill=FeedParser.ParseValue(strGoodwillItems[index])*1000000.00*multiplier;
					if(null!=strPPEItems && strPPEItems.Count>index)balanceSheet.PropertyPlantAndEquipment=FeedParser.ParseValue(strPPEItems[index])*1000000.00*multiplier;
					if(null!=strCashAndEquivalentsItems && strCashAndEquivalentsItems.Count>index)balanceSheet.CashAndCashEquivalents=FeedParser.ParseValue(strCashAndEquivalentsItems[index])*1000000.00*multiplier;
					if(null!=strTotalInventoryItems && strTotalInventoryItems.Count>index)balanceSheet.Inventory=FeedParser.ParseValue(strTotalInventoryItems[index])*1000000.00*multiplier;
					if(null!=strIntangiblesItems && strIntangiblesItems.Count>index)balanceSheet.IntangibleAssets=FeedParser.ParseValue(strIntangiblesItems[index])*1000000.00*multiplier;
					if(null!=strTotalCurrentLiabilityItems && strTotalCurrentLiabilityItems.Count>index)balanceSheet.TotalCurrentLiabilities=FeedParser.ParseValue(strTotalCurrentLiabilityItems[index])*1000000.00*multiplier;
					if(null!=strOtherLiabilityItems && strOtherLiabilityItems.Count>index)balanceSheet.OtherLiabilities=FeedParser.ParseValue(strOtherLiabilityItems[index])*1000000.00*multiplier;
					if(null!=strTotalEquityItems && strTotalEquityItems.Count>index)balanceSheet.TotalStockHolderEquity=FeedParser.ParseValue(strTotalEquityItems[index])*1000000.00*multiplier;
					balanceSheets.Add(balanceSheet);
				}
				return balanceSheets;
			}
			catch(Exception exception)
			{
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
				return null;
			}
		}
    public static List<BalanceSheet> GetBalanceSheetNASDAQ(String symbol,BalanceSheet.PeriodType periodType)
    {
      List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      double multiplier=1000.00;
      try
      {
        StringBuilder sb=new StringBuilder();
        String strRequest;
        sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
        if(periodType.Equals(BalanceSheet.PeriodType.Annual)) sb.Append("?frequency=1");
        else sb.Append("?frequency=2");

        strRequest=sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success) return balanceSheets;
        byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        JObject json=JObject.Parse(httpNetResponse.ResponseString);
        JToken jsonToken=null;
        if(null==(jsonToken=json.SelectToken("data.balanceSheetTable.headers")))
        {
          String reason="unknown";
          jsonToken=json.SelectToken("message");
          if(null!=jsonToken) reason=jsonToken.ToString();
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
          return balanceSheets;
        }
        JObject headerObject=(JObject)jsonToken;
        int elements=headerObject.Count-1;
        String headerElement=headerObject["value1"].ToString();
        if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
          return balanceSheets;
        }

        for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
        {
          BalanceSheet balanceSheet =new BalanceSheet();
          String itemIndex="value"+headerIndex;
          jsonToken=json.SelectToken("data.balanceSheetTable.headers."+itemIndex);
          if(null==jsonToken) continue;
          String strDate=jsonToken.ToString();
					if(string.IsNullOrEmpty(strDate))continue;
          balanceSheet.AsOf=Utility.ParseDate(strDate);
          balanceSheet.Symbol=symbol.ToUpper();
          balanceSheet.Period=periodType;
          balanceSheets.Add(balanceSheet);
        }

        jsonToken=json.SelectToken("data.balanceSheetTable.rows");
        if(null==jsonToken)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
          return balanceSheets;
        }

        JArray collection=(JArray)jsonToken;
        for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
        {
          JObject rowItem=(JObject)collection[rowIndex];
          String strHeader=rowItem.SelectToken("value1").ToString();

          if(null==strHeader) continue;
          if("Current Assets".Equals(strHeader))
          {
            continue;
          }
          else if("Cash and Cash Equivalents".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].CashAndCashEquivalents=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Short-Term Investments".Equals(strHeader))
          {
            continue;
          }
          else if("Net Receivables".Equals(strHeader))
          {
            continue;
          }
          else if("Inventory".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].Inventory=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Other Current Assets".Equals(strHeader))
          {
            continue;
          }
          else if("Total Current Assets".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].TotalCurrentAssets=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Long-Term Assets".Equals(strHeader))
          {
            continue;
          }
          else if("Long-Term Investments".Equals(strHeader))
          {
            continue;
          }
          else if("Fixed Assets".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].PropertyPlantAndEquipment=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Goodwill".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].Goodwill=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Intangible Assets".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].IntangibleAssets=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Other Assets".Equals(strHeader))
          {
            continue;
          }
          else if("Deferred Asset Charges".Equals(strHeader))
          {
            continue;
          }
          else if("Total Assets".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].TotalAssets=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Current Liabilities".Equals(strHeader))
          {
            continue;
          }
          else if("Accounts Payable".Equals(strHeader))
          {
            continue;
          }
          else if("Short-Term Debt / Current Portion of Long-Term Debt".Equals(strHeader))
          {
            continue;
          }
          else if("Other Current Liabilities".Equals(strHeader))
          {
            continue;
          }
          else if("Total Current Liabilities".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].TotalCurrentLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Long-Term Debt".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].LongTermDebt=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Other Liabilities".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].OtherLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Deferred Liability Charges".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].DeferredLongTermLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Misc. Stocks".Equals(strHeader))
          {
            continue;
          }
          else if("Minority Interest".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].MinorityInterest=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Total Liabilities".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].TotalLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Stock Holders Equity".Equals(strHeader))
          {
            for(int index=0;index<balanceSheets.Count;index++)
            {
              JToken element=rowItem.SelectToken("value"+(index+2));
              balanceSheets[index].TotalStockHolderEquity=Utility.ParseCurrency(element.ToString())*multiplier;
            }
          }
          else if("Total Equity".Equals(strHeader))
          {
            continue;
          }
          else if("Common Stocks".Equals(strHeader))
          {
            continue;
          }
          else if("Capital Surplus".Equals(strHeader))
          {
            continue;
          }
          else if("Retained Earnings".Equals(strHeader))
          {
            continue;
          }
          else if("Treasury Stock".Equals(strHeader))
          {
            continue;
          }
          else if("Other Equity".Equals(strHeader))
          {
            continue;
          }
          else if("Total Equity".Equals(strHeader))
          {
            continue;
          }
          else if("Total Liabilities & Equity".Equals(strHeader)||"Total Liabilities &amp; Equity".Equals(strHeader))
          {
            continue;
          }
        }
        return balanceSheets;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetNASDAQ Symbol:{0} Exception:{1}",symbol,exception.ToString()));
        return null;
      }
      finally
      {
        if(null!=httpNetResponse) httpNetResponse.Dispose();
        if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
      }
    }

// *****************************************************************************************************************************************************************************
// **************************************************************************** C U R R E N C Y  C O N V E R S I O N   ****************************************************
// *****************************************************************************************************************************************************************************
    public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate)
    {
      HttpNetResponse httpNetResponse=null;
      CurrencyConversionCollection currencyConversionElements=new CurrencyConversionCollection();

      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sourceCurrency=sourceCurrency.ToUpper();
        sb.Append("https://www.xe.com/currencytables/?from=").Append(sourceCurrency).Append("&date=").Append(Utility.DateTimeToStringYYYYHMMHDD(conversionDate)).Append("#table-section");
        strRequest = sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        int itemIndex=0;
        List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);
        if(!Sections.FindInSections(sections,"Currency Table:",0,ref itemIndex,false))return null;
        if(!Sections.FindInSections(sections,"Currency",itemIndex,ref itemIndex))return null;
        itemIndex+=20;

        while(true)
        {
          CurrencyConversionElement currencyConversionElement=new CurrencyConversionElement();
          currencyConversionElement.AsOf=conversionDate;
          currencyConversionElement.SourceCurrency=sourceCurrency;
          currencyConversionElement.DestinationCurrency=sections[itemIndex];
          itemIndex+=3;
          currencyConversionElement.DestinationCurrencyName=sections[itemIndex];
          itemIndex+=2;
          currencyConversionElement.UnitsPerSource=FeedParser.ParseValue(sections[itemIndex]);
          itemIndex+=2;
          currencyConversionElement.SourcePerUnit=FeedParser.ParseValue(sections[itemIndex]);
          currencyConversionElements.Add(currencyConversionElement);
          itemIndex+=5;
          if(itemIndex>=sections.Count || null==sections[itemIndex] || "".Equals(sections[itemIndex]) || sections[itemIndex].Length<3)break;
        }
        return currencyConversionElements;
      }
      catch(Exception)
      {
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// ***************************************************************************************************************************************************************************************
// **************************************************************************  F U N D A M E N T A L  - F I N V I S *********************************************************
// ***************************************************************************************************************************************************************************************
    public static Fundamental GetFundamentalFinViz(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      try
      {
        Fundamental fundamental = new Fundamental();
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower());
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
        WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz");
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Get Fundamental for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"js-snapshot-table snapshot-table2 screener_snapshot-table-body\"]");
        if(null==nodeCollection || nodeCollection.Count<1)return null;
        fundamental.AsOf=DateTime.Now;
        fundamental.Symbol=symbol;
        fundamental.Source = "FINVIZ";
        for(int index=0;index<nodeCollection[0].ChildNodes.Count;index++)
        {
          HtmlNodeCollection dataCollection = nodeCollection[0].ChildNodes[index].SelectNodes(".//td");
          if(null==dataCollection)continue;
          for(int dataIndex=0;dataIndex<dataCollection.Count;dataIndex++)
          {
            String header=dataCollection[dataIndex].InnerText;
            if(header.Equals("Market Cap"))
            {
              fundamental.MarketCap=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("P/E"))
            {
              fundamental.PE=fundamental.TrailingPE=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("EPS (ttm)"))
            {
              fundamental.EPS=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("Shs Outstand"))
            {
              fundamental.SharesOutstanding=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("Income"))
            {
              fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("PEG"))
            {
              fundamental.PEG=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("Book/sh"))
            {
              fundamental.BookValuePerShare=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("ROA"))
            {
              fundamental.ReturnOnAssets=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }
            else if(header.Equals("ROE"))
            {
              fundamental.ReturnOnEquity=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }   
            else if(header.Equals("52W Range"))
            {
              String strLowHigh=dataCollection[dataIndex+1].InnerText;
              if(null==strLowHigh)continue;
              String[] items=strLowHigh.Split('-');
              if(null==items||2!=items.Length)continue;
              items[0]=items[0].Trim();
              items[1]=items[1].Trim();
              fundamental.Low52=FeedParser.ParseValue(items[0]);
              fundamental.High52=FeedParser.ParseValue(items[1]);
              dataIndex++;
              continue;
            }     
            else if(header.Equals("Beta"))
            {
              fundamental.Beta=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            } 
            else if(header.Equals("Earnings"))
            {
              String earnings=dataCollection[dataIndex+1].InnerText;
              if(null==earnings)continue;
              String[] items=earnings.Split(' ');
              if(null==items||3!=items.Length)continue;
              items[0]=items[0].Trim();
              items[1]=items[1].Trim();
              fundamental.NextEarningsDate=Utility.ParseDate(items[0]+" "+items[1]+" "+DateTime.Now.Year.ToString());
              dataIndex++;
              continue;
            }   
            else if(header.Equals("Avg Volume"))
            {
              fundamental.Volume=(long)FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
              dataIndex++;
              continue;
            }              
          }
        } // for all child nodes
        fundamental.Equity=fundamental.BookValuePerShare*fundamental.SharesOutstanding;  

        IncomeStatement incomeStatement=null;
        List<IncomeStatement> incomeStatements=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
        if(null!=incomeStatements&&0!=incomeStatements.Count) incomeStatement=incomeStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault(); 

        CashflowStatement cashflowStatement=null;
        List<CashflowStatement> cashflowStatements=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
        if(null != cashflowStatements && 0!=cashflowStatements.Count)
        {
          cashflowStatement = cashflowStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
        }

        BalanceSheet balanceSheet=null;
        List<BalanceSheet> balanceSheets=GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
        if(null != balanceSheets && 0!=balanceSheets.Count)
        {
          balanceSheet = balanceSheets.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
        }

        if(null!=cashflowStatement)
        {
          fundamental.OperatingCashflow=cashflowStatement.OperatingCashflow;
          fundamental.LeveragedFreeCashflow=cashflowStatement.FreeCashflow;
          fundamental.EBITDA=fundamental.EBIT+cashflowStatement.DepreciationAndAmortization;
        }
        else
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO CASHFLOW STATEMENT *************",symbol));
          fundamental.OperatingCashflow=double.NaN;
          fundamental.LeveragedFreeCashflow=double.NaN;
          fundamental.EBITDA=double.NaN;
        }

        if(null!=incomeStatement)
        {
          fundamental.GrossProfit=incomeStatement.GrossProfit;
          fundamental.Revenue=incomeStatement.TotalRevenue;
          fundamental.EBIT=incomeStatement.EBIT;
        }                                                                                         
        else
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO INCOME STATEMENT *************",symbol));
          fundamental.GrossProfit=double.NaN;
          fundamental.Revenue=double.NaN;
          fundamental.EBIT=double.NaN;
        }

        if(null!=balanceSheet)
        {
          fundamental.TotalCash=balanceSheet.CashAndCashEquivalents;
          fundamental.TotalDebt=balanceSheet.TotalLiabilities;
        }
        else
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO BALANCESHEET *************",symbol));
          fundamental.TotalCash=double.NaN;
          fundamental.TotalDebt=double.NaN;
        }

        if(!double.IsNaN(fundamental.MarketCap)&&!double.IsNaN(fundamental.TotalDebt)&&!double.IsNaN(fundamental.TotalCash)) fundamental.EnterpriseValue=fundamental.MarketCap+fundamental.TotalDebt-fundamental.TotalCash;
        else fundamental.EnterpriseValue=double.NaN;
        if(!Double.IsNaN(fundamental.SharesOutstanding)&&0!=fundamental.SharesOutstanding)fundamental.RevenuePerShare=fundamental.Revenue/fundamental.SharesOutstanding;
        else fundamental.RevenuePerShare=double.NaN;
        if(null!=balanceSheet&&!double.IsNaN(balanceSheet.TotalStockHolderEquity)&&!double.IsNaN(fundamental.TotalDebt)&&0!=fundamental.TotalDebt)fundamental.DebtToEquity=fundamental.TotalDebt/balanceSheet.TotalStockHolderEquity;
        else fundamental.DebtToEquity=double.NaN;
        if(double.IsNaN(fundamental.DebtToEquity)&&!double.IsNaN(fundamental.Equity)&&0.00!=fundamental.Equity)fundamental.DebtToEquity=fundamental.TotalDebt/fundamental.Equity;
        return fundamental;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}.",exception.ToString()));
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
        if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
      }
    }

// ***************************************************************************************************************************************************************************************
// **************************************************************************  F U N D A M E N T A L  - Y A H O O  F I N A N C E *********************************************************
// ***************************************************************************************************************************************************************************************
    public static Fundamental GetFundamental(String symbol)
    {
      Fundamental fundamental=GetFundamentalEx(symbol);
      if(null==fundamental)return fundamental;
      if(fundamental.GetLoad()<80)
      {
        Fundamental fundamental2 = GetFundamentalEx(symbol);
        if(null!=fundamental2)fundamental.MergeFrom(fundamental2);
      }
      return fundamental;
    }
    private static Fundamental GetFundamentalEx(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      int TIMEOUT_MS=1000*30;
      try
      {
        Fundamental fundamental = new Fundamental();
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("?p=").Append(symbol);
        strRequest = sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
        WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx");
//        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
//        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy);     // user agents refreshed in this version
        //MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS, webProxy);     // user agents refreshed in this version
        if(!httpNetResponse.Success)
        {
          sb=new StringBuilder();
          sb.Append("http://finance.yahoo.com/q?s=").Append(symbol).Append("&ql=1");
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FAILED!, TRYING {1}",strRequest,sb.ToString()));
          strRequest = sb.ToString();
          //MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
          httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
          if(!httpNetResponse.Success)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FINAL FAILURE.",strRequest));
            return null;
          }
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} SUCCESS.",strRequest));
        fundamental.Symbol=symbol;
        fundamental.AsOf=DateTime.Now;
        fundamental.Source = "YAHOO";
        fundamental.Beta=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Beta<"));
        fundamental.NextEarningsDate=FeedParser.ParseValueDateTimeMonthFormat(Sections.LocateItemAcceptAnyText(httpNetResponse.ResponseString,">Earnings Date<"));
        String strText=Sections.LocateFirstItem(httpNetResponse.ResponseString,">52 Week Range<");
        if(null!=strText)
        {
          String[] strArray = strText.Split('-');
          fundamental.Low52 = FeedParser.ParseValue(strArray[0].Trim());
          fundamental.High52 = FeedParser.ParseValue(strArray[1].Trim());
        }
        fundamental.Volume=FeedParser.ParseValueLong(Sections.LocateItem(httpNetResponse.ResponseString,">Volume<"));
        fundamental.MarketCap=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Market Cap<"));
        fundamental.PE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PE Ratio (TTM)<"));
        fundamental.EPS=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EPS (TTM)<"));
        try { Thread.Sleep(250);PopulateKeyStatistics(fundamental,webProxy); }
        catch (Exception exception)
        { 
          MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate key statistics.");
          MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
        }
        try { Thread.Sleep(250);PopulateFinancials(fundamental); }
        catch (Exception exception)
        { 
          MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate financials.");
          MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
        }
        if(fundamental.IsZero())
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Fundamental for {0} is Invalid *****",symbol));
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",fundamental.ToString()));
          return null;
        }
        return fundamental;
      }
      catch (Exception)
      {
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// *************************************************************************************************************************************************************************************
// *****************************************************************  K E Y  S T A T I S T I C S   - Y A H O O  F I N A N C E **********************************************************
// *************************************************************************************************************************************************************************************
    private static bool PopulateKeyStatistics(Fundamental fundamental,WebProxy webProxy=null)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append("http://finance.yahoo.com/q/ks?s=").Append(fundamental.Symbol);
        strRequest = sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
        if(!httpNetResponse.Success)
        {
          try{Thread.Sleep(250);}catch(Exception){;}
          httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
          if(!httpNetResponse.Success)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
            return false;
          }
        }
        fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));

        fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
        if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<"));

        fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
        if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets  (ttm)<"));

        fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
        if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity  (ttm)<"));

        fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
        if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share  (ttm)<"));
        
        fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
        if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash  (mrq)<"));

        fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
        if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt  (mrq)<"));

        fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
        if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding <sup>6</sup> "));

        fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
        if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue  (ttm)<"));

        fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
        if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth  (yoy)<"));

        fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
        if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit  (ttm)<"));

        fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
        if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA  <"));

        fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
        if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common  (ttm)<"));

        fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
        if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share  (mrq)<"));

        fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
        if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow  (ttm)<"));

        fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
        if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow  (ttm)<"));

        fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));

        return true;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
        return false;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
// *************************************************************************************************************************************************************************************
// *****************************************************************  F I N A N C I A L S   - Y A H O O  F I N A N C E *****************************************************************
// *************************************************************************************************************************************************************************************
    private static bool PopulateFinancials(Fundamental fundamental)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        sb.Append("https://finance.yahoo.com/quote/").Append(fundamental.Symbol).Append("/financials?p=").Append(fundamental.Symbol);
        strRequest = sb.ToString();
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
        if(!httpNetResponse.Success)
        {
          try{Thread.Sleep(250);}catch(Exception){;}
          httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
          if(!httpNetResponse.Success)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
            return false;
          }
        }
        fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
        if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00;

        return true;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
        return false;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
//**************************************************************************************************************************************************************************************
//*************************************************************************** I N T R A D A Y   P R I C I N G **************************************************************************
// ************************************************************** M A I N : Y A H O O **************************************************************************************************
// ************************************************************** B A C K U P : B I G C H A R T S **************************************************************************************
//**************************************************************************************************************************************************************************************

    private delegate Price LatestPriceDelegate(String symbol);
		public static Price GetLatestPrice(String symbol)
		{
      if(null==symbol)return null;
      CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
      if(null!=companyProfile && companyProfile.FreezePricing)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
        return null;
      }
      DateTime currentMarketDate=PremarketDA.GetLatestMarketDate();
		  LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceYahoo,GetLatestPriceBigCharts,GetLatestPriceGoogle};
			Price latestPrice=null;
			foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates)
			{
			  latestPrice=latestPriceDelegate.Invoke(symbol);
				if(null!=latestPrice && latestPrice.IsValid)
				{
          if(latestPrice.Date.Date.Equals(currentMarketDate.Date.Date))
          {
				    MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Price request from {0} succeeded for {1}",latestPriceDelegate.GetMethodInfo().Name,symbol)); 
				    return latestPrice;
          }
          else
          {
					  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date.  Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString()));
          }
				}
			}
      latestPrice=GetPriceAsOf(symbol,currentMarketDate);       // try to get the price from MarketWatch historical feed using todays date
      if(null!=latestPrice && latestPrice.Date.Date.Equals(currentMarketDate.Date)) return latestPrice;
      
      if(null!=companyProfile && companyProfile.CanRollPrevious)
      {
        latestPrice=PricingMarketDataHelper.RollPriceForward(symbol);
        if(null!=latestPrice)return latestPrice;
      }
	    MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol)); 
			return null; 
		}
// ***************************************************************************************************************
// ******************** L A T E S T   P R I C E   R E T R I E V A L   F O R   I N T R A - D A Y   F E E D ********
// ***************************************************************************************************************

    public static Price GetLatestPriceYahoo(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb=new StringBuilder();
        String strRequest;
        if(null==symbol)return null;
        sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol);
        strRequest=sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        Price price=new Price();
        price.Symbol=symbol.ToUpper();
        price.Date=DateTime.Now;
        price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN;
        price.Source=Price.PriceSource.Yahoo;
        return BuildPriceFromResponse(httpNetResponse.ResponseString,price);
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse) httpNetResponse.Dispose();
      }
    }

    public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
    {
      if(httpNetResponse.Contains("HTML_TAG_END"))
      {
        price = BuildPriceFromSections(httpNetResponse, price);
      }
      else
      {
        price = BuildPriceFromTable(httpNetResponse, price);
      }
      return price;
    }

    private static Price BuildPriceFromSections(String httpNetResponse,Price price)
    {
      try
      {
        String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END");
        List<String> sections = null;

        sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");

        String marker ="data-symbol=\""+price.Symbol+"\""; 
        List<String> dataSections = sections.Where(x => x.Contains(marker)).ToList();
        dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
        if(dataSections.Count>0)
        {
          dataSections = Sections.GetSections(dataSections[0]);
          if(null!=dataSections && dataSections.Count==1)
          {
            price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[0]);
          }
          else if(null!=dataSections && dataSections.Count>0)
          {
            price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]);
          }
        }

        marker = "<fin-streamer data-symbol=\""+price.Symbol+"\""; 
        sections = sections.Where(x => x.StartsWith(marker)).ToList();
        for(int index=0;index<sections.Count;index++)
        {
          String sectionText = sections[index];
          String itemValue = null;

          if(sectionText.Contains("regularMarketPreviousClose"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
            price.PrevClose=FeedParser.ParseValue(itemValue);
          }
          else if(sectionText.Contains("regularMarketOpen"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
            price.Open=FeedParser.ParseValue(itemValue);
          }
          else if(sectionText.Contains("regularMarketDayRange"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
            String[] subItems = itemValue.Split('-');
            price.Low=FeedParser.ParseValue(subItems[0]);
            price.High=FeedParser.ParseValue(subItems[1]);
            if(double.IsNaN(price.Close))
            {
              price.Close=price.AdjClose=(price.Low+price.High)/2.00;
            }
          }
          else if(sectionText.Contains("fiftyTwoWeekRange"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
          }
          else if(sectionText.Contains("regularMarketVolume"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
            price.Volume=FeedParser.ParseValueLong(itemValue);
          }
          else if(sectionText.Contains("averageVolume"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
          }
          else if(sectionText.Contains("averageVolume"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
          }
          else if(sectionText.Contains("marketCap"))
          {
            itemValue = Sections.GetSections(sections[index])[0];
          }
        }
        CheckPrice(price);
        return price;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: for {0} -> {1}",price.Symbol,exception.ToString()));
        return null;
      }
    }

    private static Price BuildPriceFromTable(String httpNetResponse,Price price)
    {
      MemoryStream memoryStream = null;

      try
      {
        List<String> sections = null;
        double bid = double.NaN;
        double bidContracts = double.NaN;
        double ask = double.NaN;
        double askContracts = double.NaN;

        String symbol=price.Symbol.ToUpper();
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);

        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]");
			  if(null == table || 0 == table.Count)return null;
			  HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
        if (null == rows || 0 == rows.Count) return null;
        for(int index=0;index< rows.Count; index++)
        {
          int itemIndex=0;
          String itemValue=null;
          String innerHtml = rows[index].InnerHtml;
          sections=Sections.GetSections(innerHtml);

          if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
          }
          else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
            price.Open=FeedParser.ParseValue(itemValue);
          }
          else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
            if(String.IsNullOrEmpty(itemValue))continue;
            String[] bidContractSplit = itemValue.Split('x');
            bid=FeedParser.ParseValue(bidContractSplit[0]);
            bidContracts=FeedParser.ParseValue(bidContractSplit[1]);
          }
          else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
            if(String.IsNullOrEmpty(itemValue))continue;
            String[] askContractSplit = itemValue.Split('x');
            ask = FeedParser.ParseValue(askContractSplit[0]);
            askContracts = FeedParser.ParseValue(askContractSplit[1]);
          }
          else if(Sections.FindInSections(sections, "Day&#x27;s Range",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
            String[] subItems=itemValue.Split('-');
            price.Low=FeedParser.ParseValue(subItems[0]);
            price.High=FeedParser.ParseValue(subItems[1]);
          }
          else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
          }
          else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+4];
            price.Volume=FeedParser.ParseValueLong(itemValue);
          }
          else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true))
          {
            itemValue = sections[itemIndex+3];
          }
        }

        if (double.IsNaN(price.Close))
        {
          sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
          String marker = "data-symbol=\"" + price.Symbol + "\"";
          sections = sections.Where(x => x.Contains(marker)).ToList();
          sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
          if (sections.Count > 0)
          {
            sections = Sections.GetSections(sections[0]);
            if (null != sections && sections.Count > 0)
            {
              price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]);
            }
          }
        }

        if(double.IsNaN(price.Close))
        {
          double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts);
          if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice;
          if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00;
        }

        CheckPrice(price);
        return price;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: for {0} -> {1}",price.Symbol,exception.ToString()));
        return null;
      }
      finally
      {
        if(null!=memoryStream)
        {
          memoryStream.Close();
          memoryStream.Dispose();
          memoryStream=null;
        }
      }
    }

    private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts)
    {
      if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN;
      double totalContracts = bidContracts + askContracts;
      double bidWeight = bidContracts/totalContracts;
      double askWeight = askContracts/totalContracts;
      double weightedBid = bid*bidWeight;
      double weightedAsk = ask*askWeight;
      MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2)));
      return weightedBid + weightedAsk;
    }

    public static void CheckPrice(Price price)
    {
      if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
      {
        price.Close=price.AdjClose=(price.High+price.Low)/2.00;
      }
      if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)  
      {  // if the close price is garbage set it to the midpoint of the high and the low
        if(price.Close<price.Low||price.Close>price.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
      }
      if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close;
      if(double.IsNaN(price.Open))price.Open=0.00;
      if(double.IsNaN(price.Low))price.Low=0.00;
      if(double.IsNaN(price.High))price.High=0.00;
      if(0==price.High && !double.IsNaN(price.Close) && 0!=price.Close)
      {
        price.High=price.Close;
      }
      if(0==price.Low && !double.IsNaN(price.Close) && 0!=price.Close)
      {
        price.Low=price.Close;
      }
      if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
      {
        price.Open=price.High=price.Low=price.Close;
      }
      if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low))
      {
        price.Open=(price.High+price.Low)/2.00;
      }
    }

// This is used in the intra-day price feed.
    public static Price GetLatestPriceBigCharts(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      MemoryStream memoryStream=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        if(null==symbol) return null;
        sb.Append("http://bigcharts.marketwatch.com/quickchart/quickchart.asp?symb=").Append(symbol).Append("&insttype=Stock").Append(symbol);
        strRequest = sb.ToString();

        WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceBigCharts");
        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        Price price = new Price();
        price.Symbol = symbol.ToUpper();
        price.Date = DateTime.Now;
        price.Source=Price.PriceSource.BigCharts;
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quickchart\"]");
        if(null==table||0==table.Count)return null;
        HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"maincontent\"]");
        if(null==divSection)return null;
        HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
        if(rows.Count<3)return null;
        HtmlNode softTimeNode = htmlDocument.DocumentNode.SelectSingleNode("//*[@class=\"soft time\"]");
        if(null!=softTimeNode)
        {
          DateTime givenDate=Utility.ParseDate(softTimeNode.InnerHtml);
          if(!Utility.IsEpoch(givenDate))price.Date=givenDate;
        }
        HtmlNodeCollection dataColumns = rows[2].SelectNodes(".//td");
        for(int index=0;index<dataColumns.Count;index++)
        {
          String innerText=Utility.RemoveControlChars(dataColumns[index].InnerText).Trim();
          String[] items=innerText.Split(':');
          if(null==items||2!=items.Length)continue;
          if("Last".Equals(items[0])){price.Close=FeedParser.ParseValue(items[1]);price.AdjClose=price.Close;}
          else if("Open".Equals(items[0]))price.Open=FeedParser.ParseValue(items[1]);
          else if("High".Equals(items[0]))price.High=FeedParser.ParseValue(items[1]);
          else if("Low".Equals(items[0]))price.Low=FeedParser.ParseValue(items[1]);
          else if("Volume".Equals(items[0]))price.Volume=FeedParser.ParseValueLong(items[1]);
        }
        if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",price.Symbol));
          return null;
        }
        if(double.IsNaN(price.Open))price.Open=0.00;
        if(double.IsNaN(price.Low))price.Low=0.00;
        if(double.IsNaN(price.High))price.High=0.00;
        if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
        {
          price.Open=price.High=price.Low=price.Close;
        }
        return price;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
        if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
      }
    }

    public static Price GetLatestPriceGoogle(String symbol)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        StringBuilder sb = new StringBuilder();
        String strRequest;
        if(null==symbol) return null;
        String exchangeNASDAQ="NASDAQ";
        String exchangeNYSE="NYSE";

        sb = new StringBuilder();
        sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNASDAQ);
        strRequest = sb.ToString();
        WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceGoogle");
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }

        List<String> divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div");
        if(null==divSections || 0==divSections.Count)return null;
        List<String> searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList();
        if(null!=searchPattern && searchPattern.Count>0)
        {
          httpNetResponse.Dispose();
          sb = new StringBuilder();
          sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNYSE);
          strRequest = sb.ToString();
          httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
          if(!httpNetResponse.Success)
          {
            MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
            return null;
          }
          divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div");
          searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList();
          if(null!=searchPattern && searchPattern.Count>0)
          {
            return null;
          }
        }

        List<String> sections = Sections.GetSections(httpNetResponse.ResponseString);
        sections = Sections.FlattenSection(sections);

        int index=0;

        Price price = new Price();
        price.Symbol = symbol.ToUpper();
        price.Source=Price.PriceSource.Google;
				price.Close=price.Open=price.High=price.Low=price.AdjClose=double.NaN;
        price.Open=0.00;
        price.Date = DateTime.Now;

        if(Sections.FindInSections(sections, "After Hours:", 0,ref index, true))
        {
          price.Close=price.AdjClose=FeedParser.ParseValue(sections[index-1]);  
        }
        else if(Sections.FindInSections(sections, "Share", 0,ref index, true))
        {
          price.Close=price.AdjClose=FeedParser.ParseValue(sections[index+1]);
        }

        if(Sections.FindInSections(sections, "Previous close", 0,ref index, true))
        {
          price.Open=FeedParser.ParseValue(sections[index+2]);  
        }

        if(Sections.FindInSections(sections, "Day range", 0,ref index, true))
        {
          String itemValue = sections[index+2];
          String[] highLow = itemValue.Split('-');
          price.Low=FeedParser.ParseValue(highLow[0]);
          price.High=FeedParser.ParseValue(highLow[1]);
        }
        if(!price.IsValid)return null;
        return price;
      }
      catch (Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }

// *******************************************************************************************************************************************************************************
// *******************************************************************************************************************************************************************************
// *******************************************************************************************************************************************************************************



// *******************************************************************************************************************************************************************************
// ************************************************************** H I S T O R I C A L  P R I C E S - B I G   C H A R T S  ********************************************************
// *******************************************************************************************************************************************************************************
    public static Prices GetPricesAsOf(String symbol, DateTime startDate,DateTime endDate)
    {
      Prices prices=new Prices();

      if(null==symbol)return null;
      CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
      if(null!=companyProfile && companyProfile.FreezePricing)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
        return null;
      }
      DateGenerator dateGenerator=new DateGenerator();
      List<DateTime> dates=dateGenerator.GenerateHistoricalDates(startDate,endDate);
      for(int index=0;index<dates.Count;index++)
      {
        DateTime currentDate=dates[index];
        Price price=GetPriceAsOf(symbol,currentDate);
        if(null==price){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data for {0} on {1}",symbol,Utility.DateTimeToStringMMHDDHYYYY(currentDate)));continue;};
        prices.Add(price);
      }
      return prices;
    }

    public static Price GetPriceAsOf(String symbol, DateTime asOf)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
        String strRequest;
        StringBuilder sb = null;

        if (null == symbol) return null;
        CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
        if(null!=companyProfile && companyProfile.FreezePricing)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
          return null;
        }
        String requestSymbol=symbol;
        if(requestSymbol.StartsWith("^"))requestSymbol=requestSymbol.Substring(1);
        sb = new StringBuilder();
        sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
        sb.Append(requestSymbol);
        sb.Append("&closeDate=");
        sb.Append(asOf.Month.ToString());
        sb.Append("%2F");
        sb.Append(asOf.Day.ToString());
        sb.Append("%2F");
        sb.Append((asOf.Year-2000).ToString());
        sb.Append("&x=38&y=25");
        strRequest = sb.ToString();

        httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
        if(!httpNetResponse.Success)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
          return null;
        }
        byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
        MemoryStream memoryStream = new MemoryStream(streamBytes);
        HtmlDocument htmlDocument = new HtmlDocument();
        htmlDocument.Load(memoryStream);
        HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]");
        if (null == tables || 0 == tables.Count) return null;
        HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
        if (rows.Count < 7) return null;
        Price price=new Price();
        price.Source=Price.PriceSource.BigCharts;
        price.Symbol = symbol.ToUpper();
        price.Date=FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
        price.Close=FeedParser.ParseValueFromMarketWatch("Closing Price:",rows[2].InnerText);
        price.AdjClose = price.Close;
        price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText);
        price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText);
        price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText);
        price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText);
        if(!(price.Date.Date.Equals(asOf.Date.Date)))
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The price retrieved for {0} does not contain the requested date.  Requested date {1} Retrieved date {2}",symbol,price.Date.ToShortDateString(),asOf.ToShortDateString()));
          return null;
        }
        return price;
      }
      catch(Exception)
      {
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
//********************************************************************************************************************************************************************************************************
// ******************************************************************************** H I S T O R I C A L  P R I C I N G  Y A H O O  *********************************************************************
//********************************************************************************************************************************************************************************************************

    public static Price GetDailyPrice(String symbol,DateTime pricingDate)
    {
      if(null==symbol)return null;
      CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
      if(null!=companyProfile && companyProfile.FreezePricing)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
        return null;
      }
      Prices prices=GetDailyPrices(symbol,pricingDate,pricingDate);
      if(null==prices||0==prices.Count)return null;
      Price price=prices.FirstOrDefault();
      if(!price.Date.Date.Equals(pricingDate.Date))
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested.  Date requested '{1}', date returned '{2}'",symbol,pricingDate.ToShortDateString(),price.Date.Date.ToShortDateString()));
        return null;
      }
      return price;
    }

    public static Prices GetDailyPrices(String symbol, DateTime startDate, DateTime endDate)
    {
      HttpNetResponse httpNetResponse=null;
      try
      {
         if(symbol==null)return null;
         CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
         if(null!=companyProfile && companyProfile.FreezePricing)
         {
           MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
           return null;
         }
        startDate=startDate.Date;
        endDate=endDate.Date;
        if(startDate>endDate)
        {
          DateTime tempDate=startDate;
          startDate=endDate;
          endDate=tempDate;
        }

			  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[MarketDataHelper:GetDailyPrices]{0} start:{1} end:{2}",symbol,Utility.DateTimeToStringMMHDDHYYYY(startDate),Utility.DateTimeToStringMMHDDHYYYY(endDate)));
        StringBuilder sb = new StringBuilder();
        String strRequest;
        Prices prices=null;
        sb.Append("https://query1.finance.yahoo.com/v7/finance/chart/").Append(symbol).Append("?period1=").Append(Utility.DateToUnixDate(startDate)).Append("&period2=").Append(Utility.DateToUnixDate(endDate)).Append("&interval=1d&events=history");
        strRequest=sb.ToString();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Query:{0}",strRequest));
        WebProxy webProxy=HttpNetRequest.GetProxy("GetDailyPrices");
        CookieCollection cookieCollection=new CookieCollection();
        httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3B(strRequest,cookieCollection,webProxy);
        if(!httpNetResponse.Success) return null;
        JObject json=JObject.Parse(httpNetResponse.ResponseString);
        prices=GetPricesFromJSONString(json,symbol);
        return prices;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
        return null;
      }
      finally
      {
        if(null!=httpNetResponse)httpNetResponse.Dispose();
      }
    }
    private static Prices GetPricesFromJSONString(JObject json,String symbol)
    { 
      Prices prices=new Prices();

      try
      {
        JToken token=null;
        int items=0;
        
        try{token=json["chart"]["result"][0]["timestamp"];}catch(Exception){;}
        if(null==token)
        {
          string instrumentType=null;
          try{instrumentType=json["chart"]["result"][0]["meta"]["instrumentType"].ToString();}catch(Exception){;}
          if(null==instrumentType||!instrumentType.Equals("MUTUALFUND"))return prices;
          String strPrice=null;
          String strDate=null;
          try{strPrice=json["chart"]["result"][0]["meta"]["regularMarketPrice"].ToString();}catch(Exception){;}
          try{strDate=json["chart"]["result"][0]["meta"]["currentTradingPeriod"]["regular"]["start"].ToString();}catch(Exception){;}
          if(null==strPrice||null==strDate)return prices;
          Price price=new Price();
          price.Symbol=symbol;
          price.Source=Price.PriceSource.Yahoo;
          price.High=price.Low=price.Close=price.Open=price.AdjClose=FeedParser.ParseValue(strPrice);
          DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strDate));
          price.Date=dateTimeOffset.DateTime;
          price.Volume=0L;
          prices.Add(price);
          return prices;
        }
        try{items=token.Children().Count();}catch(Exception){;}
        if(0==items)return prices;
        for(int index=0;index<items;index++)
        {
          String strTradeDate=(json["chart"]["result"][0]["timestamp"][index]).ToString();
          String strHigh=null;
          String strOpen=null;
          String strLow=null;
          String strClose=null;
          String strVolume=null;
          String strAdjClose=null;

          try{strHigh=(json["chart"]["result"][0]["indicators"]["quote"][0]["high"][index]).ToString();}catch(Exception){;}
          try{strOpen=(json["chart"]["result"][0]["indicators"]["quote"][0]["open"][index]).ToString();}catch(Exception){;}
          try{strLow=(json["chart"]["result"][0]["indicators"]["quote"][0]["low"][index]).ToString();}catch(Exception){;}
          try{strClose=(json["chart"]["result"][0]["indicators"]["quote"][0]["close"][index]).ToString();}catch(Exception){;}
          try{strVolume=(json["chart"]["result"][0]["indicators"]["quote"][0]["volume"][index]).ToString();}catch(Exception){;}
          try{strAdjClose=(json["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"][index]).ToString();}catch(Exception){;}

          DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strTradeDate));
          Price price=new Price();
          price.Source=Price.PriceSource.Yahoo;
          price.Symbol=symbol;
          price.Date=dateTimeOffset.DateTime;
          price.High=FeedParser.ParseValue(strHigh);
          price.Open=FeedParser.ParseValue(strOpen);
          price.Close=FeedParser.ParseValue(strClose);
          price.Low=FeedParser.ParseValue(strLow);
          price.Volume=(long)FeedParser.ParseValue(strVolume);
          price.AdjClose=FeedParser.ParseValue(strAdjClose);

          if(price.Open<=0.05&&price.High<=0.05&&price.Low<=0.05&&price.Close<=0.05&&price.AdjClose<=0.05)continue;
          if(double.IsNaN(price.Close)||price.Close==0.00)continue;
          if(price.Volume<0)continue;
          if(price.Open==0.00||double.IsNaN(price.Open))price.Open=price.Close;
          if(price.High==0.00||double.IsNaN(price.High))price.High=price.Close;
          if(price.Low==0.00||double.IsNaN(price.Low))price.Low=price.Close;
          if(price.Close<price.Low||price.Close>price.High)price.Close=(price.High+price.Low)/2.00;

          prices.Add(price);

        }
        return prices;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
        return prices;
      }
    }

// ******************************************************************************************************************************************************************************************************************
// *************************************************************************** E N D   Y A H O O  H I S T O R I C A L  P R I C I N G  *******************************************************************************
// ******************************************************************************************************************************************************************************************************************


// ****************************************************************************************************************************************************************************************
// **************************************************************************** E N D  D A T A  F E E D S *********************************************************************************
// ****************************************************************************************************************************************************************************************




// ***************************************************************************************************************************************************************************************
// ***************************************************************************************************************************************************************************************
// ***************************************************************************************************************************************************************************************

// 2017 Yahoo Finance Modified Format of all feeds.  Most of the HTMLDocument parser code is broken because the response strings are no longer in tabular format.
// It's difficult to make sense of the new response string outside of noting that it remains in aa tag format.    The following code is an attempt to make sense of it all.

    public static List<String> LocateJSONItems(String strInput,String item,bool isHeading=false)
    {
      try
      {
        if(null==item||null==strInput)return null;
        int startPos=strInput.IndexOf(item);
        if(-1==startPos)return null;
        strInput=strInput.Substring(startPos-1);
        if(isHeading)
        {
          strInput=strInput.Substring(item.Length);
          startPos=strInput.IndexOf(item);
          if(-1==startPos)return null;
          strInput=strInput.Substring(startPos-1);
        }
        strInput=Utility.BetweenString(strInput,"[","]");
        if(null==strInput)return null;
        String[] itemsArray=strInput.Split(',');
        return itemsArray.ToList();
      }
      catch(Exception)
      {
        return null;
      }
    }

    public static List<KeyValue> LocateJSONKeyValuePairs(String strInput, String item1,String item2)
    {
      try
      {
        List<KeyValue> items=new List<KeyValue>();
        if (null == item1 || null == strInput) return null;
        int startPos = strInput.IndexOf(item1);
        while(true)
        {
          KeyValue keyValue=new KeyValue();
          if (-1 == startPos||null==strInput) break;
          String strItem = strInput.Substring(startPos);
          strItem = Utility.BetweenString(strItem, "\"", "\"");
          String strValue = strInput.Substring((startPos+strItem.Length+4)-1);
          strValue = Utility.BetweenString(strValue, "\"", "\"");
          keyValue.Key=strValue;
          try{strInput=strInput.Substring((startPos+strItem.Length+4+strValue.Length+4)-1);}
          catch(Exception){break;}
          if(null==strInput)break;
          startPos = strInput.IndexOf(item2);
          if(-1==startPos)break;
          strItem = strInput.Substring(startPos);
          strItem = Utility.BetweenString(strItem, "\"", "\"");
          strValue = strInput.Substring((startPos + strItem.Length + 4) - 1);
          strValue = Utility.BetweenString(strValue, "\"", "\"");
          keyValue.Value=strValue;
          try { strInput = strInput.Substring((startPos + strValue.Length + 4) - 1); }
          catch (Exception) { break; }
          if(null==strInput)break;
          startPos = strInput.IndexOf(item1);
          items.Add(keyValue);
        }
        return items;
      }
      catch (Exception)
      {
        return null;
      }
    }
    public static List<List<String>> LocateJSONItems(String strInput)
    {
      List<List<String>> items=new List<List<String>>();
      try
      {
        if(null==strInput)return null;
        strInput=Utility.BetweenString(strInput,"[","]");
        if(null==strInput)return null;
        String[] itemsArray=strInput.Split(',');
        if(null==itemsArray||0==itemsArray.Length)return null;
        for(int index=0;index<itemsArray.Length;index++)
        {
          String item=itemsArray[index];
          item=item.Replace("{",null);
          item=item.Replace("}",null);
          item=item.Replace("\"",null);
          String[] subItemArray=item.Split(':');
          if(null==subItemArray||2!=subItemArray.Length)continue;
          if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
          if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
          items.Add(subItemArray.ToList());
        }
        return items;
      }
      catch(Exception)
      {
        return null;
      }
    }

    public static List<List<String>> LocateJSONItemsTags(String strInput)
    {
      List<List<String>> items=new List<List<String>>();
      try
      {
        if(null==strInput)return null;
//        strInput=Utility.BetweenString(strInput,"[","]");
//        if(null==strInput)return null;
        String[] itemsArray=strInput.Split(',');
        if(null==itemsArray||0==itemsArray.Length)return null;
        for(int index=0;index<itemsArray.Length;index++)
        {
          String item=itemsArray[index];
          item=item.Replace("{",null);
          item=item.Replace("}",null);
          item=item.Replace("\"",null);
          String[] subItemArray=item.Split(':');
          if(null==subItemArray||2!=subItemArray.Length)continue;
          if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
          if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
          items.Add(subItemArray.ToList());
        }
        return items;
      }
      catch(Exception)
      {
        return null;
      }
    }


		private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
		{
		  DateTime referenceDate=DateTime.Now;
		  string[] items=relativeDate.Split(' ');
			if(items.Length<2)return Utility.Epoch;
			if("days".Equals(items[1])||"day".Equals(items[1]))
			{
			  TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0,0,0);
				referenceDate-=timeSpan;
			}
			else if("hours".Equals(items[1])||"hour".Equals(items[1]))
			{
			  TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
				referenceDate-=timeSpan;
			}
			else if("mins".Equals(items[1]) || "minutes".Equals(items[1]))
			{
			  TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
        referenceDate-=timeSpan;			  
			}
			else if( (new string[]{"Jul","Aug","Sep","Oct","Nov","Dec","Jan","Feb","Mar","Apr","May","Jun"}).Any(x=>x.Equals(items[0])))
			{
			  DateTime now=DateTime.Now;
			  StringBuilder sb=new StringBuilder();
				sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year);
        referenceDate=Utility.ParseDate(sb.ToString());			  
				if(referenceDate>now)
				{
					sb=new StringBuilder();
					sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year-1);
          referenceDate=Utility.ParseDate(sb.ToString());			  
				}
			}
			else referenceDate=Utility.Epoch; 
			if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch;
			return referenceDate;
		}

// ****************************************************************************************************************************************************************************************
// **************************************************************************** H E L P E R  M E T H O D S  *******************************************************************************
// ****************************************************************************************************************************************************************************************
    private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName)
    {
      StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt");
      for (int row = 0; row < rows.Count; row++)
      {
        streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml);
      }
      streamWriter.Flush();
      streamWriter.Close();
    }

    private static void WriteToDisk(String responseString, String strPathFileName)
    {
      StreamWriter streamWriter = File.CreateText(strPathFileName);
      streamWriter.WriteLine(responseString);
      streamWriter.Flush();
      streamWriter.Close();
    }

    private static void WriteToDisk(List<String> sections, String strPathFileName)
    {
      StreamWriter streamWriter = File.CreateText(strPathFileName);
      for(int index=0;index<sections.Count;index++)
      {
        streamWriter.WriteLine(String.Format("Section[{0}]",index));
        streamWriter.WriteLine(sections[index]);
      }
      streamWriter.Flush();
      streamWriter.Close();
    }
  }
}
