marketdata/MarketDataLib/Helper/HeadlinesMarketDataHelper.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
using MarketData.Utils;

namespace MarketData.Helper
{
  public class HeadlinesMarketDataHelper
  {
    private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
    private static int WAIT_BETWEEN_REQUESTS_MS = 500;   // wait 500 ms between requests
    private List<String> symbols;
    private int currentIndex = 0;

    public HeadlinesMarketDataHelper()
    {
    }
		public bool LoadHeadlines()
		{
			Profiler profiler=new Profiler();
			try
			{
        symbols=PricingDA.GetSymbols();
				currentIndex = 0;
				while (true)
				{
					List<String> queueSymbols = GetQueueSymbols();
					if (null == queueSymbols || 0 == queueSymbols.Count) break;
					ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
					for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
					{
						resetEvents[eventIndex] = new ManualResetEvent(false);
					}
					for (int index = 0; index < queueSymbols.Count; index++)
					{
						ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
						ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
					}
					MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
					WaitHandle.WaitAll(resetEvents);
				}
				return true;
			}
			finally
			{
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHistorical]End, total took {0}(ms)",profiler.End()));
			}
		}
		public bool LoadHeadlines(List<String> requestSymbols)
		{
			Profiler profiler=new Profiler();
			try
			{
        symbols=requestSymbols;
				currentIndex = 0;
				while (true)
				{
					List<String> queueSymbols = GetQueueSymbols();
					if (null == queueSymbols || 0 == queueSymbols.Count) break;
					ManualResetEvent[] resetEvents = new ManualResetEvent[queueSymbols.Count];
					for (int eventIndex = 0; eventIndex < resetEvents.Length; eventIndex++)
					{
						resetEvents[eventIndex] = new ManualResetEvent(false);
					}
					for (int index = 0; index < queueSymbols.Count; index++)
					{
						ThreadHelper threadHelper = new ThreadHelper(queueSymbols[index],resetEvents[index]);
						ThreadPool.QueueUserWorkItem(ThreadPoolCallbackLoadHeadline, threadHelper);
            try{Thread.Sleep(WAIT_BETWEEN_REQUESTS_MS);}catch{;}
					}
					MDTrace.WriteLine(LogLevel.DEBUG,"Load Headline, waiting for queued items to complete.");
					WaitHandle.WaitAll(resetEvents);
				}
				return true;
			}
			finally
			{
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[LoadHistorical]End, total took {0}(ms)",profiler.End()));
			}
		}
    private List<String> GetQueueSymbols()
    {
      List<String> queueSymbols = new List<String>();
      int index = currentIndex;
      for (; index < currentIndex + MaxThreads && index < symbols.Count; index++)
      {
        queueSymbols.Add(symbols[index]);
      }
      currentIndex = index;
      return queueSymbols;
    }
    public void ThreadPoolCallbackLoadHeadline(Object threadHelperContext)
    {
      ThreadHelper threadHelper = (ThreadHelper)threadHelperContext;
			try
			{
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Headline, Thread {0} started for {1}...", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
				LoadHeadlineEx(threadHelper.Symbol);
				MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Load Headline, Thread {0} ended for {1}", Thread.CurrentThread.ManagedThreadId, threadHelper.Symbol));
			}
			finally
			{
			  threadHelper.ResetEvent.Set();
			}
    }
// This method is used by unit test
    public static Headlines GetHeadlinesEx(String symbol)
    {
      Headlines headlinesCollection=new Headlines();
      Headlines headlines=null;
// SEEKING ALPHA
      //headlines=MarketDataHelper.GetCompanyHeadlines(symbol);
      //if(null!=headlines)headlinesCollection.AddRange(headlines);
// NASDAQ
      headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
      if(null!=headlines)headlinesCollection.AddRange(headlines);
// MARKETWATCH
      headlines=MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
      if(null!=headlines)headlinesCollection.AddRange(headlines);
      return headlinesCollection;
    }
    public static void LoadHeadlineEx(String symbol)
    {
      DateTime marketDate=PremarketDA.GetLatestMarketDate();
      marketDate=new DateTime(marketDate.Year,marketDate.Month,marketDate.Day,23,59,59);
     
      Headlines headlines=null;
// SEEKING ALPHA
      //MDTrace.WriteLine(LogLevel.DEBUG,String.Format("LoadHeadlinesSymbol {0}",symbol));
      //headlines=MarketDataHelper.GetCompanyHeadlines(symbol);
      //if(headlines.IsNullOrEmpty())
      //{
      //  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
      //}
      //else
      //{
      //  MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
      //  headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
      //  HeadlinesDA.InsertHeadlines(headlines);
      //  foreach(Headline headline in headlines)
      //  {
      //    MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
      //  }
      //}
// NASDAQ
      headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
      if(headlines.IsNullOrEmpty())
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from NASDAQ",symbol));
      }
      else
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from NASDAQ",headlines.Count,symbol));
        headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
        HeadlinesDA.InsertHeadlines(headlines);
        foreach(Headline headline in headlines)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("NASDAQ: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
        }
      }
// MARKETWATCH
      headlines=MarketDataHelper.GetCompanyHeadlinesMarketWatch(symbol);
      if(headlines.IsNullOrEmpty())
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from MARKETWATCH",symbol));
      }
      else
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from MARKETWATCH",headlines.Count,symbol));
        headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
        HeadlinesDA.InsertHeadlines(headlines);
        foreach(Headline headline in headlines)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("MARKETWATCH: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
        }
      }
      return;
    }
  }
}