5495 lines
274 KiB
C#
5495 lines
274 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
using System.Linq;
|
|
using System.Text.RegularExpressions;
|
|
using System.Data;
|
|
using System.Threading;
|
|
using System.Net;
|
|
using System.Collections;
|
|
using System.Reflection;
|
|
using System.Web;
|
|
using System.Xml.Linq;
|
|
using Newtonsoft.Json.Linq;
|
|
using HtmlAgilityPack;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Utils;
|
|
using MarketData.Numerical;
|
|
using MarketData.CSVHelper;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Integration;
|
|
using System.Globalization;
|
|
using MarketDataLib.Utility;
|
|
|
|
//Zacks Rank - Zacks
|
|
//Splits - EODDATA
|
|
//GDP Data - WorldBank
|
|
//Dividend History - NASDAQ
|
|
//Options - NASDAQ
|
|
//Analyst Ratings - Briefing.com
|
|
//SEC Filings - SEC.GOV (EDGAR)
|
|
//Yield Curve - Treasury.Gov
|
|
//ETF Holdings - Yahoo Finance
|
|
//Insider Transactions - Insider Tracking.com
|
|
//Company Profile - MorningStar + Reuters
|
|
//Headlines - Seeking Alpha + NASDAQ
|
|
//CIK Codes - SEC.GOV
|
|
//Analyst Price Target - Yahoo Finance , Market Beat as backup
|
|
//Historical - MorningStar
|
|
//Income Statement - NASDAQ
|
|
//Balance Sheet - NASDAQ
|
|
//Statement of Cashflows - MorningStar
|
|
//Fundamentals - 2 Sources....Yahoo Finance & FINVIZ
|
|
// a) Key Statistics
|
|
// b) Financials
|
|
//Intraday Pricing - Yahoo Finance + BigCharts
|
|
//Historical Pricing - BigCharts + Yahoo Finance
|
|
//CompanyDescription - Reuters
|
|
//Premarket - CNN
|
|
|
|
namespace MarketData.Helper
|
|
{
|
|
public class MarketDataHelper
|
|
{
|
|
public static int DEFAULT_TIMEOUT_MS=30000;
|
|
public static String SEC_BASE_URL="https://www.sec.gov";
|
|
|
|
|
|
public static Dictionary<String,bool> RunTests()
|
|
{
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
Dictionary<String,bool> items=new Dictionary<String,bool>();
|
|
String symbol="MIDD";
|
|
String dividendSymbol="AAPL";
|
|
String insiderTransactionSymbol="AAVL";
|
|
String etfSymbol="JFNNX";
|
|
String analystPriceTargetSymbol="MGPI";
|
|
String sourceCurrency="USD";
|
|
DateTime analysisDate=dateGenerator.FindPrevBusinessDay(DateTime.Now);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetDividendHistory");
|
|
DividendHistory dividendHistory=GetDividendHistory(dividendSymbol);
|
|
if(null==dividendHistory||0==dividendHistory.Count)items.Add("GetDividendHistory",false);
|
|
else items.Add("GetDividendHistory",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetPremarketData");
|
|
PremarketElements premarketElements=GetPremarketData();
|
|
if(null==premarketElements||0==premarketElements.Count)items.Add("GetPremarketData",false);
|
|
else items.Add("GetPremarketData",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetEarningsAnnouncements");
|
|
EarningsAnnouncements earningsAnnouncements=GetEarningsAnnouncements(symbol);
|
|
if(null==earningsAnnouncements||0==earningsAnnouncements.Count)items.Add("GetEarningsAnnouncements",false);
|
|
else items.Add("GetEarningsAnnouncements",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetZacksRank");
|
|
ZacksRank zacksRank=GetZacksRank(symbol);
|
|
if(null==zacksRank)items.Add("GetZacksRank",false);
|
|
else items.Add("GetZacksRank",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetSplits");
|
|
Splits splits=GetSplits();
|
|
if(null==splits)items.Add("GetSplits",false);
|
|
else items.Add("GetSplits",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetGDPPerCapita");
|
|
EconomicIndicators economicIndicators=GetGDPPerCapita();
|
|
if(null==economicIndicators)items.Add("GetGDPPerCapita",false);
|
|
else items.Add("GetGDPPerCapita",true);
|
|
|
|
//MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetOptions");
|
|
//Options options=GetOptions(symbol);
|
|
//if(null==options||0==options.Count) items.Add("GetOptions",false);
|
|
//else items.Add("GetOptions",true);
|
|
items.Add("GetOptions",false);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetLatestAnalystRatings");
|
|
AnalystRatings analystRatings=GetLatestAnalystRatings();
|
|
if(null==analystRatings||0==analystRatings.Count)items.Add("GetLatestAnalystRatings",false);
|
|
else items.Add("GetLatestAnalystRatings",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystRatingsMarketBeat");
|
|
analystRatings=GetAnalystRatingsMarketBeat(symbol);
|
|
if(null==analystRatings||0==analystRatings.Count)items.Add("GetAnalystRatingsMarketBeat",false);
|
|
else items.Add("GetAnalystRatingsMarketBeat",true);;
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCIK");
|
|
String strCik=GetCIK(symbol);
|
|
if(null==strCik)items.Add("GetCIK",false);
|
|
else items.Add("GetCIK",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetSECFilings");
|
|
String cik=PricingDA.GetCIKForSymbol(symbol);
|
|
SECFilings secFilings=GetSECFilings(symbol,cik);
|
|
if(null==secFilings||0==secFilings.Count)items.Add("GetSECFilings",false);
|
|
else items.Add("GetSECFilings",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetYieldCurve");
|
|
YieldCurve yieldCurve=GetYieldCurve(analysisDate.Year);
|
|
if(null==yieldCurve||0==yieldCurve.Count)items.Add("GetYieldCurve",false);
|
|
else items.Add("GetYieldCurve",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetETFHoldings");
|
|
ETFHoldings etfHoldings=GetETFHoldings(etfSymbol);
|
|
if(null==etfHoldings||0==etfHoldings.Count)items.Add("GetETFHoldings",false);
|
|
else items.Add("GetETFHoldings",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetInsiderTransactions");
|
|
InsiderTransactions insiderTransactions=GetInsiderTransactions(insiderTransactionSymbol);
|
|
if(null==insiderTransactions||0==insiderTransactions.Count)items.Add("GetInsiderTransactions",false);
|
|
else items.Add("GetInsiderTransactions",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCompanyProfile");
|
|
CompanyProfile companyProfile=GetCompanyProfile(symbol);
|
|
if(null==companyProfile)items.Add("GetCompanyProfile",false);
|
|
else items.Add("GetCompanyProfile",true);
|
|
|
|
|
|
//MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCompanyDescription");
|
|
//String companyDescription=GetCompanyDescription(symbol);
|
|
//if(null==companyDescription)items.Add("GetCompanyDescription",false);
|
|
//else items.Add("GetCompanyDescription",true);
|
|
|
|
Headlines headlines=HeadlinesMarketDataHelper.GetHeadlinesEx(symbol);
|
|
if(null==headlines || 0==headlines.Count)items.Add("GetHeadlines",false);
|
|
else items.Add("GetHeadlines",true);
|
|
|
|
//MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystPriceTargetYahoo");
|
|
//AnalystPriceTarget analystPriceTarget=GetAnalystPriceTargetYahoo(analystPriceTargetSymbol);
|
|
//if(null==analystPriceTarget)items.Add("GetAnalystPriceTargetYahoo",false);
|
|
//else items.Add("GetAnalystPriceTargetYahoo",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetAnalystPriceTargetMarketBeat");
|
|
AnalystPriceTarget analystPriceTarget=GetAnalystPriceTargetMarketBeat(analystPriceTargetSymbol);
|
|
if(null==analystPriceTarget)items.Add("GetAnalystPriceTargetYahoo",false);
|
|
else items.Add("GetAnalystPriceTargetMarketBeat",true);
|
|
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetHistoricalValues");
|
|
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> timeSeries=GetHistoricalValues(symbol);
|
|
if(null==timeSeries)items.Add("GetHistoricalValues",false);
|
|
else items.Add("GetHistoricalValues",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetIncomeStatement");
|
|
List<IncomeStatement> incomeStatement=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
|
|
if(null==incomeStatement||0==incomeStatement.Count)items.Add("GetIncomeStatement",false);
|
|
else items.Add("GetIncomeStatement",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetBalanceSheetNASDAQ");
|
|
List<BalanceSheet> balanceSheet=GetBalanceSheetNASDAQ(symbol,BalanceSheet.PeriodType.Annual);
|
|
if(null==balanceSheet||0==balanceSheet.Count)items.Add("GetBalanceSheetNASDAQ",false);
|
|
else items.Add("GetBalanceSheetNASDAQ",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCashflowStatement");
|
|
List<CashflowStatement> cashflowStatement=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
|
|
if(null==cashflowStatement||0==cashflowStatement.Count)items.Add("GetCashflowStatement",false);
|
|
else items.Add("GetCashflowStatement",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetCurrencyConversion");
|
|
CurrencyConversionCollection currencyConversionCollection=GetCurrencyConversion(sourceCurrency,analysisDate);
|
|
if(null==currencyConversionCollection||0==currencyConversionCollection.Count)items.Add("GetCurrencyConversion",false);
|
|
else items.Add("GetCurrencyConversion",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetFundamentalFinViz");
|
|
Fundamental fundamental=GetFundamentalFinViz(symbol);
|
|
if(null==fundamental)items.Add("GetFundamentalFinViz",false);
|
|
else items.Add("GetFundamentalFinViz",true);
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Testing GetFundamental");
|
|
fundamental=GetFundamental(symbol);
|
|
if(null==fundamental)items.Add("GetFundamental",false);
|
|
else items.Add("GetFundamental",true);
|
|
|
|
fundamental=FundamentalDA.GetFundamental(symbol);
|
|
if(!PopulateKeyStatistics(fundamental))items.Add("PopulateKeyStatistics",false);
|
|
else items.Add("PopulateKeyStatistics",true);
|
|
|
|
if(!PopulateFinancials(fundamental))items.Add("PopulateFinancials",false);
|
|
else items.Add("PopulateFinancials",true);
|
|
|
|
Price price=GetLatestPriceYahoo(symbol);
|
|
if(null==price)items.Add("GetLatestPriceYahoo",false);
|
|
else items.Add("GetLatestPriceYahoo",true);
|
|
|
|
price=GetLatestPriceBigCharts(symbol);
|
|
if(null==price)items.Add("GetLatestPriceBigCharts",false);
|
|
else items.Add("GetLatestPriceBigCharts",true);
|
|
|
|
price=GetPriceAsOf(symbol,analysisDate);
|
|
if(null==price)items.Add("GetPriceAsOf",false);
|
|
else items.Add("GetPriceAsOf",true);
|
|
|
|
Prices prices=GetDailyPrices(symbol,analysisDate,analysisDate);
|
|
if(null==prices||0==prices.Count)items.Add("GetDailyPrices",false);
|
|
else items.Add("GetDailyPrices",true);
|
|
|
|
return items;
|
|
}
|
|
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************************ P R E M A R K E T D A T A ***********************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static PremarketElements GetPremarketData()
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
PremarketElements premarketElements=new PremarketElements();
|
|
DateTime timestamp=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest=null;
|
|
|
|
sb.Append("https://production.dataviz.cnn.io/markets/futures/summary");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. Failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
if(null==items || 0==items.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. No Data",strRequest));
|
|
return premarketElements;
|
|
}
|
|
for(int index=0;index<items.Count;index++)
|
|
{
|
|
List<String> pair=items[index];
|
|
if(pair[1].Equals("S&P 500"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="S&P";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
else if(pair[1].Equals("NASDAQ"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="NASDAQ";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
else if(pair[1].Equals("DOW"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="DOW";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
}
|
|
return premarketElements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
//public static PremarketElements GetPremarketData()
|
|
//{
|
|
// HttpNetResponse httpNetResponse=null;
|
|
// MemoryStream memoryStream=null;
|
|
// PremarketElements premarketElements=new PremarketElements();
|
|
// DateTime timestamp=DateTime.Now;
|
|
// try
|
|
// {
|
|
// StringBuilder sb=new StringBuilder();
|
|
// String strRequest=null;
|
|
|
|
// sb.Append("https://money.cnn.com/data/premarket/");
|
|
// strRequest=sb.ToString();
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
|
|
// httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
|
|
// if(!httpNetResponse.Success)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}' failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
// return null;
|
|
// }
|
|
// byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
// memoryStream=new MemoryStream(streamBytes);
|
|
// HtmlDocument htmlDocument=new HtmlDocument();
|
|
// htmlDocument.Load(memoryStream);
|
|
// HtmlNodeCollection divSection=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"wsod_dataTable\"]");
|
|
// if(null==divSection||0==divSection.Count)
|
|
// {
|
|
// if(!ParsePremarketElementsBySection(httpNetResponse.ResponseString,premarketElements))
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, cannot locate data in result for request '{0}'.",strRequest));
|
|
// return null;
|
|
// }
|
|
// return premarketElements;
|
|
// }
|
|
// HtmlNodeCollection rows=divSection[0].SelectNodes(".//tr");
|
|
// for(int index=0;index<rows.Count;index++)
|
|
// {
|
|
// PremarketElement premarketElement=new PremarketElement();
|
|
// HtmlNodeCollection headerColumns=rows[index].SelectNodes(".//th");
|
|
// if(null!=headerColumns&&headerColumns.Count>0) continue;
|
|
// HtmlNodeCollection dataColumns=rows[index].SelectNodes(".//td");
|
|
// if(dataColumns.Count<2) continue;
|
|
// String market=dataColumns[0].InnerText.Trim().ToUpper();
|
|
// market=market.Replace("&","&");
|
|
// String quoteHtml=dataColumns[1].InnerHtml;
|
|
// List<String> sections=Sections.GetSections(quoteHtml);
|
|
// if(3!=sections.Count)continue;
|
|
// String changeValue=sections[0];
|
|
// String changePercent=sections[2];
|
|
// premarketElement.Market=market;
|
|
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
|
|
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
|
|
// premarketElement.Timestamp=timestamp;
|
|
// premarketElements.Add(premarketElement);
|
|
// }
|
|
// if(0==premarketElements.Count)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, Did not find any results for request '{0}'.",strRequest));
|
|
// }
|
|
// return premarketElements;
|
|
// }
|
|
// catch(Exception exception)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
// return null;
|
|
// }
|
|
// finally
|
|
// {
|
|
// if(null!=memoryStream) memoryStream.Close();
|
|
// if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
// }
|
|
//}
|
|
//private static bool ParsePremarketElementsBySection(String strResponseString,PremarketElements premarketElements)
|
|
//{
|
|
// if(null==strResponseString)return false;
|
|
// List<String> sections=Sections.GetSections(strResponseString);
|
|
// if(null==sections||0==sections.Count)return false;
|
|
// DateTime timestamp=DateTime.Now;
|
|
// PremarketElement premarketElement=null;
|
|
|
|
// String changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",8);
|
|
// String changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",4);
|
|
// if(!(null==changePercent||null==changeValue))
|
|
// {
|
|
// premarketElement=new PremarketElement();
|
|
// premarketElement.Market="S&P";
|
|
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
|
|
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
|
|
// premarketElement.Timestamp=timestamp;
|
|
// premarketElements.Add(premarketElement);
|
|
// }
|
|
|
|
// changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",8);
|
|
// changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",4);
|
|
// if(!(null==changePercent||null==changeValue))
|
|
// {
|
|
// premarketElement=new PremarketElement();
|
|
// premarketElement.Market="DOW";
|
|
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
|
|
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
|
|
// premarketElement.Timestamp=timestamp;
|
|
// premarketElements.Add(premarketElement);
|
|
// }
|
|
|
|
// changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",8);
|
|
// changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",4);
|
|
// if(!(null==changePercent||null==changeValue))
|
|
// {
|
|
// premarketElement=new PremarketElement();
|
|
// premarketElement.Market="NASDAQ";
|
|
// premarketElement.ChangePercent=FeedParser.ParseValue(changePercent);
|
|
// premarketElement.ChangeValue=FeedParser.ParseValue(changeValue);
|
|
// premarketElement.Timestamp=timestamp;
|
|
// premarketElements.Add(premarketElement);
|
|
// }
|
|
|
|
// return true;
|
|
//}
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S *******************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static EarningsAnnouncements GetEarningsAnnouncements(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
EarningsAnnouncements earningsAnnouncements=new EarningsAnnouncements();
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.zacks.com/stock/research/").Append(symbol).Append("/earnings-announcements");
|
|
String strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
String responseString=httpNetResponse.ResponseString;
|
|
|
|
// get next earnings date
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"key-expected-earnings-data-module\"]");
|
|
if (null != table && 0 != table.Count)
|
|
{
|
|
HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
|
|
String[] requiredElements={"Report Date","Period Ending","Zacks Consensus Estimate","Earnings ESP"};
|
|
if(2==rows.Count)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[0].SelectNodes(".//th");
|
|
if (null!=headerColumns && 0!=headerColumns.Count && HeaderContains(headerColumns,requiredElements))
|
|
{
|
|
HtmlNodeCollection dataColumns = rows[1].ChildNodes;
|
|
String[] reportElements=dataColumns[1].InnerText.Split(' ');
|
|
String reportDate=reportElements[0];
|
|
if(!"NA".Equals(reportDate))
|
|
{
|
|
String[] formats=new[] {"MM/yyyy","M/yyyy"};
|
|
String periodEnding=dataColumns[3].InnerText;
|
|
EarningsAnnouncement earningsAnnouncement=new EarningsAnnouncement();
|
|
earningsAnnouncement.Symbol=symbol;
|
|
earningsAnnouncement.Date=Utility.ParseDate(reportDate);
|
|
earningsAnnouncement.PeriodEnding=DateTime.ParseExact(periodEnding, formats, new System.Globalization.CultureInfo("en-US"),DateTimeStyles.AssumeLocal);
|
|
if(reportElements[1].Equals("(AMC)"))earningsAnnouncement.Time="After Close";
|
|
else if(reportElements[1].Equals("(NONE)"))earningsAnnouncement.Time="Before Open";
|
|
else earningsAnnouncement.Time=reportElements[1];
|
|
earningsAnnouncements.Add(earningsAnnouncement);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// get historical
|
|
responseString=Utility.KeepAfter(responseString,"earnings_announcements_earnings_table");
|
|
if(null==responseString)return null;
|
|
responseString=Utility.KeepBefore(responseString,"] ]");
|
|
if(null==responseString)return null;
|
|
responseString=responseString+"]";
|
|
responseString=Utility.KeepAfter(responseString,"[ [");
|
|
if(null==responseString)return null;
|
|
responseString="["+responseString;
|
|
responseString=Utility.RemoveDivs(responseString);
|
|
if(null==responseString)return null;
|
|
String[] items=responseString.Split(']');
|
|
for(int index=0;index<items.Length;index++)
|
|
{
|
|
EarningsAnnouncement earningsAnnouncement=new EarningsAnnouncement();
|
|
String itemText=items[index];
|
|
if(null==itemText||0==itemText.Length)continue;
|
|
itemText=Utility.BetweenString(itemText,"[",null);
|
|
itemText=itemText.Replace("\"",null);
|
|
String[] columnData=itemText.Split(',');
|
|
for(int columnIndex=0;columnIndex<columnData.Length;columnIndex++)columnData[columnIndex]=columnData[columnIndex].Trim();
|
|
if(null==columnData||7!=columnData.Length)continue;
|
|
earningsAnnouncement.Symbol=symbol;
|
|
earningsAnnouncement.Date=Utility.ParseDate(columnData[0]);
|
|
|
|
String[] periodEndingItems=columnData[1].Split('/');
|
|
if(null!=periodEndingItems&&2==periodEndingItems.Length)
|
|
{
|
|
int month=(int)FeedParser.ParseValueLong(periodEndingItems[0]);
|
|
int year=(int)FeedParser.ParseValueLong(periodEndingItems[1]);
|
|
earningsAnnouncement.PeriodEnding=new DateTime(year,month,DateTime.DaysInMonth(year,month));
|
|
}
|
|
earningsAnnouncement.Estimate=FeedParser.ParseValue(columnData[2]);
|
|
earningsAnnouncement.Reported=FeedParser.ParseValue(columnData[3]);
|
|
earningsAnnouncement.Surprise=FeedParser.ParseValue(columnData[4]);
|
|
earningsAnnouncement.SurprisePct=FeedParser.ParseValue(columnData[5]);
|
|
earningsAnnouncement.Time=columnData[6];
|
|
earningsAnnouncements.Add(earningsAnnouncement);
|
|
}
|
|
return earningsAnnouncements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream)memoryStream.Dispose();
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************** Z A C K S R A N K ******************************************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static ZacksRank GetZacksRank(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.zacks.com/stock/quote/");
|
|
sb.Append(symbol);
|
|
String strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
if(httpNetResponse.StatusCode.Equals("Forbidden"))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
|
|
try{Thread.Sleep(15000);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
else return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"rank_view\"]");
|
|
if(null==table||0==table.Count)return null;
|
|
String rank=Utility.BetweenString(table[0].InnerHtml,null,"<").Trim();
|
|
if(null==rank||"".Equals(rank))return null;
|
|
ZacksRank zacksRank=new ZacksRank();
|
|
zacksRank.Symbol=symbol;
|
|
zacksRank.Date=DateTime.Now.Date;
|
|
zacksRank.Rank=rank;
|
|
return zacksRank;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** S P L I T S F R O M E O D D A T A **********************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Splits GetSplits()
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
Splits splits=new Splits();
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("http://eoddata.com/splits.aspx");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Getting splits {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamZIPV2(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
MemoryStream memoryStream=new MemoryStream(httpNetResponse.ResponseStream.GetBuffer());
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quotes\"]");
|
|
if (null == divSection || 0 == divSection.Count) return null;
|
|
HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
|
|
for (int index = 0; index < rows.Count; index++)
|
|
{
|
|
Split split=new Split();
|
|
HtmlNodeCollection headerColumns = rows[index].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td");
|
|
if(dataColumns.Count<4)continue;
|
|
if(null!=dataColumns[0])split.Exchange=dataColumns[0].InnerText.Trim().ToUpper(); // exchange
|
|
if(null!=dataColumns[1])split.Symbol=dataColumns[1].InnerText.Trim().ToUpper();
|
|
if(null!=dataColumns[2])split.EffectiveDate=Utility.ParseDate(dataColumns[2].InnerText.Trim().ToUpper());
|
|
if(null!=dataColumns[3])split.StrRatio=dataColumns[3].InnerText.Trim().ToUpper();
|
|
splits.Add(split);
|
|
}
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
return splits;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** G D P P E R C A P I T A D A T A W O R L D B A N K *************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static EconomicIndicators GetGDPPerCapita(bool debug=false)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
// Retrieve compressed CSV from World Bank web site and write to disk
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
String currentWorkingDirectory=Directory.GetCurrentDirectory();
|
|
String strExtractFolder=currentWorkingDirectory+"\\"+"extracts";
|
|
String strFileName="API_NY.GDP.MKTP.CD_DS2_V2_USD.zip";
|
|
String strPathFileName=currentWorkingDirectory+"\\"+strFileName;
|
|
sb.Append("http://api.worldbank.org/v2/en/indicator/NY.GDP.MKTP.CD?downloadformat=csv");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Downloading {0} from {1}",strFileName,strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamZIP(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
if(File.Exists(strPathFileName))File.Delete(strPathFileName);
|
|
FileStream outStream=new FileStream(strPathFileName,FileMode.Create);
|
|
byte[] streamBytes=httpNetResponse.ResponseStream.GetBuffer();
|
|
outStream.Write(streamBytes,0,streamBytes.Length);
|
|
outStream.Flush();
|
|
outStream.Close();
|
|
return EconomicIndicators.FromZipFile(strPathFileName,strExtractFolder,debug);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** D I V I D E N D H I S T O R Y N A S D A Q ***********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static DividendHistory GetDividendHistory(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
DividendHistory dividendHistory = new DividendHistory();
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
CookieCollection cookieCollection = new CookieCollection();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetDividendHistory");
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf");
|
|
else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks");
|
|
strRequest = sb.ToString();
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest));
|
|
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy);
|
|
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy);
|
|
//httpNetResponse = HttpNetRequest.GetRequestNoEncodingV7(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
if(httpNetResponse.StatusCode.Equals(System.Net.HttpStatusCode.Forbidden))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
|
|
try{Thread.Sleep(15000);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest, cookieCollection, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
else return null;
|
|
}
|
|
|
|
int groupBy=6;
|
|
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
if(null==items || 0==items.Count || items.Count<groupBy)return dividendHistory;
|
|
for(int index=0;index<items.Count;index+=groupBy)
|
|
{
|
|
if(items.Count<index+groupBy)break;
|
|
String strExEffDate = items[index][1];
|
|
String strType = items[index+1][1];
|
|
String strCashAmount = items[index+2][1];
|
|
String strDeclarationDate = items[index+3][1];
|
|
String strRecordDate = items[index+4][1];
|
|
String strPaymentDate = items[index+5][1];
|
|
|
|
try
|
|
{
|
|
if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
|
|
if("EX-DIVIDEND DATE".Equals(strExEffDate, StringComparison.InvariantCultureIgnoreCase))
|
|
{
|
|
strExEffDate = items[index+1][1];
|
|
strType = items[index+4][1];
|
|
strCashAmount = items[index+5][1];
|
|
strDeclarationDate = items[index+1][1];
|
|
strRecordDate = items[index+1][1];
|
|
strPaymentDate = items[index+1][1];
|
|
}
|
|
if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
|
|
DividendHistoryItem dividendHistoryItem = new DividendHistoryItem();
|
|
dividendHistoryItem.Modified=DateTime.Now;
|
|
dividendHistoryItem.Symbol = symbol.ToUpper();
|
|
try{dividendHistoryItem.DivExDate = Utility.ParseDate(strExEffDate);}
|
|
catch(Exception exception){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception parsing value '{0}', {1}",strExEffDate,exception.ToString()));continue;}
|
|
dividendHistoryItem.DividendType = strType;
|
|
if (!"--".Equals(strCashAmount)&&!"".Equals(strCashAmount)) dividendHistoryItem.CashAmount = FeedParser.ParseValue(strCashAmount);
|
|
else dividendHistoryItem.CashAmount = null;
|
|
if (!"--".Equals(strDeclarationDate)&&!"N/A".Equals(strDeclarationDate)) dividendHistoryItem.DeclarationDate = DateTime.Parse(strDeclarationDate);
|
|
else dividendHistoryItem.DeclarationDate=null;
|
|
if (!"--".Equals(strRecordDate)&&!"N/A".Equals(strRecordDate)) dividendHistoryItem.RecordDate = DateTime.Parse(strRecordDate);
|
|
if (!"--".Equals(strPaymentDate)&&!"N/A".Equals(strPaymentDate)) dividendHistoryItem.PaymentDate = DateTime.Parse(strPaymentDate);
|
|
else dividendHistoryItem.PaymentDate = null;
|
|
|
|
if(null==dividendHistoryItem.CashAmount || double.IsNaN(dividendHistoryItem.CashAmount.Value))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Encountered invalid CashAmount in DividendHistory record for {0} ",symbol));
|
|
continue;
|
|
}
|
|
if(dividendHistoryItem.DividendType.Equals("ANNUAL DIVIDEND"))dividendHistoryItem.DividendType="Cash";
|
|
dividendHistory.Add(dividendHistoryItem);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Exception:{0}",exception.ToString()));
|
|
continue;
|
|
}
|
|
}
|
|
return dividendHistory;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return dividendHistory;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** O P T I O N S N A S D A Q *****************************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
// month=0=current month
|
|
// month=1=current month
|
|
// month=2=current month
|
|
// month=3=current month
|
|
// month=4=current month
|
|
// month=5=current month
|
|
// http://www.nasdaq.com/symbol/msft/option-chain?money=all&dateindex=-1
|
|
public static Options GetOptions(String symbol)
|
|
{
|
|
Options options = new Options();
|
|
Dictionary<int,Option> optionsDictionary=new Dictionary<int,Option>();
|
|
|
|
for (int dateIndex = 0; dateIndex < 5 ; dateIndex++)
|
|
{
|
|
bool morePages=true;
|
|
for(int pageIndex=1;pageIndex<10 && morePages;pageIndex++)
|
|
{
|
|
Options datedOptions = GetOptions(symbol, dateIndex,pageIndex);
|
|
if (null == datedOptions || 0 == datedOptions.Count) continue;
|
|
for(int optionIndex=0;optionIndex<datedOptions.Count && morePages;optionIndex++)
|
|
{
|
|
Option option=datedOptions[optionIndex];
|
|
if(!optionsDictionary.ContainsKey(option.GetHashCode()))
|
|
{
|
|
optionsDictionary.Add(option.GetHashCode(),option);
|
|
options.Add(option);
|
|
}
|
|
else
|
|
{
|
|
morePages=false;
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
private static Options GetOptions(String symbol, int dateIndex,int page)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
Options options = new Options();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
// sb.Append("http://www.nasdaq.com/symbol/").Append(symbol).Append("/option-chain?money=all&dateindex=").Append(dateIndex).Append("&page=").Append(page);
|
|
sb.Append("http://old.nasdaq.com/symbol/").Append(symbol).Append("/option-chain?money=all&dateindex=").Append(dateIndex).Append("&page=").Append(page);
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV2(strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"OptionsChain-chart borderAll thin\"]");
|
|
if (null == divSection || 0 == divSection.Count) return null;
|
|
HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
|
|
if (null == rows) return null;
|
|
for (int index = 0; index < rows.Count; index++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[index].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td");
|
|
Option callOption = new Option();
|
|
Option putOption = new Option();
|
|
callOption.Expiration=FeedParser.ParseValueDateTimeMonthFormat(dataColumns[0].InnerText);
|
|
callOption.Last = FeedParser.ParseValue(dataColumns[1].InnerText);
|
|
callOption.Change = FeedParser.ParseValue(dataColumns[2].InnerText);
|
|
callOption.Bid = FeedParser.ParseValue(dataColumns[3].InnerText);
|
|
callOption.Ask = FeedParser.ParseValue(dataColumns[4].InnerText);
|
|
callOption.Volume = FeedParser.ParseValueLong(dataColumns[5].InnerText);
|
|
callOption.OpenInterest = FeedParser.ParseValue(dataColumns[6].InnerText);
|
|
callOption.Symbol=dataColumns[7].InnerText.Trim().ToUpper();
|
|
callOption.Strike = FeedParser.ParseValue(dataColumns[8].InnerText);
|
|
callOption.Type = OptionTypeEnum.CallOption;
|
|
if(!double.IsNaN(callOption.Bid)&&0!=callOption.Strike)options.Add(callOption);
|
|
putOption.Symbol = dataColumns[7].InnerText.Trim().ToUpper();
|
|
putOption.Strike = FeedParser.ParseValue(dataColumns[8].InnerText);
|
|
putOption.Expiration = FeedParser.ParseValueDateTimeMonthFormat(dataColumns[9].InnerText);
|
|
putOption.Last = FeedParser.ParseValue(dataColumns[10].InnerText);
|
|
putOption.Change = FeedParser.ParseValue(dataColumns[11].InnerText);
|
|
putOption.Bid = FeedParser.ParseValue(dataColumns[12].InnerText);
|
|
putOption.Ask = FeedParser.ParseValue(dataColumns[13].InnerText);
|
|
putOption.Volume = FeedParser.ParseValueLong(dataColumns[14].InnerText);
|
|
putOption.OpenInterest = FeedParser.ParseValue(dataColumns[15].InnerText);
|
|
putOption.Type = OptionTypeEnum.PutOption;
|
|
if (!double.IsNaN(putOption.Bid)&&0!=putOption.Strike) options.Add(putOption);
|
|
}
|
|
return options;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** A N A L Y S T R A T I N G S - B R I E F I N G . C O M *************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Upgrades/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/downgrades/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Reiterated/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Initiated/2014/08/04
|
|
// The feed no longer accepts a date parameter
|
|
public static AnalystRatings GetLatestAnalystRatings()
|
|
{
|
|
AnalystRatings analystRatings = GetLatestAnalystRatingsBriefing();
|
|
return analystRatings;
|
|
}
|
|
public static AnalystRatings GetLatestAnalystRatingsBriefing()
|
|
{
|
|
Dictionary<String, String> uniqueSymbols = new Dictionary<String, String>();
|
|
AnalystRatings analystRatings = new AnalystRatings();
|
|
AnalystRatings ratings = null;
|
|
|
|
ratings = GetLatestAnalystRatingsBriefing("Upgrades");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Downgrades");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Reiterated");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Initiated");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
return analystRatings;
|
|
}
|
|
private static AnalystRatings GetLatestAnalystRatingsBriefing(String type)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse = null;
|
|
AnalystRatings analystRatings = new AnalystRatings();
|
|
try
|
|
{
|
|
String strRequest = null;
|
|
if ("Upgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/upgrade.htm";
|
|
else if ("Downgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/downgrade.htm";
|
|
else if ("Reiterated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/reiterated.htm";
|
|
else if ("Initiated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/initiated.htm";
|
|
else return null;
|
|
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if (!httpNetResponse.Success || null == httpNetResponse.ResponseString)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSectionDate = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calDATE\"]");
|
|
HtmlNodeCollection divSectionHeader = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calHDR2\"]");
|
|
HtmlNodeCollection divSections = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calEVENT\"]");
|
|
|
|
if (null == divSectionDate || 0 == divSectionDate.Count) return null;
|
|
if (null == divSectionHeader || 0 == divSectionHeader.Count) return null;
|
|
if (null == divSections || 0 == divSections.Count) return null;
|
|
|
|
DateTime ratingsDate = Utility.ParseDate(divSectionDate[0].InnerText);
|
|
String ratingsType = divSectionHeader[0].InnerText;
|
|
if (!ratingsType.StartsWith(type.ToUpper())) return null;
|
|
|
|
// calDATA
|
|
for (int index = 0; index < divSections.Count; index++)
|
|
{
|
|
|
|
List<String> items = Sections.GetSections(divSections[index].InnerHtml);
|
|
if (null == items || 0 == items.Count) continue;
|
|
|
|
AnalystRating analystRating = new AnalystRating();
|
|
analystRating.CompanyName = items.Count > 0 ? items[0] : null;
|
|
analystRating.Date = ratingsDate;
|
|
analystRating.Symbol = items.Count > 4 ? items[4] : null;
|
|
analystRating.Type = type;
|
|
if (items.Count > 10 && items[10].StartsWith("Brokerage Firm")) analystRating.BrokerageFirm = items.Count > 12 ? items[12] : null;
|
|
if (items.Count > 22 && items[22].StartsWith("Ratings Change")) analystRating.RatingsChange = items.Count > 24 ? items[24] : null;
|
|
if (null != analystRating.RatingsChange & analystRating.RatingsChange.Contains("»")) analystRating.RatingsChange = analystRating.RatingsChange.Replace("»", "->");
|
|
if (items.Count > 28 && items[28].StartsWith("Price Tgt"))
|
|
{
|
|
String priceTarget = items.Count > 31 ? items[31] : null;
|
|
if (null != priceTarget && priceTarget.Contains("»")) priceTarget = priceTarget.Replace("»", "-");
|
|
if (priceTarget.Contains("-"))
|
|
{
|
|
String[] prices = priceTarget.Split('-');
|
|
priceTarget = prices[1];
|
|
}
|
|
analystRating.PriceTarget = FeedParser.ParseValue(priceTarget);
|
|
}
|
|
analystRatings.Add(analystRating);
|
|
}
|
|
return analystRatings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return analystRatings;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************************
|
|
// ****************************************************************************** A N A L Y S T R A T I N G S M A R K E T B E A T *******************************************************
|
|
// ******************************************************************************************************************************************************************************************
|
|
public static AnalystRatings GetAnalystRatingsMarketBeat(String symbol)
|
|
{
|
|
AnalystRatings analystRatings=new AnalystRatings();
|
|
String nasdaq="NASDAQ";
|
|
String nyse="NYSE";
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
HtmlNodeCollection ratingsTable=null;
|
|
ratingsTable=GetRatingsTableOnMarketBeat(symbol,nyse);
|
|
if(null==ratingsTable) ratingsTable=GetRatingsTableOnMarketBeat(symbol,nasdaq);
|
|
if(null==ratingsTable)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No ratings for {0} on MarketBeat",symbol));
|
|
return null;
|
|
}
|
|
HtmlNodeCollection header=ratingsTable[0].SelectNodes(".//th");
|
|
string[] requiredElements=new string[]{"Date","Brokerage","Action","Rating","Price Target","Details"};
|
|
if(!HeaderContains(header,requiredElements))return null;
|
|
HtmlNodeCollection data=ratingsTable[0].SelectNodes(".//tr//td");
|
|
if(null==data)return analystRatings;
|
|
for(int index=0,rowIndex=0;index<data.Count&&index+(header.Count-1)<data.Count;index+=header.Count,rowIndex++)
|
|
{
|
|
String strDate=GetElementData(header,data,rowIndex,"Date");
|
|
if(null==strDate || "".Equals(strDate))break;
|
|
String strBrokerage=GetElementData(header,data,rowIndex,"Brokerage");
|
|
if(null!=strBrokerage)strBrokerage=strBrokerage.Trim();
|
|
String strAction=GetElementData(header,data,rowIndex,"Action");
|
|
String strRating=GetElementData(header,data,rowIndex,"Rating");
|
|
String strPriceTarget=GetElementData(header,data,rowIndex,"Price Target");
|
|
AnalystRating analystRating=new AnalystRating();
|
|
analystRating.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;
|
|
analystRating.Date=Utility.ParseDate(strDate);
|
|
analystRating.Symbol=symbol;
|
|
analystRating.Type=strAction;
|
|
analystRating.BrokerageFirm=strBrokerage;
|
|
if(null!=strRating&strRating.Contains("➝")) strRating=strRating.Replace("➝","->");
|
|
analystRating.RatingsChange=strRating;
|
|
if(null!=strPriceTarget)
|
|
{
|
|
|
|
if(strPriceTarget.Contains("➝")) strPriceTarget=strPriceTarget.Replace("➝","-");
|
|
if(strPriceTarget.Contains("-"))
|
|
{
|
|
String[] prices=strPriceTarget.Split('-');
|
|
strPriceTarget=prices[1];
|
|
}
|
|
analystRating.PriceTarget=FeedParser.ParseValue(strPriceTarget);
|
|
}
|
|
if(analystRating.Type.Equals("Boost Price Target")) analystRating.Type="Upgrades";
|
|
if(analystRating.Type.Equals("Upgrade")) analystRating.Type="Upgrades";
|
|
if(analystRating.Type.Equals("Downgrade")) analystRating.Type="Downgrades";
|
|
if(analystRating.Type.StartsWith("Reiterated")) analystRating.Type="Reiterated";
|
|
if(analystRating.Type.StartsWith("Initiated")) analystRating.Type="Initiated";
|
|
if(analystRating.Type.StartsWith("Set Price Target"))analystRating.Type="Initiated";
|
|
if(!analystRating.Type.Equals("Upgrades")&&!analystRating.Type.Equals("Downgrades")&&!analystRating.Type.Equals("Reiterated")&&!analystRating.Type.Equals("Initiated")) continue;
|
|
analystRatings.Add(analystRating);
|
|
}
|
|
return new AnalystRatings(analystRatings.Distinct().ToList());
|
|
// return analystRatings;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return analystRatings;
|
|
}
|
|
finally
|
|
{
|
|
}
|
|
}
|
|
public static bool HeaderContains(HtmlNodeCollection header,String[] requiredElements)
|
|
{
|
|
List<String> headerElements=new List<String>();
|
|
if(null==header)return false;
|
|
for(int index=0;index<header.Count;index++)
|
|
{
|
|
headerElements.Add(header[index].InnerText);
|
|
}
|
|
List<String> exceptions=requiredElements.Except(headerElements).ToList();
|
|
return exceptions.Count()>0?false:true;
|
|
}
|
|
public static int HeaderIndexOf(HtmlNodeCollection header,String identifier)
|
|
{
|
|
if(null==header)return -1;
|
|
for(int index=0;index<header.Count;index++)
|
|
{
|
|
if(header[index].InnerText.Equals(identifier))return index;
|
|
}
|
|
return -1;
|
|
}
|
|
public static String GetElementData(HtmlNodeCollection header,HtmlNodeCollection data,int row,String identifier)
|
|
{
|
|
int index=HeaderIndexOf(header,identifier);
|
|
if(-1==index)return null;
|
|
int indexer=(header.Count*row)+index;
|
|
List<String> sections=Sections.GetSections(data[indexer].InnerHtml);
|
|
if(null==sections || 0==sections.Count)return data[indexer].InnerText;
|
|
return sections[0];
|
|
}
|
|
private static HtmlNodeCollection GetRatingsTableOnMarketBeat(String symbol,String exchange)
|
|
{
|
|
MemoryStream memoryStream=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
String strRequest=null;
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3(strRequest, "www.marketbeat.com");
|
|
if(!httpNetResponse.Success||String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream=new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
|
|
HtmlNodeCollection ratingsTable=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"scroll-table sort-table\"]");
|
|
if(null==ratingsTable||0==ratingsTable.Count) return null;
|
|
return ratingsTable;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** C I K C O D E S - S E C . G O V ***********************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static String GetCIK(String symbol)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany");
|
|
strRequest = sb.ToString();
|
|
// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"companyInfo\"]"); // yfnc_tablehead1,yfnc_modtitle1,yfnc_tabledata1
|
|
if (null == rows || 1!=rows.Count)
|
|
{
|
|
if (null == rows) MDTrace.WriteLine(LogLevel.DEBUG,"[GetCIK] Received no rows for '" + symbol + "'");
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Expected 1 row, got " + rows.Count + " for '" + symbol + "'");
|
|
return null;
|
|
}
|
|
HtmlNode htmlNodeTitle = rows[0];
|
|
HtmlNodeCollection nodes = htmlNodeTitle.SelectNodes(".//a");
|
|
if (null == nodes || 0==nodes.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Expected <a> tag for symbol '" + symbol + "'");
|
|
return null;
|
|
}
|
|
String cik = null;
|
|
String[] strings = nodes[0].InnerText.Split(' ');
|
|
if (strings.Length >= 1) cik = strings[0];
|
|
return cik;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************ S E C F I L I N G S - F O R M 4 & F O R M 5 - F O R I N S I D E R T R A N S A C T I O N S S E C . G O V ***************************S
|
|
// ***************************************************************************************************************************************************************************************
|
|
// public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=7)
|
|
public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=30)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
|
|
DateTime minFilingDate=DateTime.Now;
|
|
int maxFilings=80;
|
|
bool continuationFlag=true;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} dayCount:{1}",symbol,timePeriodDays));
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
String cik=PricingDA.GetCIKForSymbol(symbol);
|
|
if(null==cik)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: No CIK for symbol {0}",symbol));
|
|
return null;
|
|
}
|
|
minFilingDate=dateGenerator.DaysAddActual(minFilingDate,Math.Abs(timePeriodDays)*-1);
|
|
SECFilings secFilings = new SECFilings();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
for (int row = 0; row < rows.Count && continuationFlag; row ++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
|
|
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
|
|
{
|
|
SECFiling innerFiling = innerCollection[index];
|
|
if(innerFiling.FilingDate<minFilingDate)
|
|
{
|
|
continuationFlag=false;
|
|
break;
|
|
}
|
|
if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
|
|
secFilings.Add(innerFiling);
|
|
}
|
|
}
|
|
return InsiderTransactionsParser.GetInstance().Parse(secFilings);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** S E C F I L I N G S - S E C . G O V ********************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
SECFilings secFilings = new SECFilings();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
for (int row = 0; row < rows.Count; row ++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
|
|
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
|
|
{
|
|
SECFiling innerFiling = innerCollection[index];
|
|
secFilings.Add(innerFiling);
|
|
}
|
|
}
|
|
return secFilings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// Get the filing document in XML format
|
|
private static SECFilings GetSECFilingDocuments(String symbol,DateTime filingDate,String secFilingDocumentUrl)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
SECFilings secFilings = new SECFilings();
|
|
String secFilingDocument = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Retrieving document at {0}]",secFilingDocumentUrl));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
|
|
try { Thread.Sleep(1000); }catch(Exception) { ;}
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Request:{0} failed with status {1}]",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"secNum\"]");
|
|
String secAccessionNumber = secNodes[0].InnerText;
|
|
secAccessionNumber = secAccessionNumber.Replace("\n","");
|
|
secAccessionNumber = secAccessionNumber.Trim();
|
|
if (secAccessionNumber.Contains("SEC Accession No.")) secAccessionNumber = secAccessionNumber.Substring(18, secAccessionNumber.Length - 18);
|
|
else return null;
|
|
secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"formName\"]");
|
|
String formName = secNodes[0].InnerText;
|
|
formName = formName.Replace("\n", "");
|
|
formName = formName.Trim();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments[Examining form {0}]",formName));
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
httpNetResponse.Dispose();
|
|
// go through the rows in the table file, searching for the different submission form content
|
|
int sequence = 0;
|
|
for (int row = 0; row < rows.Count; row++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
String document = dataColumns[2].InnerText.Trim();
|
|
String type = dataColumns[3].InnerText.Trim();
|
|
if(null!=type && type.Equals("GRAPHIC"))continue;
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("SEQ:{0},DOC:{1},TYPE:{2}",sequence,document,type));
|
|
if (document.Contains(".htm"))
|
|
{
|
|
HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",secFilingDocumentXmlUrl));
|
|
try{Thread.Sleep(1000);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
continue;
|
|
}
|
|
secFilingDocument =httpNetResponse.ResponseString;
|
|
if (null == secFilingDocument) continue;
|
|
SECFiling secFiling = new SECFiling();
|
|
secFiling.SECAccessionNumber = secAccessionNumber;
|
|
secFiling.Sequence = sequence++;
|
|
secFiling.Symbol = symbol;
|
|
secFiling.Form = type;
|
|
secFiling.Description = formName;
|
|
secFiling.FilingDate = filingDate;
|
|
secFiling.FileNumber = dataColumns[4].InnerText.Trim();
|
|
secFiling.SecFilingUrl = secFilingDocumentXmlUrl;
|
|
secFiling.FormText = secFilingDocument;
|
|
secFiling.FormText = secFiling.FormText.Replace("\n","");
|
|
if (null == secFiling.FormText) continue;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilingDocuments. Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString()));
|
|
secFilings.Add(secFiling);
|
|
httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
return secFilings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream = null;
|
|
}
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** Y I E L D C U R V E - T R E A S U R Y . G O V **************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static YieldCurve GetYieldCurve(int year)
|
|
{
|
|
YieldCurve yieldCurve = new YieldCurve();
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/pages/xml?data=daily_treasury_yield_curve&field_tdr_date_value=").Append(year);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
XElement po = XElement.Parse(httpNetResponse.ResponseString);
|
|
IEnumerable<XElement> childElements = from el in po.Elements() where el.Name.LocalName.Equals("entry") select el;
|
|
foreach (XElement element in childElements)
|
|
{
|
|
IEnumerable<XElement> childElementsL1 = from el in element.Elements() select el;
|
|
foreach (XElement elementL1 in childElementsL1)
|
|
{
|
|
if(elementL1.Name.LocalName.Equals("content"))
|
|
{
|
|
IEnumerable<XElement> elementsL2=elementL1.Elements();
|
|
if(!elementsL2.FirstOrDefault().Name.LocalName.Equals("properties"))continue;
|
|
IEnumerable<XElement> elementsL3=elementsL2.FirstOrDefault().Elements();
|
|
YieldCurveData yieldCurveData = new YieldCurveData();
|
|
yieldCurveData.Date=Utility.Epoch;
|
|
foreach (XElement elementL3 in elementsL3)
|
|
{
|
|
if(elementL3.Name.LocalName.EndsWith("NEW_DATE")) yieldCurveData.Date=Utility.ParseDate(Utility.RemoveAfter(elementL3.Value,'T'));
|
|
else if(elementL3.Name.LocalName.EndsWith("1MONTH"))yieldCurveData.Mo1=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("2MONTH"))yieldCurveData.Mo2=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("3MONTH"))yieldCurveData.Mo3=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("6MONTH"))yieldCurveData.Mo6=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("1YEAR"))yieldCurveData.Yr1=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("2YEAR"))yieldCurveData.Yr2=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("3YEAR"))yieldCurveData.Yr3=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("5YEAR"))yieldCurveData.Yr5=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("7YEAR"))yieldCurveData.Yr7=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("10YEAR"))yieldCurveData.Yr10=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("20YEAR"))yieldCurveData.Yr20=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("30YEAR"))yieldCurveData.Yr30=FeedParser.ParseValue(elementL3.Value);
|
|
}
|
|
if(yieldCurveData.IsValid())yieldCurve.Add(yieldCurveData);
|
|
}
|
|
}
|
|
}
|
|
yieldCurve=new YieldCurve(yieldCurve.Distinct().ToList());
|
|
return yieldCurve;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return yieldCurve;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static ETFHoldings GetETFHoldings(String etfSymbol)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
ETFHoldings etfHoldings = new ETFHoldings();
|
|
HttpNetResponse httpNetResponse=null;
|
|
DateTime modified=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
etfSymbol = etfSymbol.ToUpper();
|
|
sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings");
|
|
strRequest = sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
|
|
// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
|
|
strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\"");
|
|
List<List<String>> items=LocateJSONItems(strResponse);
|
|
if(null==items||0==items.Count)return null;
|
|
for(int index=0;index<items.Count;index+=3)
|
|
{
|
|
if(index+3>items.Count)break;
|
|
List<String> symbolPair=items[index];
|
|
List<String> holdingNamePair=items[index+1];
|
|
List<String> percentOfAssetsPair=items[index+2];
|
|
if(!symbolPair[0].Equals("symbol"))continue;
|
|
if(!holdingNamePair[0].Equals("holdingName"))continue;
|
|
if(!percentOfAssetsPair[0].Equals("fmt"))continue;
|
|
if(String.IsNullOrEmpty(symbolPair[1]))continue;
|
|
ETFHolding etfHolding = new ETFHolding();
|
|
etfHolding.ETFSymbol = etfSymbol;
|
|
etfHolding.HoldingSymbolShareClass=null;
|
|
etfHolding.HoldingCompanyName=holdingNamePair[1];
|
|
etfHolding.HoldingSymbolShareClass=symbolPair[1];
|
|
etfHolding.HoldingSymbol=symbolPair[1];
|
|
etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]);
|
|
etfHolding.Modified=modified;
|
|
etfHoldings.Add(etfHolding);
|
|
}
|
|
return etfHoldings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// **************************************************************************C O M P A N Y P R O F I L E **********************************************************************
|
|
// ***************************************************************** P R O F I L E : M O R N I N G S T A R *******************************************************************
|
|
// ***************************************************************** D E S C R I P T I O N : R E U T E R S ********************************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static CompanyProfile GetCompanyProfile(String symbol)
|
|
{
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnys";
|
|
CompanyProfile companyProfile = null;
|
|
companyProfile = GetCompanyProfileYahoo(symbol);
|
|
if (null==companyProfile)companyProfile=GetCompanyProfileMorningStar(symbol,nasdaq);
|
|
if (null == companyProfile) companyProfile = GetCompanyProfileMorningStar(symbol, nyse);
|
|
return companyProfile;
|
|
}
|
|
// GetCompanyProfile - DataSource Yahoo Finance
|
|
public static CompanyProfile GetCompanyProfileYahoo(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyProfileYahoo");
|
|
sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile");
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
String strIndustry=Sections.LocateItem(httpNetResponse.ResponseString,"Industry",4);
|
|
String strSector=Sections.LocateItem(httpNetResponse.ResponseString,"Sector",4);
|
|
if(null==strSector)strSector=Sections.LocateItem(httpNetResponse.ResponseString,"Sector(s)",4);
|
|
|
|
if(null!=strIndustry && strIndustry.Contains(Constants.CONST_QUESTION))strIndustry=strIndustry.Replace(Constants.CONST_QUESTION," - ");
|
|
if(null!=strSector && strSector.Contains(Constants.CONST_QUESTION))strSector=strSector.Replace(Constants.CONST_QUESTION," - ");
|
|
|
|
String strDescription=Sections.LocateItem(httpNetResponse.ResponseString,"Description",4);
|
|
if(null==strIndustry && null==strSector && null==strDescription)return null;
|
|
CompanyProfile companyProfile = new CompanyProfile();
|
|
companyProfile.Symbol = symbol;
|
|
companyProfile.Sector = strSector;
|
|
companyProfile.Industry = strIndustry;
|
|
companyProfile.Description=strDescription;
|
|
if(null!=companyProfile.Sector)companyProfile.Sector=companyProfile.Sector.Trim();
|
|
if(null!=companyProfile.Industry)companyProfile.Industry=companyProfile.Industry.Trim();
|
|
if(null!=companyProfile.Description)companyProfile.Description=Utility.RemoveHtml(companyProfile.Description.Trim());
|
|
return companyProfile;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
public static CompanyProfile GetCompanyProfileMorningStar(String symbol,String exchange)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyProfileMorningStar");
|
|
sb.Append(String.Format("https://www.morningstar.com/stocks/{0}/{1}/quote",exchange,symbol));
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
String strSector=null;
|
|
String strIndustry=null;
|
|
String strDescription=null;
|
|
int itemIndex=0;
|
|
List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);
|
|
|
|
bool found=Sections.FindInSections(sections,"Sector",0,ref itemIndex);
|
|
if(found)strSector=sections[itemIndex+3];
|
|
|
|
found=Sections.FindInSections(sections,"Industry",itemIndex,ref itemIndex);
|
|
if(found)strIndustry=sections[itemIndex+3];
|
|
|
|
found=Sections.FindInSections(sections,"Business Description",itemIndex,ref itemIndex);
|
|
if(found)strDescription=sections[itemIndex+2];
|
|
|
|
if(null==strIndustry && null==strSector)return null;
|
|
strSector=strSector.Trim();
|
|
strIndustry=strIndustry.Trim();
|
|
CompanyProfile companyProfile = new CompanyProfile();
|
|
companyProfile.Symbol = symbol;
|
|
companyProfile.Sector = strSector;
|
|
companyProfile.Industry = strIndustry;
|
|
companyProfile.Description=strDescription;
|
|
if(null!=companyProfile.Sector)companyProfile.Sector=companyProfile.Sector.Trim();
|
|
if(null!=companyProfile.Industry)companyProfile.Industry=companyProfile.Industry.Trim();
|
|
if(null!=strDescription)companyProfile.Description=companyProfile.Description.Trim();
|
|
|
|
if(null!=companyProfile.Industry && companyProfile.Industry.Contains(Constants.CONST_QUESTION))companyProfile.Industry=companyProfile.Industry.Replace(Constants.CONST_QUESTION," - ");
|
|
if(null!=companyProfile.Sector && companyProfile.Sector.Contains(Constants.CONST_QUESTION))companyProfile.Sector=companyProfile.Sector.Replace(Constants.CONST_QUESTION," - ");
|
|
// if(companyProfile.Sector.Equals(Constants.CONST_QUESTION) || companyProfile.Industry.Equals(Constants.CONST_QUESTION) || companyProfile.Description.Equals(Constants.CONST_QUESTION))return null;
|
|
if((null!=companyProfile.Sector && companyProfile.Sector.Equals(Constants.CONST_QUESTION))
|
|
|| (null!=companyProfile.Industry && companyProfile.Industry.Equals(Constants.CONST_QUESTION))
|
|
|| (null!=companyProfile.Description && companyProfile.Description.Equals(Constants.CONST_QUESTION)))return null;
|
|
return companyProfile;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** H E A D L I N E S - S E E K I N G A L P H A ***********************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
/*
|
|
public static Headlines GetCompanyHeadlines(String symbol)
|
|
{
|
|
Headlines headlines=GetCompanyHeadlinesEx(symbol);
|
|
return headlines;
|
|
}
|
|
public static Headlines GetCompanyHeadlinesEx(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
Headlines headlines=new Headlines();
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
|
|
CookieCollection cookieCollection=new CookieCollection();
|
|
httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.seekingalpha.com",cookieCollection);
|
|
Thread.Sleep(250);
|
|
sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlines");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3B(strRequest,cookieCollection,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
|
|
if(null==keyValuePairs)return null;
|
|
|
|
foreach (KeyValue keyValue in keyValuePairs)
|
|
{
|
|
Headline headline=null;
|
|
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
|
|
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
|
|
headline.Entry=headline.Entry.Replace("\\"," ");
|
|
headline.Entry=headline.Entry.Trim();
|
|
headline.Entry=Uri.UnescapeDataString(headline.Entry);
|
|
headline.Source = "Seeking Alpha";
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
*/
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** H E A D L I N E S - M A R K E T W A T C H *************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static Headlines GetCompanyHeadlinesMarketWatch(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
Headlines headlines=new Headlines();
|
|
DateTime modified=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
MemoryStream memoryStream=null;
|
|
CookieCollection cookieCollection=new CookieCollection();
|
|
httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.marketwatch.com",cookieCollection);
|
|
Thread.Sleep(250);
|
|
sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol).Append("?mod=search_symbol");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesMarketWatch");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream=new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSections=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"article__content\"]");
|
|
|
|
if(null == divSections)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for request {0} ",strRequest));
|
|
return headlines;
|
|
}
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
|
|
|
|
for(int nodeIndex=0;nodeIndex<divSections.Count;nodeIndex++)
|
|
{
|
|
HtmlNode node=divSections[nodeIndex];
|
|
|
|
byte[] bytes=Encoding.ASCII.GetBytes(node.InnerHtml);
|
|
memoryStream=new MemoryStream(bytes);
|
|
HtmlDocument document=new HtmlDocument();
|
|
document.Load(memoryStream);
|
|
HtmlNode htmlNodeHeadline = document.DocumentNode.SelectSingleNode("//*[@class=\"article__headline\"]");
|
|
HtmlNode htmlNodeTimestamp = document.DocumentNode.SelectSingleNode("//*[@class=\"article__timestamp\"]");
|
|
if(null==htmlNodeHeadline || null==htmlNodeTimestamp)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
List<String> sections=Sections.GetSections(htmlNodeHeadline.InnerHtml);
|
|
if(null == sections || 0==sections.Count)continue;
|
|
String strHeadline = Utility.RemoveHtml(sections[sections.Count-1]);
|
|
String strTimestamp = htmlNodeTimestamp.InnerText;
|
|
|
|
if(String.IsNullOrEmpty(strHeadline) || String.IsNullOrEmpty(strTimestamp))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Headline headline= new Headline();
|
|
headline.Symbol=symbol;
|
|
headline.Date=FeedParser.ParseValueDateTimeMonthFormatTZ(strTimestamp);
|
|
headline.Entry=strHeadline.Trim();
|
|
headline.Source="MarketWatch";
|
|
headline.Modified=modified;
|
|
headline.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;
|
|
|
|
if(Utility.IsEpoch(headline.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Invalid date for symbol {0}",symbol));
|
|
continue;
|
|
}
|
|
headlines.Add(headline);
|
|
}
|
|
headlines=new Headlines(headlines.Distinct().ToList());
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
//************************************************************************ H E A D L I N E S - N A S D A Q ******************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static Headlines GetCompanyHeadlinesNASDAQ(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
Headlines headlines=new Headlines();
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
symbol=symbol.ToUpper();
|
|
sb.Append("https://api.nasdaq.com/api/news/topic/articlebysymbol?q=").Append(symbol).Append("|etf&offset=0&limit=8&fallback=false");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
if(null==items || items.Count<7)return null;
|
|
int index=0;
|
|
for(;index<items.Count;index++)
|
|
{
|
|
if((items[index])[0].Equals("title"))break;
|
|
}
|
|
for(;index<items.Count;index+=7)
|
|
{
|
|
if(!(items[index])[0].Equals("title"))continue;
|
|
if(!(items[index+3])[0].Equals("ago"))continue;
|
|
DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
|
|
if(Utility.IsEpoch(referenceDate))continue;
|
|
Headline headline=new Headline();
|
|
headline.Source="NASDAQ";
|
|
headline.Symbol=symbol;
|
|
headline.Date=referenceDate;
|
|
headline.Entry=Uri.UnescapeDataString((items[index])[1]);
|
|
headline.Entry=Utility.RemoveHtml(headline.Entry);
|
|
headlines.Add(headline);
|
|
}
|
|
// headlines=new Headlines(headlines.GroupBy(x=>x.Entry).Select(y=>y.First()).ToList());
|
|
return headlines;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T I S T I C S ************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
|
|
public static PriceIndices GetConsumerPriceIndices()
|
|
{
|
|
String indexCode=null;
|
|
String indexName=null;
|
|
String indexSource="BLS";
|
|
HttpNetResponse httpNetResponse=null;
|
|
PriceIndices priceIndices=null;
|
|
MemoryStream memoryStream=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.bls.gov/regions/mid-atlantic/data/consumerpriceindexhistorical_us_table.htm");
|
|
String strRequest=sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,300000,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"ro3fxusu_cms\"]");
|
|
HtmlNodeCollection rowData=table[0].SelectNodes(".//tr");
|
|
|
|
HtmlNodeCollection titles=table[0].SelectNodes("//*[@class=\"tableTitle\"]");
|
|
if(titles.Count<1)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate title."));
|
|
return null;
|
|
}
|
|
String titleInfo=titles[0].InnerText;
|
|
indexName=Utility.BetweenString(titleInfo,null,"(").Trim();
|
|
indexCode=Utility.BetweenString(titleInfo,"(",")").Trim();
|
|
if(null==indexName || null==indexCode)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate indexName/indexCode."));
|
|
return null;
|
|
}
|
|
if(rowData.Count<3)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, insufficient row data."));
|
|
return null;
|
|
}
|
|
HtmlNode htmlNode=rowData[2].SelectNodes(".//th").FirstOrDefault();
|
|
if(null==htmlNode)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate proper header."));
|
|
return null;
|
|
}
|
|
if(!"Consumer Price Index".Equals(htmlNode.InnerText))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate consumer price index."));
|
|
return null;
|
|
}
|
|
|
|
priceIndices=new PriceIndices();
|
|
for(int index=3;index<rowData.Count;index++)
|
|
{
|
|
HtmlNodeCollection headerElements=rowData[index].SelectNodes(".//th");
|
|
if(null==headerElements || 0==headerElements.Count)break;
|
|
HtmlNode headerNode=headerElements.FirstOrDefault();
|
|
if(null==headerNode ||"".Equals(headerNode.InnerText))break;
|
|
int year=int.Parse(headerNode.InnerText);
|
|
HtmlNodeCollection monthsDataCollection=rowData[index].SelectNodes(".//td");
|
|
for(int monthIndex=0;monthIndex<monthsDataCollection.Count;monthIndex++)
|
|
{
|
|
String strValue=monthsDataCollection[monthIndex].InnerText;
|
|
if(null==strValue || "".Equals(strValue))break;
|
|
double value=FeedParser.ParseValue(strValue);
|
|
if(double.IsNaN(value))break;
|
|
PriceIndex priceIndex=new PriceIndex();
|
|
priceIndex.AsOf=new DateTime(year,monthIndex+1,1);
|
|
priceIndex.Code=indexCode;
|
|
priceIndex.Name=indexName;
|
|
priceIndex.Value=value;
|
|
priceIndex.Source=indexSource;
|
|
priceIndices.Add(priceIndex);
|
|
}
|
|
}
|
|
return priceIndices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)
|
|
{
|
|
httpNetResponse.Dispose();
|
|
}
|
|
if(null!=memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
memoryStream=null;
|
|
}
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
// ************************************ A N A L Y S T P R I C E T A R G E T - Y A H O O F I N A N C E / M A R K E T B E A T (B A C K U P )******************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static AnalystPriceTarget GetAnalystPriceTarget(String symbol)
|
|
{
|
|
//AnalystPriceTarget analystPriceTarget = GetAnalystPriceTargetYahoo(symbol);
|
|
AnalystPriceTarget analystPriceTarget = GetAnalystPriceTargetMarketBeat(symbol);
|
|
return analystPriceTarget;
|
|
}
|
|
public static AnalystPriceTarget GetAnalystPriceTargetMarketBeat(String symbol)
|
|
{
|
|
AnalystPriceTarget analystPriceTarget = null;
|
|
String nyse="NYSE";
|
|
String nasdaq="NASDAQ";
|
|
|
|
try
|
|
{
|
|
HtmlNodeCollection table = GetMarketBeatAnalystPriceTargetTable(symbol, nasdaq);
|
|
if(null==table)table = GetMarketBeatAnalystPriceTargetTable(symbol, nyse);
|
|
if(null==table)return null;
|
|
HtmlNodeCollection headerColumns=table[0].SelectNodes(".//th");
|
|
HtmlNodeCollection dataColumns=table[0].SelectNodes(".//td");
|
|
analystPriceTarget=new AnalystPriceTarget();
|
|
analystPriceTarget.Symbol=symbol;
|
|
analystPriceTarget.Date=DateTime.Now.Date;
|
|
for(int index=0;index<headerColumns.Count;index++)
|
|
{
|
|
String forecastType=headerColumns[index].InnerText;
|
|
String forecastData=dataColumns[index].InnerText;
|
|
|
|
if(forecastType.Equals("High Forecast"))
|
|
{
|
|
analystPriceTarget.HighTargetPrice=FeedParser.ParseValue(forecastData);
|
|
}
|
|
else if(forecastType.Equals("Average Forecast"))
|
|
{
|
|
analystPriceTarget.MeanTargetPrice=FeedParser.ParseValue(forecastData);
|
|
analystPriceTarget.MedianTargetPrice=analystPriceTarget.MeanTargetPrice;
|
|
}
|
|
else if(forecastType.Equals("Low Forecast"))
|
|
{
|
|
analystPriceTarget.LowTargetPrice=FeedParser.ParseValue(forecastData);
|
|
}
|
|
}
|
|
if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.LowTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
|
|
return analystPriceTarget;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
}
|
|
private static HtmlNodeCollection GetMarketBeatAnalystPriceTargetTable(String symbol, String exchange)
|
|
{
|
|
MemoryStream memoryStream=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target");
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetMarketBeat");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetMarketBeat: {0}", strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
memoryStream = null;
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"w-100 my-2\"]");
|
|
if(null==table || 1!=table.Count)
|
|
{
|
|
return null;
|
|
}
|
|
return table;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
memoryStream=null;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
//private static AnalystPriceTarget GetAnalystPriceTargetYahoo(String symbol)
|
|
//{
|
|
// AnalystPriceTarget analystPriceTarget = null;
|
|
// HttpNetResponse httpNetResponse=null;
|
|
// try
|
|
// {
|
|
// StringBuilder sb = new StringBuilder();
|
|
// String strRequest;
|
|
// symbol = symbol.ToUpper();
|
|
// sb.Append("http://finance.yahoo.com/q/ao?s=").Append(symbol).Append("+Analyst+Opinion");
|
|
// WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetYahoo");
|
|
// strRequest=sb.ToString();
|
|
// MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetYahoo: {0}", strRequest));
|
|
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
// if(!httpNetResponse.Success)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
// return null;
|
|
// }
|
|
// analystPriceTarget=new AnalystPriceTarget();
|
|
// analystPriceTarget.Symbol=symbol;
|
|
// analystPriceTarget.Date=DateTime.Now.Date;
|
|
// String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */");
|
|
// analystPriceTarget.HighTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetHighPrice\":{\"raw\":",','));
|
|
// analystPriceTarget.MeanTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMeanPrice\":{\"raw\":",','));
|
|
// analystPriceTarget.LowTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetLowPrice\":{\"raw\":",','));
|
|
// analystPriceTarget.MedianTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMedianPrice\":{\"raw\":",','));
|
|
// if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
|
|
// double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
|
|
// double.IsNaN(analystPriceTarget.LowTargetPrice)&&
|
|
// double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
|
|
// return analystPriceTarget;
|
|
// }
|
|
// catch (Exception exception)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
// return null;
|
|
// }
|
|
// finally
|
|
// {
|
|
// if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
// }
|
|
//}
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
|
|
{
|
|
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnyse";
|
|
String nys="xnys";
|
|
String salVersion="3.79.0";
|
|
String exchange=nasdaq;
|
|
// String url="https://www.morningstar.com/stocks/";
|
|
String url="https://www.morningstar.com/api/v2/stocks/";
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
StringBuilder strReferer = new StringBuilder();
|
|
String strRequest;
|
|
sb=new StringBuilder();
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetHistoricalValues");
|
|
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
exchange=nys;
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
exchange=nyse;
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
|
|
if(null==securityId)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
|
|
return null;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/operatingPerformance/v2/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-oper-perf&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsOperatingPerformance=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
Dictionary<String,MStarDataSet> dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(salVersion);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// check result sets
|
|
String strDefinition="Return on Assets %";
|
|
String strColumnDefinition="columnDefs";
|
|
if(dataSetsOperatingPerformance.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Return on Invested Capital %";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsOperatingPerformance.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsOperatingPerformance[strColumnDefinition],dataSetsOperatingPerformance[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="bookValuePerShare";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsFinancialHealth.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.BVPS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsFinancialHealth[strColumnDefinition],dataSetsFinancialHealth[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Inventories";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Inventory;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Trade/Accounts Receivable, Current";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.AccountsReceivable;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Cost of Goods and Services";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.COGS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Total Operating Profit/Loss";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingIncome;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Interest Expense Net of Capitalized Interest";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.InterestExpense;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Reported Effective Tax Rate";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=100.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.TaxRate;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Total Revenue";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Revenue;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Net Income Available to Common Stockholders";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
multiplier/=1000000;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.NetIncomeAvailableToCommonShareholders;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Diluted EPS";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.EPS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Working Capital";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.WorkingCapital;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString());
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Free Cash Flow";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.FreeCashflow;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
strDefinition="Cash Generated from Operating Activities";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsCashflowStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsCashflowStatement["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingCashflow;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsCashflowStatement[strColumnDefinition],dataSetsCashflowStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
return values;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
internal class MStarDataSet
|
|
{
|
|
private String name;
|
|
private List<String> values;
|
|
public MStarDataSet()
|
|
{
|
|
values=new List<String>();
|
|
}
|
|
public MStarDataSet(String name,List<String> values)
|
|
{
|
|
this.name=name;
|
|
this.values=values;
|
|
}
|
|
public MStarDataSet(String name,String value)
|
|
{
|
|
values=new List<String>();
|
|
this.name=name;
|
|
this.values.Add(value);
|
|
}
|
|
public String Name
|
|
{
|
|
get{return name;}
|
|
set{name=value;}
|
|
}
|
|
public List<String> Values
|
|
{
|
|
get{return values;}
|
|
private set{;}
|
|
}
|
|
public T GetItem<T>(int index=0)
|
|
{
|
|
T result=default(T);
|
|
try {result = (T)Convert.ChangeType(values[index], typeof(T));}
|
|
catch {result = default(T);}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
|
|
//\"orderOfMagnitude\":\"Million\",
|
|
private static double GetMultiplierForDataSet(String responseString)
|
|
{
|
|
double multiplier=1.00;
|
|
try
|
|
{
|
|
string token="\"indexName\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)
|
|
{
|
|
token="\"orderOfMagnitude\"";
|
|
index=responseString.IndexOf(token);
|
|
if(-1==index)return multiplier;
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(1!=parts.Length)return multiplier;
|
|
if(parts[0].Equals("Million"))multiplier=1000000;
|
|
else if(parts[0].Equals("Billion"))multiplier=1000000000;
|
|
else if(parts[0].Equals("Trillion"))multiplier=1000000000000;
|
|
else multiplier=1.00;
|
|
return multiplier;
|
|
}
|
|
else
|
|
{
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(2>parts.Length)return multiplier;
|
|
if(parts[parts.Length-2].Equals("TR"))multiplier=1000000000;
|
|
else
|
|
{
|
|
multiplier=1;
|
|
}
|
|
return multiplier;
|
|
}
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMultiplierForDataSet encountered an exception:{0}",exception.ToString()));
|
|
return multiplier;
|
|
}
|
|
}
|
|
|
|
// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
|
|
//\"orderOfMagnitude\":\"Million\",
|
|
// {"currency":"USD","currencySymbol":"$","orderOfMagnitude":"Billion","fiscalYearEndDate":"12-31"},"userType":"Free"}
|
|
private static String GetCurrencyForDataSet(String responseString)
|
|
{
|
|
String currency=null;
|
|
try
|
|
{
|
|
string token="\"indexName\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)
|
|
{
|
|
token="\"currency\"";
|
|
index=responseString.IndexOf(token);
|
|
if(-1==index)return currency;
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(1!=parts.Length)return currency;
|
|
currency=parts[0];
|
|
return currency;
|
|
}
|
|
else
|
|
{
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(2>parts.Length)return currency;
|
|
currency=parts[parts.Length-1];
|
|
return currency;
|
|
}
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCurrencyForDataSet encountered an exception:{0}",exception.ToString()));
|
|
return currency;
|
|
}
|
|
}
|
|
//public static String GetMStarSecurityId(String symbol,String responseString)
|
|
//{
|
|
// String securityIdentifier=GetMStarSecurityId_1(responseString);
|
|
// if(null!=securityIdentifier)return securityIdentifier;
|
|
// securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
|
|
// if(null!=securityIdentifier)return securityIdentifier;
|
|
// return null;
|
|
//}
|
|
|
|
|
|
public static String GetMStarSecurityId(String symbol,String responseString)
|
|
{
|
|
String securityIdentifier=null;
|
|
if(MStarSecurityDA.HasSecurityId(symbol))
|
|
{
|
|
securityIdentifier=MStarSecurityDA.GetSecurityId(symbol);
|
|
return securityIdentifier;
|
|
}
|
|
securityIdentifier=GetMStarSecurityId_1(responseString);
|
|
if(null==securityIdentifier)
|
|
{
|
|
securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
|
|
if(null==securityIdentifier)
|
|
{
|
|
securityIdentifier=GetMStarSecurityId_3(responseString);
|
|
}
|
|
}
|
|
if(null==securityIdentifier)return null;
|
|
MStarSecurityDA.PutSecurityId(symbol,securityIdentifier);
|
|
return securityIdentifier;
|
|
}
|
|
|
|
//byId:{\"0P000003MU\":$}
|
|
// AAPL:0P000000GY
|
|
//private static String GetMStarSecurityId_2(String responseString)
|
|
//{
|
|
// try
|
|
// {
|
|
// string token="byId";
|
|
// int index=responseString.IndexOf(token);
|
|
// if(-1==index)return null;
|
|
// string securityIdentifier=responseString.Substring(index+token.Length);
|
|
// securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
|
|
// return securityIdentifier;
|
|
// }
|
|
// catch(Exception exception)
|
|
// {
|
|
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId encountered an exception:{0}",exception.ToString()));
|
|
// return null;
|
|
// }
|
|
//}
|
|
|
|
// 5","0P000003MU","MIDD",60
|
|
private static String GetMStarSecurityId_2(String symbol,String responseString)
|
|
{
|
|
try
|
|
{
|
|
String searchString="\""+symbol+"\"";
|
|
int index=responseString.LastIndexOf(searchString);
|
|
if(-1==index)return null;
|
|
int commaCount=0;
|
|
while(index>=0 && commaCount!=2)
|
|
{
|
|
char ch=responseString[index--];
|
|
if(ch.Equals(','))commaCount++;
|
|
}
|
|
if(index<0)return null;
|
|
responseString=responseString.Substring(index+1);
|
|
String securityIdentifier=Utility.BetweenString(responseString,"\"","\"");
|
|
if(securityIdentifier.Equals("en-us")||securityIdentifier.StartsWith("blt"))return null;
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_2 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// us","AAON",401,"subscription","blt8b91befd99376df6","bltc8ff21a187c6e079",2,"image\u002Fsvg+xml","blt187f65f0e94c77b5","0P00000012",60,80,"AAON In
|
|
private static String GetMStarSecurityId_3(String responseString)
|
|
{
|
|
try
|
|
{
|
|
string token="0P";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)return null;
|
|
string securityIdentifier=responseString.Substring(index-token.Length);
|
|
securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_3 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
//","performanceID":"0P000000GY","securityID":"0P000000GY","ticker":"AAPL","exc
|
|
private static String GetMStarSecurityId_1(String responseString)
|
|
{
|
|
try
|
|
{
|
|
string token="\"securityID\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)return null;
|
|
string securityIdentifier=responseString.Substring(index+token.Length);
|
|
securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_1 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static Dictionary<String,MStarDataSet> GetData(String responseString)
|
|
{
|
|
try
|
|
{
|
|
Dictionary<String,String> uniqueDataLabels=new Dictionary<String,String>();
|
|
String label="\"label\"";
|
|
String datum="\"datum\"";
|
|
String columnDefs="\"columnDefs\"";
|
|
String strData=null;
|
|
Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
|
|
if(null==responseString)return data;
|
|
int index=responseString.IndexOf(columnDefs);
|
|
if(-1==index)return null;
|
|
strData=responseString.Substring(index+columnDefs.Length);
|
|
strData=Utility.BetweenString(strData,"[","]");
|
|
List<String> items = Utility.ToList(strData,',');
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
String strItem=items[itemIndex];
|
|
strItem=Utility.RemoveQuotes(strItem);
|
|
items[itemIndex]=strItem;
|
|
}
|
|
String labelName=Utility.RemoveQuotes(columnDefs);
|
|
data.Add(labelName,new MStarDataSet(labelName,items));
|
|
|
|
int dataIndex=0;
|
|
for(index=responseString.IndexOf(label);index!=-1;index=responseString.IndexOf(label,dataIndex))
|
|
{
|
|
|
|
dataIndex=responseString.IndexOf(datum,index);
|
|
String strLabel=responseString.Substring(index+label.Length);
|
|
strLabel=Utility.BetweenString(strLabel,"\"","\"");
|
|
if(uniqueDataLabels.ContainsKey(strLabel))break;
|
|
uniqueDataLabels.Add(strLabel,strLabel);
|
|
strData=responseString.Substring(dataIndex+datum.Length);
|
|
strData=Utility.BetweenString(strData,"[","]");
|
|
items=Utility.ToList(strData,',');
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
String strItem=items[itemIndex];
|
|
strItem=Utility.RemoveQuotes(strItem);
|
|
items[itemIndex]=strItem;
|
|
}
|
|
data.Add(strLabel,new MStarDataSet(strLabel,items));
|
|
}
|
|
double multiplier=GetMultiplierForDataSet(responseString);
|
|
String currency=GetCurrencyForDataSet(responseString);
|
|
data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString()));
|
|
data.Add("currency",new MStarDataSet("currency",currency));
|
|
return data;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String responseString) encountered an exception:{0}",exception.ToString()));
|
|
return new Dictionary<String,MStarDataSet>();
|
|
}
|
|
}
|
|
private static Dictionary<String,MStarDataSet> GetData(String mainHeading, List<List<String>> items,String responseString)
|
|
{
|
|
try
|
|
{
|
|
Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
|
|
Dictionary<String,Dictionary<String,String>> dataDictionary=new Dictionary<String,Dictionary<String,String>>();
|
|
Dictionary<String,String> uniqueElementNamesDictionary=new Dictionary<String,String>();
|
|
|
|
if(null==mainHeading || null==items || null==responseString)
|
|
{
|
|
return data;
|
|
}
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
List<String> subItems=items[itemIndex];
|
|
if(subItems[0].Equals(mainHeading))
|
|
{
|
|
String strDate=subItems[1];
|
|
dataDictionary.Add(strDate,new Dictionary<String,String>());
|
|
for(++itemIndex;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
List<String> subsubItems=items[itemIndex];
|
|
if(subsubItems[0].Equals(mainHeading))
|
|
{
|
|
itemIndex--;
|
|
break;
|
|
}
|
|
Dictionary<String,String> datedItems=dataDictionary[strDate];
|
|
String subItemElementName=subsubItems[0];
|
|
if(!datedItems.ContainsKey(subItemElementName))
|
|
{
|
|
if(!uniqueElementNamesDictionary.ContainsKey(subItemElementName))uniqueElementNamesDictionary.Add(subItemElementName,subItemElementName);
|
|
datedItems.Add(subItemElementName,subsubItems[1]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
List<String> strDates=new List<String>(dataDictionary.Keys);
|
|
List<String> uniqueElementNames=new List<String>(uniqueElementNamesDictionary.Keys);
|
|
data.Add("columnDefs",new MStarDataSet("columnDefs",strDates));
|
|
|
|
foreach(String elementName in uniqueElementNames)
|
|
{
|
|
List<String> elementValues=new List<String>();
|
|
foreach(String strDate in strDates)
|
|
{
|
|
Dictionary<String,String> inner=dataDictionary[strDate];
|
|
elementValues.Add(inner[elementName]);
|
|
}
|
|
data.Add(elementName,new MStarDataSet(elementName,elementValues));
|
|
}
|
|
double multiplier=GetMultiplierForDataSet(responseString);
|
|
if(double.IsNaN(multiplier))multiplier=1.00; // This GetData method is only used to retrieve ratios so far so the data sets do not have a multiplier section.
|
|
data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString())); // If this method is expanded to retrieve data points other than non-ratios then you will have to come up with method for retrieving the multiplier.
|
|
return data; // so that we know if the data is in trillions, billions, millions and what not.
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String mainHeading, List<List<String>> items) encountered an exception {0}",exception.ToString()));
|
|
return new Dictionary<String,MStarDataSet>();
|
|
}
|
|
}
|
|
|
|
private static TimeSeriesCollection CreateTimeSeriesCollection(String symbol, MStarDataSet colDefData, MStarDataSet dataElements, TimeSeriesElement.ElementType elementType,double multiplier=1.00)
|
|
{
|
|
TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection();
|
|
Dictionary<DateTime,DateTime> dates = new Dictionary<DateTime,DateTime>();
|
|
DateTime today=DateTime.Now;
|
|
try
|
|
{
|
|
if(null==colDefData || null==dataElements || null==colDefData.Values || null==dataElements.Values || dataElements.Values.Count()!=colDefData.Values.Count())
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot create time series for symbol {0} for element type {1}",symbol,TimeSeriesElement.StringForType(elementType)));
|
|
return timeSeriesCollection;
|
|
}
|
|
for (int index = 0; index < colDefData.Values.Count;index++)
|
|
{
|
|
DateTime reportDate=DateTime.Now;
|
|
String strDateElement=colDefData.Values[index];
|
|
String strDataElement=dataElements.Values[index];
|
|
|
|
if("_PO_".Equals(strDataElement))
|
|
{
|
|
continue;
|
|
}
|
|
if("null".Equals(strDataElement))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if(strDateElement.Length.Equals(4))
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
if(reportDate.Year.Equals(today.Year))
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,today.Month,1);
|
|
}
|
|
else
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,12,31);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy-MM"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
}
|
|
TimeSeriesElement timeSeriesElement = new TimeSeriesElement();
|
|
timeSeriesElement.Type = elementType;
|
|
timeSeriesElement.Symbol = symbol;
|
|
timeSeriesElement.AsOf =reportDate;
|
|
timeSeriesElement.Value = FeedParser.ParseValue(dataElements.Values[index])*multiplier;
|
|
if (double.IsNaN(timeSeriesElement.Value)) continue;
|
|
if (!dates.ContainsKey(timeSeriesElement.AsOf))
|
|
{
|
|
dates.Add(timeSeriesElement.AsOf, timeSeriesElement.AsOf);
|
|
timeSeriesCollection.Add(timeSeriesElement);
|
|
}
|
|
}
|
|
return timeSeriesCollection;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateTimeSeriesCollection encountered an exception processing symbol '{0}' {1}",symbol,exception.ToString()));
|
|
return new TimeSeriesCollection();
|
|
}
|
|
}
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 E N D ***********************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 **********************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
public static List<CashflowStatement> GetCashflowStatement(String symbol,CashflowStatement.PeriodType periodType)
|
|
{
|
|
List<CashflowStatement> cashflowStatements=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnyse";
|
|
String nys="xnys";
|
|
|
|
try
|
|
{
|
|
if(!CashflowStatement.PeriodType.Annual.Equals(periodType))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unsupported CashflowStatement.PeriodType Quarterly. Only Annual supported from M*"));
|
|
return null;
|
|
}
|
|
StringBuilder sb = new StringBuilder();
|
|
StringBuilder strReferer = new StringBuilder();
|
|
String strRequest;
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nasdaq).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement");
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nys).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nyse).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
|
|
if(null==securityId)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
|
|
return null;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));
|
|
try{Thread.Sleep(250);}finally{;}
|
|
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=3.74.0");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} cashflow statement items for {1}",dataSetsCashflowStatement.Count,symbol));
|
|
|
|
cashflowStatements=CreateCashflowStatements(symbol, dataSetsCashflowStatement);
|
|
return cashflowStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCashflowStatement encountered an exception {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static List<CashflowStatement> CreateCashflowStatements(String symbol,Dictionary<String,MStarDataSet> cashflowDataElements)
|
|
{
|
|
DateTime today=DateTime.Now;
|
|
List<CashflowStatement> cashflowStatements=new List<CashflowStatement>();
|
|
String currencyCashflow=cashflowDataElements["currency"].Values[0];
|
|
double cashflowStatementMultiplier=double.Parse(cashflowDataElements["multiplier"].Values[0]);
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
String baseCurrency="USD";
|
|
|
|
try
|
|
{
|
|
Dictionary<DateTime,Dictionary<String,String>> cashflowDictionaryByDate=CreateDictionaryByDate(cashflowDataElements);
|
|
List<DateTime> cashflowDateKeys=new List<DateTime>(cashflowDictionaryByDate.Keys);
|
|
if(0==cashflowDateKeys.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: No dates in cashflowDataElement."));
|
|
return null;
|
|
}
|
|
|
|
foreach(DateTime cashflowDate in cashflowDateKeys)
|
|
{
|
|
Dictionary<String,String> cashflowDatedElements=cashflowDictionaryByDate[cashflowDate];
|
|
|
|
CashflowStatement cashflowStatement=new CashflowStatement();
|
|
cashflowStatement.Symbol=symbol;
|
|
cashflowStatement.AsOf=cashflowDate;
|
|
cashflowStatement.Modified=DateTime.Now;
|
|
cashflowStatement.Period=CashflowStatement.PeriodType.Annual;
|
|
|
|
if(!baseCurrency.Equals(currencyCashflow))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currencyCashflow,baseCurrency,cashflowStatement.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currencyCashflow,cashflowStatement.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currencyCashflow,cashflowStatement.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString()));
|
|
return null;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currencyCashflow,baseCurrency,cashflowStatement.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement);
|
|
}
|
|
|
|
double currencyMultiplier=1.00;
|
|
currencyMultiplier=conversionRates.ContainsKey(cashflowStatement.AsOf)?conversionRates[cashflowStatement.AsOf].UnitsPerSource:currencyMultiplier;
|
|
|
|
if(cashflowDatedElements.ContainsKey("Depreciation and Amortization, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Depreciation and Amortization, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.DepreciationAndAmortization=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.DepreciationAndAmortization=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Deferred Taxes, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Deferred Taxes, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.DeferredIncomeTaxes=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.DeferredIncomeTaxes=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Stock-Based Compensation, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Stock-Based Compensation, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.StockBasedCompensation=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.StockBasedCompensation=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Change in Trade/Accounts Receivable"))
|
|
{
|
|
String value = cashflowDatedElements["Change in Trade/Accounts Receivable"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.AccountsReceivable=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.AccountsReceivable=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Change in Accrued Expenses"))
|
|
{
|
|
String value = cashflowDatedElements["Change in Accrued Expenses"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.AccruedLiabilities=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.AccruedLiabilities=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Cash Generated from Operating Activities"))
|
|
{
|
|
String value = cashflowDatedElements["Cash Generated from Operating Activities"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.OperatingCashflow=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.OperatingCashflow=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Purchase/Sale and Disposal of Property, Plant and Equipment, Net"))
|
|
{
|
|
String value = cashflowDatedElements["Purchase/Sale and Disposal of Property, Plant and Equipment, Net"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value)&& !double.IsNaN(cashflowStatement.OperatingCashflow))
|
|
{
|
|
cashflowStatement.FreeCashflow=(cashflowStatement.OperatingCashflow+(double.Parse(value)*cashflowStatementMultiplier))*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.FreeCashflow=double.NaN;
|
|
}
|
|
|
|
if(cashflowStatement.IsValid())
|
|
{
|
|
cashflowStatements.Add(cashflowStatement);
|
|
}
|
|
}
|
|
|
|
return cashflowStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: Exception encountered {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static Dictionary<DateTime,Dictionary<String,String>> CreateDictionaryByDate(Dictionary<String,MStarDataSet> dataSets)
|
|
{
|
|
try
|
|
{
|
|
DateTime today=DateTime.Now;
|
|
Dictionary<DateTime,Dictionary<String,String>> dictionaryByDate=new Dictionary<DateTime,Dictionary<String,String>>();
|
|
MStarDataSet columnDefs=dataSets["columnDefs"];
|
|
List<String> itemKeys=new List<String>(dataSets.Keys);
|
|
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("columnDefs"));
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("multiplier"));
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("currency"));
|
|
for(int dateIndex=0;dateIndex<columnDefs.Values.Count;dateIndex++)
|
|
{
|
|
String strDateElement=columnDefs.Values[dateIndex];
|
|
DateTime reportDate=Utility.Epoch;
|
|
if(strDateElement.Length.Equals(4))
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
if(reportDate.Year.Equals(today.Year))
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,today.Month,1);
|
|
}
|
|
else
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,12,31);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, new string[]{"yyyy-MM"}, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
}
|
|
|
|
foreach(String itemKey in itemKeys)
|
|
{
|
|
MStarDataSet dataSet=dataSets[itemKey];
|
|
String value=null;
|
|
value=dataSet.Values[dateIndex];
|
|
if(!dictionaryByDate.ContainsKey(reportDate))
|
|
{
|
|
dictionaryByDate.Add(reportDate,new Dictionary<String,String>());
|
|
}
|
|
Dictionary<String,String> datedCollection=dictionaryByDate[reportDate];
|
|
|
|
if(!datedCollection.ContainsKey(itemKey))
|
|
{
|
|
datedCollection.Add(itemKey,value);
|
|
}
|
|
else datedCollection[itemKey]=value;
|
|
}
|
|
}
|
|
return dictionaryByDate;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateDictionaryByDate: Exception encountered {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 E N D ************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ******************************************************************* C A S H F L O W S T A T E M E N T - M O R N I N G S T A R ********************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
public static List<CashflowStatement> GetCashflowStatement_Obsolete(String symbol,CashflowStatement.PeriodType periodType)
|
|
{
|
|
List<CashflowStatement> cashflowStatements=new List<CashflowStatement>();
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
DateTime currentDate=DateTime.Now;
|
|
String baseCurrency="USD";
|
|
String nasdaq="XNAS";
|
|
String nyse="XNYS";
|
|
|
|
try
|
|
{
|
|
CSVFileExt csvFile=null;
|
|
csvFile=GetCashflowStatementCSV(symbol,periodType,nasdaq);
|
|
if(null==csvFile)csvFile=GetCashflowStatementCSV(symbol,periodType,nyse);
|
|
if(null==csvFile)return null;
|
|
String dateMarker=csvFile.GetRowData(0,0);
|
|
if(null==dateMarker||!dateMarker.StartsWith("Fiscal"))return null;
|
|
String currency=Utility.BetweenString(dateMarker,".",".");
|
|
if(null!=currency)currency=currency.Trim().ToUpper();
|
|
else currency=baseCurrency; // if we can't find the currency in the header then assume it's USD
|
|
|
|
// build the list of asof dates for the cashflow records
|
|
List<String> rowData=csvFile.GetRow(0);
|
|
for(int index=0;index<rowData.Count-2;index++)
|
|
{
|
|
CashflowStatement cashflowStatement=new CashflowStatement();
|
|
cashflowStatement.Symbol=symbol;
|
|
cashflowStatement.Period=periodType;
|
|
cashflowStatement.AsOf=dateGenerator.GetNextBusinessDay(Utility.ParseDate(rowData[index+1])); // Utility.ParseDate(rowData[index+1]); // dateGenerator.GetNextBusinessDay(Utility.ParseDate(rowData[index+1]));
|
|
cashflowStatement.Modified=currentDate;
|
|
cashflowStatements.Add(cashflowStatement);
|
|
if(!baseCurrency.Equals(currency))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,cashflowStatement.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,cashflowStatement.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,cashflowStatement.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,cashflowStatement.AsOf.ToShortDateString()));
|
|
return null;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,cashflowStatement.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement);
|
|
}
|
|
}
|
|
for(int rowIndex=1;rowIndex<csvFile.RowCount;rowIndex++)
|
|
{
|
|
String itemHeading=csvFile.GetRowData(rowIndex,0);
|
|
if(null==itemHeading)continue;
|
|
itemHeading=itemHeading.ToLower();
|
|
if(itemHeading.Equals("Depreciation & amortization".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].DepreciationAndAmortization=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Deferred income taxes".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].DeferredIncomeTaxes=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Stock based compensation".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].StockBasedCompensation=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Accounts receivable".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].AccountsReceivable=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Accounts payable".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].AccountsPayable=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Inventory".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].Inventory=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Accrued liabilities".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].AccruedLiabilities=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Operating cash flow".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].OperatingCashflow=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
else if(itemHeading.Equals("Free Cash Flow".ToLower()))
|
|
{
|
|
for(int colIndex=1;colIndex<rowData.Count-1;colIndex++)
|
|
{
|
|
double multiplier=1.00;
|
|
multiplier=conversionRates.ContainsKey(cashflowStatements[colIndex-1].AsOf)?conversionRates[cashflowStatements[colIndex-1].AsOf].UnitsPerSource:multiplier;
|
|
cashflowStatements[colIndex-1].FreeCashflow=FeedParser.ParseValue(csvFile.GetRowData(rowIndex,colIndex))*multiplier;
|
|
}
|
|
}
|
|
}
|
|
return cashflowStatements;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
}
|
|
}
|
|
private static CSVFileExt GetCashflowStatementCSV(String symbol,CashflowStatement.PeriodType periodType,String exchange)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
String strReferer;
|
|
sb.Append("http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=").Append(exchange).Append(":");
|
|
sb.Append(symbol).Append("®ion=usa&culture=en-US&version=SAL&cur=&reportType=cf&period=");
|
|
sb.Append(periodType.Equals(CashflowStatement.PeriodType.Quarterly) ? 3 : 12).Append("&dataType=A&order=asc&columnYear=5&curYearPart=1st5year&rounding=1&view=raw&r=310540&denominatorView=raw&number=1");
|
|
strRequest = sb.ToString();
|
|
sb=new StringBuilder();
|
|
sb.Append("http://financials.morningstar.com/cash-flow/cf.html?t=").Append(symbol).Append("®ion=usa&culture=en-US");
|
|
strReferer=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestCSV(strRequest,strReferer);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
memoryStream=(MemoryStream)httpNetResponse.ResponseStringToStream();
|
|
CSVFileExt csvFile=new CSVFileExt();
|
|
csvFile.LoadCSVFileNoHeader(new StreamReader(memoryStream),1);
|
|
if(0==csvFile.RowCount)return null;
|
|
return csvFile;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ******************************************************************* C A S H F L O W S T A T E M E N T - M O R N I N G S T A R E N D ************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ I N C O M E S T A T E M E N T - N A S D A Q / F I N V I Z **************************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
public static List<IncomeStatement> GetIncomeStatement(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
List<IncomeStatement> incomeStatements=GetIncomeStatementNASDAQ(symbol,periodType);
|
|
if(null!=incomeStatements && incomeStatements.Count>0)return incomeStatements;
|
|
incomeStatements=GetIncomeStatementFinViz(symbol,periodType);
|
|
return incomeStatements;
|
|
}
|
|
|
|
|
|
//{"currency":"USD","data":{
|
|
//"Period End Date":["TTM","1/2/2021","12/28/2019","12/29/2018","12/30/2017","12/31/2016","1/2/2016","1/3/2015"],
|
|
//"Total Revenue":["2,930.65","2,513.26","2,959.45","2,722.93","2,335.54","2,267.85","1,826.60","1,636.54"],
|
|
//"Cost of Revenue":["1,872.32","1,631.21","1,855.95","1,718.79","1,422.80","1,366.67","1,120.09","995.95"],
|
|
//"Selling, General and Administrative":["596.80","531.90","583.81","538.84","468.22","471.64","344.55","320.37"],
|
|
//"Gross Profit":["1,058.33","882.05","1,103.50","1,004.14","912.74","901.18","706.50","640.59"],
|
|
//"Total Operating Expense":["2,492.86","2,188.83","2,445.40","2,276.97","1,956.93","1,848.83","1,523.99","1,336.11"],
|
|
|
|
//"Research and Development":["","35.30","41.20","35.30","29.10","26.30","22.40","22.60"],
|
|
//"Net Income":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
//"Income Avail. to Common Incl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
|
|
//"Interest Income Net":["-71.44","-81.52","-83.51","-56.21","-23.64","-21.98","-10.17","-11.99"],
|
|
//"Period Length":["12 Months","53 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","53 Weeks"],
|
|
//"Unusual Expense/Income":["23.74","25.72","5.64","19.33","65.91","10.52","59.35","19.78"],
|
|
//"Operating Income":["437.79","324.43","514.04","445.97","378.61","419.02","302.60","300.43"],
|
|
//"Other Income Net":["31.16","25.14","32.09","33.76","28.55","24.27","-11.27","-7.65"],
|
|
//"Net Income Before Taxes":["397.51","268.06","462.62","423.51","383.53","421.31","281.17","280.79"],
|
|
//"Provision for Income Taxes":["75.31","60.76","110.38","106.36","85.40","137.09","89.56","87.48"],
|
|
//"Income Avail. to Common Excl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
//"Diluted Average Shares":["55.71","55.14","55.66","55.60","56.72","57.09","56.97","56.78"],
|
|
//"Diluted EPS Excl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"],
|
|
//"Diluted EPS Incl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"]}}
|
|
public static List<IncomeStatement> GetIncomeStatementFinViz(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
String baseCurrency="USD";
|
|
try
|
|
{
|
|
List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
|
|
if(IncomeStatement.PeriodType.Annual.Equals(periodType))sb.Append("&s=IA");
|
|
else sb.Append("&s=IQ");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
String currency="USD";
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
|
|
if(null!=(jsonToken=json.SelectToken("currency")))
|
|
{
|
|
currency=jsonToken.ToObject<String>();
|
|
}
|
|
|
|
if(null==(jsonToken=json.SelectToken("data")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
|
|
return null;
|
|
}
|
|
|
|
List<String> strDateItems=null;
|
|
List<String> strTotalRevenueItems=null;
|
|
List<String> strCostOfRevenueItems=null;
|
|
List<String> strSGAItems=null;
|
|
List<String> strGrossProfitItems=null;
|
|
List<String> strTotalOperatingExpense=null;
|
|
List<String> strResearchAndDevelopmentItems=null;
|
|
List<String> strIncomeAvailableToCommonShareHolders=null;
|
|
|
|
JToken token=jsonToken["Period End Date"];
|
|
if(null==token)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No IncomeStatemnt dates returned for {0}",symbol));
|
|
return null;
|
|
}
|
|
strDateItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Revenue"];
|
|
if(null!=token)strTotalRevenueItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Cost of Revenue"];
|
|
if(null!=token)strCostOfRevenueItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Selling, General and Administrative"];
|
|
if(null!=token)strSGAItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Gross Profit"];
|
|
if(null!=token)strGrossProfitItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Operating Expense"];
|
|
if(null!=token)strTotalOperatingExpense=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Research and Development"];
|
|
if(null!=token)strResearchAndDevelopmentItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Income Avail. to Common Incl. Extraord."];
|
|
if(null!=token)strIncomeAvailableToCommonShareHolders=token.ToObject<List<String>>();
|
|
|
|
for(int index=0;index<strDateItems.Count;index++)
|
|
{
|
|
String strDateItem=strDateItems[index];
|
|
if("TTM".Equals(strDateItem,StringComparison.CurrentCultureIgnoreCase))continue;
|
|
IncomeStatement incomeStatement=new IncomeStatement();
|
|
incomeStatement.Symbol=symbol;
|
|
incomeStatement.Period=periodType;
|
|
incomeStatement.AsOf=Utility.ParseDate(strDateItems[index]);
|
|
incomeStatement.Modified=DateTime.Now;
|
|
double multiplier=1.00;
|
|
|
|
// perform currency conversion
|
|
if(!baseCurrency.Equals(currency))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,incomeStatement.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,incomeStatement.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,incomeStatement.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString()));
|
|
continue;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,incomeStatement.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for IncomeStatement : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(incomeStatement.AsOf,currencyConversionElement);
|
|
}
|
|
multiplier=conversionRates.ContainsKey(incomeStatement.AsOf)?conversionRates[incomeStatement.AsOf].UnitsPerSource:multiplier;
|
|
|
|
if(null!=strTotalRevenueItems && strTotalRevenueItems.Count>index)incomeStatement.TotalRevenue=FeedParser.ParseValue(strTotalRevenueItems[index])*1000000.00*multiplier;
|
|
if(null!=strCostOfRevenueItems && strCostOfRevenueItems.Count>index)incomeStatement.CostOfRevenue=FeedParser.ParseValue(strCostOfRevenueItems[index])*1000000.00*multiplier;
|
|
if(null!=strSGAItems && strSGAItems.Count>index)incomeStatement.SGA=FeedParser.ParseValue(strSGAItems[index])*1000000.00*multiplier;
|
|
if(null!=strGrossProfitItems && strGrossProfitItems.Count>index)incomeStatement.GrossProfit=FeedParser.ParseValue(strGrossProfitItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalOperatingExpense && strTotalOperatingExpense.Count>index)incomeStatement.OperatingExpenses=FeedParser.ParseValue(strTotalOperatingExpense[index])*1000000.00*multiplier;
|
|
if(null!=strResearchAndDevelopmentItems && strResearchAndDevelopmentItems.Count>index)incomeStatement.ResearchAndDevelopment=FeedParser.ParseValue(strResearchAndDevelopmentItems[index])*1000000.00*multiplier;
|
|
if(null!=strIncomeAvailableToCommonShareHolders && strIncomeAvailableToCommonShareHolders.Count>index)incomeStatement.NetIncomeApplicableToCommonShares=FeedParser.ParseValue(strIncomeAvailableToCommonShareHolders[index])*1000000.00*multiplier;
|
|
incomeStatements.Add(incomeStatement);
|
|
}
|
|
return incomeStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
public static List<IncomeStatement> GetIncomeStatementNASDAQ(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
double multiplier=1000.00;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
|
|
if(periodType.Equals(IncomeStatement.PeriodType.Annual))sb.Append("");
|
|
else sb.Append("?frequency=2");
|
|
|
|
strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success) return incomeStatements;
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
if(null==(jsonToken=json.SelectToken("data.incomeStatementTable.headers")))
|
|
{
|
|
String reason="unknown";
|
|
jsonToken=json.SelectToken("message");
|
|
if(null!=jsonToken)reason=jsonToken.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
|
|
return incomeStatements;
|
|
}
|
|
JObject headerObject=(JObject)jsonToken;
|
|
int elements=headerObject.Count-1;
|
|
String headerElement=headerObject["value1"].ToString();
|
|
if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
|
|
return incomeStatements;
|
|
}
|
|
|
|
for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
|
|
{
|
|
IncomeStatement incomeStatement=new IncomeStatement();
|
|
String itemIndex="value"+headerIndex;
|
|
jsonToken=json.SelectToken("data.incomeStatementTable.headers."+itemIndex);
|
|
if(null==jsonToken)continue;
|
|
String strDate=jsonToken.ToString();
|
|
if(string.IsNullOrEmpty(strDate))continue;
|
|
incomeStatement.AsOf=Utility.ParseDate(strDate);
|
|
incomeStatement.Symbol=symbol.ToUpper();
|
|
incomeStatement.Period=periodType;
|
|
incomeStatements.Add(incomeStatement);
|
|
}
|
|
|
|
jsonToken=json.SelectToken("data.incomeStatementTable.rows");
|
|
if(null==jsonToken)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
|
|
return incomeStatements;
|
|
}
|
|
|
|
JArray collection=(JArray)jsonToken;
|
|
for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
|
|
{
|
|
JObject rowItem=(JObject)collection[rowIndex];
|
|
String strHeader=rowItem.SelectToken("value1").ToString();
|
|
|
|
if(null==strHeader)continue;
|
|
if("Total Revenue".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].TotalRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Cost of Revenue".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].CostOfRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Gross Profit".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].GrossProfit=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Operating Expenses".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Research and Development".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].ResearchAndDevelopment=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Sales, General and Admin.".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].SGA=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Non-Recurring Items".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Operating Items".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Operating Income".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Add'l income/expense items".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Earnings Before Interest and Tax".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].EBIT=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Interest Expense".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].InterestExpense=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Earnings Before Tax".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Income Tax".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].IncomeTaxExpense=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Minority Interest".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Equity Earnings/Loss Unconsolidated Subsidiary".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Income-Cont. Operations".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Income".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].NetIncome=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Net Income Applicable to Common Shareholders".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].NetIncomeApplicableToCommonShares=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
}
|
|
return incomeStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
|
|
}
|
|
}
|
|
|
|
// *******************************************************************************************************************************************************************
|
|
// ************************************************************B A L A N C E S H E E T - N A S D A Q / F I N V I Z **************************************************
|
|
// *******************************************************************************************************************************************************************
|
|
public static List<BalanceSheet> GetBalanceSheet(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
List<BalanceSheet> balanceSheet=null;
|
|
balanceSheet=GetBalanceSheetNASDAQ(symbol,periodType);
|
|
if(null!=balanceSheet && balanceSheet.Count>0)return balanceSheet;
|
|
balanceSheet=GetBalanceSheetFinViz(symbol,periodType);
|
|
return balanceSheet;
|
|
}
|
|
//https://finviz.com/api/statement.ashx?t=aapl&s=BA
|
|
//https://finviz.com/api/statement.ashx?t=aapl&s=BQ
|
|
//{"currency":"USD",
|
|
//"data":{
|
|
//"Period End Date":["9/26/2020","9/28/2019","9/29/2018","9/30/2017","9/24/2016","9/26/2015","9/27/2014","9/28/2013"],
|
|
//"Cash and Equivalents":["20,243.00","36,640.00","14,338.00","12,307.00","11,883.00","9,731.00","3,612.00","5,554.00"],
|
|
//"Short Term Investments":["52,927.00","51,713.00","40,388.00","53,892.00","46,671.00","20,481.00","11,233.00","26,287.00"],
|
|
//"Cash and Short Term Investments":["90,943.00","100,557.00","66,301.00","74,181.00","67,155.00","41,601.00","25,077.00","40,546.00"],
|
|
//"Accounts Receivable - Trade, Net":["16,120.00","22,926.00","23,186.00","17,874.00","15,754.00","16,849.00","17,460.00","13,102.00"],
|
|
//"Total Receivables, Net":["37,445.00","45,804.00","48,995.00","35,673.00","29,299.00","30,343.00","27,219.00","20,641.00"],
|
|
//"Total Inventory":["4,061.00","4,106.00","3,956.00","4,855.00","2,132.00","2,349.00","2,111.00","1,764.00"],
|
|
//"Total Current Assets":["143,713.00","162,819.00","131,339.00","128,645.00","106,869.00","89,378.00","68,531.00","73,286.00"],
|
|
//"Property, Plant And Equipment - Gross":["112,096.00","95,957.00","90,403.00","75,076.00","61,245.00","49,257.00","39,015.00","28,519.00"],
|
|
//"Property, Plant And Equipment - Net":["45,336.00","37,378.00","41,304.00","33,783.00","27,010.00","22,471.00","20,624.00","16,597.00"],
|
|
//"Goodwill, Net":["","","","","5,414.00","5,116.00","4,616.00","1,577.00"],
|
|
//"Intangibles, Net":["","","","","3,206.00","3,893.00","4,142.00","4,179.00"],
|
|
//"Long Term Investments":["100,887.00","105,341.00","170,799.00","194,714.00","170,430.00","164,065.00","130,162.00","106,215.00"],
|
|
//"Other Long Term Assets":["33,952.00","32,978.00","22,283.00","18,177.00","8,757.00","5,422.00","3,764.00","5,146.00"],
|
|
//"Total Assets":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
|
|
//"Accounts Payable":["42,296.00","46,236.00","55,888.00","44,242.00","37,294.00","35,490.00","30,196.00","22,367.00"],
|
|
//"Accrued Expenses":["1,436.00","","","","20,951.00","24,169.00","7,689.00","4,782.00"],
|
|
//"Notes Payable/Short Term Debt":["4,996.00","5,980.00","11,964.00","11,977.00","8,105.00","8,499.00","6,308.00","0.00"],
|
|
//"Current Port. of LT Debt/Capital Leases":["8,797.00","10,260.00","8,784.00","6,496.00","3,500.00","2,500.00","",""],
|
|
//"Other Current liabilities":["47,867.00","43,242.00","39,293.00","38,099.00","9,156.00","9,952.00","19,255.00","16,509.00"],
|
|
//"Total Current Liabilities":["105,392.00","105,718.00","115,929.00","100,814.00","79,006.00","80,610.00","63,448.00","43,658.00"],
|
|
//"Long Term Debt":["98,667.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
|
|
//"Capital Lease Obligations":["637.00","","","","","","",""],
|
|
//"Total Long Term Debt":["99,304.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
|
|
//"Total Debt":["113,097.00","108,047.00","114,483.00","115,680.00","87,032.00","64,328.00","35,295.00","16,960.00"],
|
|
//"Other Liabilities":["53,853.00","50,503.00","48,914.00","11,747.00","12,985.00","12,989.00","7,598.00","6,344.00"],
|
|
//"Total Liabilities":["258,549.00","248,028.00","258,578.00","241,272.00","193,437.00","170,990.00","120,292.00","83,451.00"],
|
|
//"Common Stock":["50,779.00","45,174.00","40,201.00","35,867.00","31,251.00","27,416.00","23,313.00","19,764.00"],
|
|
//"Additional Paid-In Capital":["","","","","","","",""],
|
|
//"Retained Earnings":["14,966.00","45,898.00","70,400.00","98,330.00","96,364.00","92,284.00","87,152.00","104,256.00"],
|
|
//"Unrealized Gain (Loss)":["1,846.00","707.00","-3,209.00","328.00","1,174.00","-653.00","-242.00","-105.00"],
|
|
//"Other Equity":["-2,252.00","-1,291.00","-245.00","-478.00","-540.00","308.00","1,324.00","-366.00"],
|
|
//"Total Equity":["65,339.00","90,488.00","107,147.00","134,047.00","128,249.00","119,355.00","111,547.00","123,549.00"],
|
|
//"Total Liabilities and Equity":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
|
|
//"Total Common Shares Outstanding":["16,976.76","17,772.94","19,019.94","20,504.80","21,344.66","22,315.01","23,464.64","25,177.46"],
|
|
//"Full-Time Employees":["147,000","137,000","132,000","123,000","116,000","110,000","92,600","80,300"],
|
|
//"Number of Common Shareholders":["22,797","23,233","23,712","25,333","25,641","25,924","26,112","24,710"]}}
|
|
public static List<BalanceSheet> GetBalanceSheetFinViz(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
String baseCurrency="USD";
|
|
try
|
|
{
|
|
List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
|
|
if(BalanceSheet.PeriodType.Annual.Equals(periodType))sb.Append("&s=BA");
|
|
else sb.Append("&s=BQ");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
String currency="USD";
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
|
|
if(null!=(jsonToken=json.SelectToken("currency")))
|
|
{
|
|
currency=jsonToken.ToObject<String>();
|
|
}
|
|
|
|
if(null==(jsonToken=json.SelectToken("data")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
|
|
return null;
|
|
}
|
|
|
|
List<String> strDateItems=null;
|
|
List<String> strLongTermDebtItems=null;
|
|
List<String> strTotalCurrentAssetItems=null;
|
|
List<String> strTotalLiabilityItems=null;
|
|
List<String> strTotalAssetItems=null;
|
|
List<String> strGoodwillItems=null;
|
|
List<String> strPPEItems=null;
|
|
List<String> strCashAndEquivalentsItems=null;
|
|
List<String> strTotalInventoryItems=null;
|
|
List<String> strIntangiblesItems=null;
|
|
List<String> strTotalCurrentLiabilityItems=null;
|
|
List<String> strOtherLiabilityItems=null;
|
|
List<String> strTotalEquityItems=null;
|
|
|
|
JToken token=jsonToken["Period End Date"];
|
|
if(null==token)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No BalanceSheet dates returned for {0}",symbol));
|
|
return null;
|
|
}
|
|
strDateItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Long Term Debt"];
|
|
if(null!=token)strLongTermDebtItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Current Assets"];
|
|
if(null!=token)strTotalCurrentAssetItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Liabilities"];
|
|
if(null!=token)strTotalLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Assets"];
|
|
if(null!=token)strTotalAssetItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Goodwill, Net"];
|
|
if(null!=token)strGoodwillItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Property, Plant And Equipment - Net"];
|
|
if(null!=token)strPPEItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Cash and Equivalents"];
|
|
if(null!=token)strCashAndEquivalentsItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Inventory"];
|
|
if(null!=token)strTotalInventoryItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Intangibles, Net"];
|
|
if(null!=token)strIntangiblesItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Current Liabilities"];
|
|
if(null!=token)strTotalCurrentLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Other Liabilities"];
|
|
if(null!=token)strOtherLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Equity"];
|
|
if(null!=token)strTotalEquityItems=token.ToObject<List<String>>();
|
|
|
|
for(int index=0;index<strDateItems.Count;index++)
|
|
{
|
|
BalanceSheet balanceSheet=new BalanceSheet();
|
|
|
|
balanceSheet.Symbol=symbol;
|
|
balanceSheet.Period=periodType;
|
|
balanceSheet.AsOf=Utility.ParseDate(strDateItems[index]);
|
|
balanceSheet.Modified=DateTime.Now;
|
|
double multiplier=1.00;
|
|
|
|
// perform currency conversion
|
|
if(!baseCurrency.Equals(currency))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,balanceSheet.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,balanceSheet.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,balanceSheet.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString()));
|
|
continue;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,balanceSheet.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for BalanceSheet : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(balanceSheet.AsOf,currencyConversionElement);
|
|
}
|
|
multiplier=conversionRates.ContainsKey(balanceSheet.AsOf)?conversionRates[balanceSheet.AsOf].UnitsPerSource:multiplier;
|
|
|
|
if(null!=strLongTermDebtItems && strLongTermDebtItems.Count>index)balanceSheet.LongTermDebt=FeedParser.ParseValue(strLongTermDebtItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalCurrentAssetItems && strTotalCurrentAssetItems.Count>index)balanceSheet.TotalCurrentAssets=FeedParser.ParseValue(strTotalCurrentAssetItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalLiabilityItems && strTotalLiabilityItems.Count>index)balanceSheet.TotalLiabilities=FeedParser.ParseValue(strTotalLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalAssetItems && strTotalAssetItems.Count>index)balanceSheet.TotalAssets=FeedParser.ParseValue(strTotalAssetItems[index])*1000000.00*multiplier;
|
|
if(null!=strGoodwillItems && strGoodwillItems.Count>index)balanceSheet.Goodwill=FeedParser.ParseValue(strGoodwillItems[index])*1000000.00*multiplier;
|
|
if(null!=strPPEItems && strPPEItems.Count>index)balanceSheet.PropertyPlantAndEquipment=FeedParser.ParseValue(strPPEItems[index])*1000000.00*multiplier;
|
|
if(null!=strCashAndEquivalentsItems && strCashAndEquivalentsItems.Count>index)balanceSheet.CashAndCashEquivalents=FeedParser.ParseValue(strCashAndEquivalentsItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalInventoryItems && strTotalInventoryItems.Count>index)balanceSheet.Inventory=FeedParser.ParseValue(strTotalInventoryItems[index])*1000000.00*multiplier;
|
|
if(null!=strIntangiblesItems && strIntangiblesItems.Count>index)balanceSheet.IntangibleAssets=FeedParser.ParseValue(strIntangiblesItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalCurrentLiabilityItems && strTotalCurrentLiabilityItems.Count>index)balanceSheet.TotalCurrentLiabilities=FeedParser.ParseValue(strTotalCurrentLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strOtherLiabilityItems && strOtherLiabilityItems.Count>index)balanceSheet.OtherLiabilities=FeedParser.ParseValue(strOtherLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalEquityItems && strTotalEquityItems.Count>index)balanceSheet.TotalStockHolderEquity=FeedParser.ParseValue(strTotalEquityItems[index])*1000000.00*multiplier;
|
|
balanceSheets.Add(balanceSheet);
|
|
}
|
|
return balanceSheets;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
public static List<BalanceSheet> GetBalanceSheetNASDAQ(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
double multiplier=1000.00;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
|
|
if(periodType.Equals(BalanceSheet.PeriodType.Annual)) sb.Append("?frequency=1");
|
|
else sb.Append("?frequency=2");
|
|
|
|
strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success) return balanceSheets;
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
if(null==(jsonToken=json.SelectToken("data.balanceSheetTable.headers")))
|
|
{
|
|
String reason="unknown";
|
|
jsonToken=json.SelectToken("message");
|
|
if(null!=jsonToken) reason=jsonToken.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
|
|
return balanceSheets;
|
|
}
|
|
JObject headerObject=(JObject)jsonToken;
|
|
int elements=headerObject.Count-1;
|
|
String headerElement=headerObject["value1"].ToString();
|
|
if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
|
|
return balanceSheets;
|
|
}
|
|
|
|
for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
|
|
{
|
|
BalanceSheet balanceSheet =new BalanceSheet();
|
|
String itemIndex="value"+headerIndex;
|
|
jsonToken=json.SelectToken("data.balanceSheetTable.headers."+itemIndex);
|
|
if(null==jsonToken) continue;
|
|
String strDate=jsonToken.ToString();
|
|
if(string.IsNullOrEmpty(strDate))continue;
|
|
balanceSheet.AsOf=Utility.ParseDate(strDate);
|
|
balanceSheet.Symbol=symbol.ToUpper();
|
|
balanceSheet.Period=periodType;
|
|
balanceSheets.Add(balanceSheet);
|
|
}
|
|
|
|
jsonToken=json.SelectToken("data.balanceSheetTable.rows");
|
|
if(null==jsonToken)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
|
|
return balanceSheets;
|
|
}
|
|
|
|
JArray collection=(JArray)jsonToken;
|
|
for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
|
|
{
|
|
JObject rowItem=(JObject)collection[rowIndex];
|
|
String strHeader=rowItem.SelectToken("value1").ToString();
|
|
|
|
if(null==strHeader) continue;
|
|
if("Current Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Cash and Cash Equivalents".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].CashAndCashEquivalents=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Short-Term Investments".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Receivables".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Inventory".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].Inventory=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Current Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Current Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalCurrentAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Long-Term Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Long-Term Investments".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Fixed Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].PropertyPlantAndEquipment=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Goodwill".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].Goodwill=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Intangible Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].IntangibleAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Deferred Asset Charges".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Current Liabilities".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Accounts Payable".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Short-Term Debt / Current Portion of Long-Term Debt".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Current Liabilities".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Current Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalCurrentLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Long-Term Debt".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].LongTermDebt=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].OtherLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Deferred Liability Charges".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].DeferredLongTermLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Misc. Stocks".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Minority Interest".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].MinorityInterest=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Total Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Stock Holders Equity".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalStockHolderEquity=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Total Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Common Stocks".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Capital Surplus".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Retained Earnings".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Treasury Stock".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Liabilities & Equity".Equals(strHeader)||"Total Liabilities & Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
return balanceSheets;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetNASDAQ Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// **************************************************************************** B A L A N C E S H E E T - N A S D A Q ****************************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
CurrencyConversionCollection currencyConversionElements=new CurrencyConversionCollection();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sourceCurrency=sourceCurrency.ToUpper();
|
|
sb.Append("https://www.xe.com/currencytables/?from=").Append(sourceCurrency).Append("&date=").Append(Utility.DateTimeToStringYYYYHMMHDD(conversionDate)).Append("#table-section");
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
int itemIndex=0;
|
|
List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);
|
|
if(!Sections.FindInSections(sections,"Currency Table:",0,ref itemIndex,false))return null;
|
|
if(!Sections.FindInSections(sections,"Currency",itemIndex,ref itemIndex))return null;
|
|
itemIndex+=20;
|
|
|
|
while(true)
|
|
{
|
|
CurrencyConversionElement currencyConversionElement=new CurrencyConversionElement();
|
|
currencyConversionElement.AsOf=conversionDate;
|
|
currencyConversionElement.SourceCurrency=sourceCurrency;
|
|
currencyConversionElement.DestinationCurrency=sections[itemIndex];
|
|
itemIndex+=3;
|
|
currencyConversionElement.DestinationCurrencyName=sections[itemIndex];
|
|
itemIndex+=2;
|
|
currencyConversionElement.UnitsPerSource=FeedParser.ParseValue(sections[itemIndex]);
|
|
itemIndex+=2;
|
|
currencyConversionElement.SourcePerUnit=FeedParser.ParseValue(sections[itemIndex]);
|
|
currencyConversionElements.Add(currencyConversionElement);
|
|
itemIndex+=5;
|
|
if(itemIndex>=sections.Count || null==sections[itemIndex] || "".Equals(sections[itemIndex]) || sections[itemIndex].Length<3)break;
|
|
}
|
|
return currencyConversionElements;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** F U N D A M E N T A L - F I N V I S *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Fundamental GetFundamentalFinViz(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
Fundamental fundamental = new Fundamental();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower());
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Get Fundamental for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
// HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"snapshot-table2\"]");
|
|
// HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"snapshot-table2 screener_snapshot-table-body\"]");
|
|
HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"js-snapshot-table snapshot-table2 screener_snapshot-table-body\"]");
|
|
if(null==nodeCollection || nodeCollection.Count<1)return null;
|
|
fundamental.AsOf=DateTime.Now;
|
|
fundamental.Symbol=symbol;
|
|
fundamental.Source = "FINVIZ";
|
|
for(int index=0;index<nodeCollection[0].ChildNodes.Count;index++)
|
|
{
|
|
HtmlNodeCollection dataCollection = nodeCollection[0].ChildNodes[index].SelectNodes(".//td");
|
|
if(null==dataCollection)continue;
|
|
for(int dataIndex=0;dataIndex<dataCollection.Count;dataIndex++)
|
|
{
|
|
String header=dataCollection[dataIndex].InnerText;
|
|
if(header.Equals("Market Cap"))
|
|
{
|
|
fundamental.MarketCap=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("P/E"))
|
|
{
|
|
fundamental.PE=fundamental.TrailingPE=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("EPS (ttm)"))
|
|
{
|
|
fundamental.EPS=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Shs Outstand"))
|
|
{
|
|
fundamental.SharesOutstanding=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Income"))
|
|
{
|
|
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("PEG"))
|
|
{
|
|
fundamental.PEG=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Book/sh"))
|
|
{
|
|
fundamental.BookValuePerShare=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("ROA"))
|
|
{
|
|
fundamental.ReturnOnAssets=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("ROE"))
|
|
{
|
|
fundamental.ReturnOnEquity=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("52W Range"))
|
|
{
|
|
String strLowHigh=dataCollection[dataIndex+1].InnerText;
|
|
if(null==strLowHigh)continue;
|
|
String[] items=strLowHigh.Split('-');
|
|
if(null==items||2!=items.Length)continue;
|
|
items[0]=items[0].Trim();
|
|
items[1]=items[1].Trim();
|
|
fundamental.Low52=FeedParser.ParseValue(items[0]);
|
|
fundamental.High52=FeedParser.ParseValue(items[1]);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Beta"))
|
|
{
|
|
fundamental.Beta=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Earnings"))
|
|
{
|
|
String earnings=dataCollection[dataIndex+1].InnerText;
|
|
if(null==earnings)continue;
|
|
String[] items=earnings.Split(' ');
|
|
if(null==items||3!=items.Length)continue;
|
|
items[0]=items[0].Trim();
|
|
items[1]=items[1].Trim();
|
|
fundamental.NextEarningsDate=Utility.ParseDate(items[0]+" "+items[1]+" "+DateTime.Now.Year.ToString());
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Avg Volume"))
|
|
{
|
|
fundamental.Volume=(long)FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
}
|
|
} // for all child nodes
|
|
fundamental.Equity=fundamental.BookValuePerShare*fundamental.SharesOutstanding;
|
|
|
|
IncomeStatement incomeStatement=null;
|
|
List<IncomeStatement> incomeStatements=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
|
|
if(null!=incomeStatements&&0!=incomeStatements.Count) incomeStatement=incomeStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
|
|
CashflowStatement cashflowStatement=null;
|
|
List<CashflowStatement> cashflowStatements=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
|
|
if(null != cashflowStatements && 0!=cashflowStatements.Count)
|
|
{
|
|
cashflowStatement = cashflowStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
}
|
|
|
|
BalanceSheet balanceSheet=null;
|
|
List<BalanceSheet> balanceSheets=GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
|
|
if(null != balanceSheets && 0!=balanceSheets.Count)
|
|
{
|
|
balanceSheet = balanceSheets.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
}
|
|
|
|
if(null!=cashflowStatement)
|
|
{
|
|
fundamental.OperatingCashflow=cashflowStatement.OperatingCashflow;
|
|
fundamental.LeveragedFreeCashflow=cashflowStatement.FreeCashflow;
|
|
fundamental.EBITDA=fundamental.EBIT+cashflowStatement.DepreciationAndAmortization;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO CASHFLOW STATEMENT *************",symbol));
|
|
fundamental.OperatingCashflow=double.NaN;
|
|
fundamental.LeveragedFreeCashflow=double.NaN;
|
|
fundamental.EBITDA=double.NaN;
|
|
}
|
|
|
|
if(null!=incomeStatement)
|
|
{
|
|
fundamental.GrossProfit=incomeStatement.GrossProfit;
|
|
fundamental.Revenue=incomeStatement.TotalRevenue;
|
|
fundamental.EBIT=incomeStatement.EBIT;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO INCOME STATEMENT *************",symbol));
|
|
fundamental.GrossProfit=double.NaN;
|
|
fundamental.Revenue=double.NaN;
|
|
fundamental.EBIT=double.NaN;
|
|
}
|
|
|
|
if(null!=balanceSheet)
|
|
{
|
|
fundamental.TotalCash=balanceSheet.CashAndCashEquivalents;
|
|
fundamental.TotalDebt=balanceSheet.TotalLiabilities;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO BALANCESHEET *************",symbol));
|
|
fundamental.TotalCash=double.NaN;
|
|
fundamental.TotalDebt=double.NaN;
|
|
}
|
|
|
|
if(!double.IsNaN(fundamental.MarketCap)&&!double.IsNaN(fundamental.TotalDebt)&&!double.IsNaN(fundamental.TotalCash)) fundamental.EnterpriseValue=fundamental.MarketCap+fundamental.TotalDebt-fundamental.TotalCash;
|
|
else fundamental.EnterpriseValue=double.NaN;
|
|
if(!Double.IsNaN(fundamental.SharesOutstanding)&&0!=fundamental.SharesOutstanding)fundamental.RevenuePerShare=fundamental.Revenue/fundamental.SharesOutstanding;
|
|
else fundamental.RevenuePerShare=double.NaN;
|
|
if(null!=balanceSheet&&!double.IsNaN(balanceSheet.TotalStockHolderEquity)&&!double.IsNaN(fundamental.TotalDebt)&&0!=fundamental.TotalDebt)fundamental.DebtToEquity=fundamental.TotalDebt/balanceSheet.TotalStockHolderEquity;
|
|
else fundamental.DebtToEquity=double.NaN;
|
|
if(double.IsNaN(fundamental.DebtToEquity)&&!double.IsNaN(fundamental.Equity)&&0.00!=fundamental.Equity)fundamental.DebtToEquity=fundamental.TotalDebt/fundamental.Equity;
|
|
return fundamental;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}.",exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** F U N D A M E N T A L - Y A H O O F I N A N C E *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Fundamental GetFundamental(String symbol)
|
|
{
|
|
Fundamental fundamental=GetFundamentalEx(symbol);
|
|
return fundamental;
|
|
}
|
|
public static Fundamental GetFundamentalEx(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
Fundamental fundamental = new Fundamental();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("?p=").Append(symbol);
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
//MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("http://finance.yahoo.com/q?s=").Append(symbol).Append("&ql=1");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FAILED!, TRYING {1}",strRequest,sb.ToString()));
|
|
strRequest = sb.ToString();
|
|
//MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FINAL FAILURE.",strRequest));
|
|
return null;
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} SUCCESS.",strRequest));
|
|
fundamental.Symbol=symbol;
|
|
fundamental.AsOf=DateTime.Now;
|
|
fundamental.Source = "YAHOO";
|
|
fundamental.Beta=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Beta<"));
|
|
fundamental.NextEarningsDate=FeedParser.ParseValueDateTimeMonthFormat(Sections.LocateItemAcceptAnyText(httpNetResponse.ResponseString,">Earnings Date<"));
|
|
String strText=Sections.LocateFirstItem(httpNetResponse.ResponseString,">52 Week Range<");
|
|
if(null!=strText)
|
|
{
|
|
String[] strArray = strText.Split('-');
|
|
fundamental.Low52 = FeedParser.ParseValue(strArray[0].Trim());
|
|
fundamental.High52 = FeedParser.ParseValue(strArray[1].Trim());
|
|
}
|
|
fundamental.Volume=FeedParser.ParseValueLong(Sections.LocateItem(httpNetResponse.ResponseString,">Volume<"));
|
|
fundamental.MarketCap=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Market Cap<"));
|
|
fundamental.PE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PE Ratio (TTM)<"));
|
|
fundamental.EPS=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EPS (TTM)<"));
|
|
try { Thread.Sleep(250);PopulateKeyStatistics(fundamental,webProxy); }
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate key statistics.");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
|
|
}
|
|
try { Thread.Sleep(250);PopulateFinancials(fundamental); }
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate financials.");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
|
|
}
|
|
if(fundamental.IsZero())
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Fundamental for {0} is Invalid *****",symbol));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",fundamental.ToString()));
|
|
return null;
|
|
}
|
|
return fundamental;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// *************************************************************************************************************************************************************************************
|
|
// ***************************************************************** K E Y S T A T I S T I C S - Y A H O O F I N A N C E **********************************************************
|
|
// *************************************************************************************************************************************************************************************
|
|
private static bool PopulateKeyStatistics(Fundamental fundamental,WebProxy webProxy=null)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("http://finance.yahoo.com/q/ks?s=").Append(fundamental.Symbol);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return false;
|
|
}
|
|
fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));
|
|
fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
|
|
fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
|
|
fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
|
|
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
|
|
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
|
|
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
|
|
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
|
|
fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
|
|
fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
|
|
fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
|
|
fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
|
|
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
|
|
fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
|
|
fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
|
|
fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
|
|
fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));
|
|
return true;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// *************************************************************************************************************************************************************************************
|
|
// ***************************************************************** F I N A N C I A L S - Y A H O O F I N A N C E *****************************************************************
|
|
// *************************************************************************************************************************************************************************************
|
|
private static bool PopulateFinancials(Fundamental fundamental)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(fundamental.Symbol).Append("/financials?p=").Append(fundamental.Symbol);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return false;
|
|
}
|
|
fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
|
|
return true;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
//**************************************************************************************************************************************************************************************
|
|
//*************************************************************************** I N T R A D A Y P R I C I N G **************************************************************************
|
|
// ************************************************************** M A I N : Y A H O O **************************************************************************************************
|
|
// ************************************************************** B A C K U P : B I G C H A R T S **************************************************************************************
|
|
//**************************************************************************************************************************************************************************************
|
|
|
|
private delegate Price LatestPriceDelegate(String symbol);
|
|
public static Price GetLatestPrice(String symbol)
|
|
{
|
|
DateTime currentMarketDate=PremarketDA.GetLatestMarketDate();
|
|
LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceYahoo,GetLatestPriceBigCharts,GetLatestPriceFidelity};
|
|
Price latestPrice=null;
|
|
foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates)
|
|
{
|
|
latestPrice=latestPriceDelegate.Invoke(symbol);
|
|
if(null!=latestPrice && latestPrice.IsValid)
|
|
{
|
|
if(latestPrice.Date.Date.Equals(currentMarketDate.Date.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Price request from {0} succeeded for {1}",latestPriceDelegate.GetMethodInfo().Name,symbol));
|
|
return latestPrice;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date. Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString()));
|
|
}
|
|
}
|
|
}
|
|
latestPrice=GetPriceAsOf(symbol,currentMarketDate); // try to get the price from MarketWatch historical feed using todays date
|
|
if(null!=latestPrice && latestPrice.Date.Date.Equals(currentMarketDate.Date)) return latestPrice;
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if(null!=companyProfile && companyProfile.CanRollPrevious)
|
|
{
|
|
latestPrice=PricingMarketDataHelper.RollPriceForward(symbol);
|
|
if(null!=latestPrice)return latestPrice;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol));
|
|
return null;
|
|
}
|
|
|
|
// This should be proxied through TOR because Yahoo tracks IP address.
|
|
// This is used in the intra-day price feed
|
|
private static Price GetLatestPriceYahoo(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
if(null==symbol)return null;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol);
|
|
strRequest=sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
Price price=new Price();
|
|
price.Symbol=symbol.ToUpper();
|
|
price.Date=DateTime.Now;
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
return BuildPriceFromResponse(httpNetResponse.ResponseString,price);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// This is used in the intra-day price feed.
|
|
private static Price GetLatestPriceBigCharts(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
if(null==symbol) return null;
|
|
sb.Append("http://bigcharts.marketwatch.com/quickchart/quickchart.asp?symb=").Append(symbol).Append("&insttype=Stock").Append(symbol);
|
|
strRequest = sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceBigCharts");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
Price price = new Price();
|
|
price.Symbol = symbol.ToUpper();
|
|
price.Date = DateTime.Now;
|
|
price.Source=Price.PriceSource.BigCharts;
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quickchart\"]");
|
|
if(null==table||0==table.Count)return null;
|
|
HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"maincontent\"]");
|
|
if(null==divSection)return null;
|
|
HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
|
|
if(rows.Count<3)return null;
|
|
HtmlNode softTimeNode = htmlDocument.DocumentNode.SelectSingleNode("//*[@class=\"soft time\"]");
|
|
if(null!=softTimeNode)
|
|
{
|
|
DateTime givenDate=Utility.ParseDate(softTimeNode.InnerHtml);
|
|
if(!Utility.IsEpoch(givenDate))price.Date=givenDate;
|
|
}
|
|
HtmlNodeCollection dataColumns = rows[2].SelectNodes(".//td");
|
|
for(int index=0;index<dataColumns.Count;index++)
|
|
{
|
|
String innerText=Utility.RemoveControlChars(dataColumns[index].InnerText).Trim();
|
|
String[] items=innerText.Split(':');
|
|
if(null==items||2!=items.Length)continue;
|
|
if("Last".Equals(items[0])){price.Close=FeedParser.ParseValue(items[1]);price.AdjClose=price.Close;}
|
|
else if("Open".Equals(items[0]))price.Open=FeedParser.ParseValue(items[1]);
|
|
else if("High".Equals(items[0]))price.High=FeedParser.ParseValue(items[1]);
|
|
else if("Low".Equals(items[0]))price.Low=FeedParser.ParseValue(items[1]);
|
|
else if("Volume".Equals(items[0]))price.Volume=FeedParser.ParseValueLong(items[1]);
|
|
}
|
|
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",price.Symbol));
|
|
return null;
|
|
}
|
|
if(double.IsNaN(price.Open))price.Open=0.00;
|
|
if(double.IsNaN(price.Low))price.Low=0.00;
|
|
if(double.IsNaN(price.High))price.High=0.00;
|
|
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
|
|
{
|
|
price.Open=price.High=price.Low=price.Close;
|
|
}
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
// Intraday Pricing from Fidelity - which is being tested 07/22/2021
|
|
// This is being used in the intra-day price feed.
|
|
public static Price GetLatestPriceFidelity(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
if(null==symbol) return null;
|
|
sb.Append("https://quotes.fidelity.com/webxpress/get_quote?QUOTE_TYPE=D&SID_VALUE_ID=").Append(symbol).Append("&submit=Quote");
|
|
strRequest = sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceFidelity");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV1(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
Price price = new Price();
|
|
price.Symbol = symbol.ToUpper();
|
|
price.Source=Price.PriceSource.Fidelity;
|
|
price.Close=price.Open=price.High=price.Low=price.AdjClose=double.NaN;
|
|
price.Date = DateTime.Now;
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"quotesDetailsTable\"]");
|
|
if(null==table||0==table.Count)return null;
|
|
|
|
HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
|
|
if(null==rows || 0==rows.Count)return null;
|
|
double bid=double.NaN;
|
|
double ask=double.NaN;
|
|
for(int rowIndex=0;rowIndex<rows.Count;rowIndex++)
|
|
{
|
|
int itemIndex;
|
|
List<String> sections=Sections.GetSections(rows[rowIndex].InnerHtml);
|
|
if(null==sections || 0==sections.Count)continue;
|
|
|
|
itemIndex=sections.FindIndex(delegate(string str){return str.StartsWith("As of:");});
|
|
if(-1!=itemIndex)
|
|
{
|
|
String strDate=Utility.BetweenString(sections[itemIndex],":",",");
|
|
if(null==strDate)continue;
|
|
{
|
|
strDate=strDate.Trim();
|
|
price.Date=Utility.ParseDate(strDate);
|
|
}
|
|
}
|
|
itemIndex=sections.IndexOf("Last Trade [tick]");
|
|
if(-1!=itemIndex)
|
|
{
|
|
price.Close=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Volume");
|
|
if(-1!=itemIndex)
|
|
{
|
|
price.Volume=FeedParser.ParseValueLong(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Bid");
|
|
if(-1!=itemIndex)
|
|
{
|
|
bid=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Ask");
|
|
if(-1!=itemIndex)
|
|
{
|
|
ask=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Open");
|
|
if(-1!=itemIndex)
|
|
{
|
|
price.Open=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Day High");
|
|
if(-1!=itemIndex)
|
|
{
|
|
price.High=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
itemIndex=sections.IndexOf("Day Low");
|
|
if(-1!=itemIndex)
|
|
{
|
|
price.Low=FeedParser.ParseValue(sections[itemIndex+2]);
|
|
}
|
|
}
|
|
if((double.IsNaN(price.Close) || 0==price.Close) && !double.IsNaN(bid) && !double.IsNaN(ask))
|
|
{
|
|
price.Close=(bid+ask)/2.00;
|
|
}
|
|
if(0!=price.Close && 0==price.Open && 0==price.High && 0==price.Low)
|
|
{
|
|
price.Open=price.High=price.Low=price.Close;
|
|
}
|
|
price.AdjClose=price.Close;
|
|
if(double.IsNaN(price.Open))price.Open=0.00;
|
|
if(double.IsNaN(price.Low))price.Low=0.00;
|
|
if(double.IsNaN(price.High))price.High=0.00;
|
|
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",price.Symbol));
|
|
return null;
|
|
}
|
|
if(!price.IsValid)return null;
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
// *******************************************************************************************************************************************************************************
|
|
// ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ********************************************************
|
|
// *******************************************************************************************************************************************************************************
|
|
public static Prices GetPricesAsOf(String symbol, DateTime startDate,DateTime endDate)
|
|
{
|
|
Prices prices=new Prices();
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
List<DateTime> dates=dateGenerator.GenerateHistoricalDates(startDate,endDate);
|
|
for(int index=0;index<dates.Count;index++)
|
|
{
|
|
DateTime currentDate=dates[index];
|
|
Price price=GetPriceAsOf(symbol,currentDate);
|
|
if(null==price){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data for {0} on {1}",symbol,Utility.DateTimeToStringMMHDDHYYYY(currentDate)));continue;};
|
|
prices.Add(price);
|
|
}
|
|
return prices;
|
|
}
|
|
public static Price GetPriceAsOf(String symbol, DateTime asOf)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
String strRequest;
|
|
StringBuilder sb = null;
|
|
|
|
if (null == symbol) return null;
|
|
String requestSymbol=symbol;
|
|
if(requestSymbol.StartsWith("^"))requestSymbol=requestSymbol.Substring(1);
|
|
sb = new StringBuilder();
|
|
sb.Append("http://bigcharts.marketwatch.com/historical/default.asp?symb=");
|
|
sb.Append(requestSymbol);
|
|
sb.Append("&closeDate=");
|
|
sb.Append(asOf.Month.ToString());
|
|
sb.Append("%2F");
|
|
sb.Append(asOf.Day.ToString());
|
|
sb.Append("%2F");
|
|
sb.Append((asOf.Year-2000).ToString());
|
|
sb.Append("&x=38&y=25");
|
|
strRequest = sb.ToString();
|
|
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
MemoryStream memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"historicalquote fatbottomed\"]");
|
|
if (null == tables || 0 == tables.Count) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
if (rows.Count < 7) return null;
|
|
Price price=new Price();
|
|
price.Source=Price.PriceSource.BigCharts;
|
|
price.Symbol = symbol.ToUpper();
|
|
price.Date=FeedParser.ParseValueDateTimeMonthFormatFromMarketWatch(rows[1].InnerText);
|
|
price.Close=FeedParser.ParseValueFromMarketWatch("Closing Price:",rows[2].InnerText);
|
|
price.AdjClose = price.Close;
|
|
price.Open = FeedParser.ParseValueFromMarketWatch("Open:", rows[3].InnerText);
|
|
price.High = FeedParser.ParseValueFromMarketWatch("High:", rows[4].InnerText);
|
|
price.Low = FeedParser.ParseValueFromMarketWatch("Low:", rows[5].InnerText);
|
|
price.Volume = FeedParser.ParseLongValueFromMarketWatch("Volume:", rows[6].InnerText);
|
|
if(!(price.Date.Date.Equals(asOf.Date.Date)))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The price retrieved for {0} does not contain the requested date. Requested date {1} Retrieved date {2}",symbol,price.Date.ToShortDateString(),asOf.ToShortDateString()));
|
|
return null;
|
|
}
|
|
return price;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
//********************************************************************************************************************************************************************************************************
|
|
// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O *********************************************************************
|
|
//********************************************************************************************************************************************************************************************************
|
|
// *********************************************************************************************************************************************************
|
|
// *********************************************************************************************************************************************************
|
|
|
|
// ********************************************************************************************************************************************************************************************************************
|
|
// ******************************************************************************************************** N E W V E R S I O N *************************************************************************************
|
|
// ********************************************************************************************************************************************************************************************************************
|
|
// Implemented 06/10/2022
|
|
|
|
public static Price GetDailyPrice(String symbol,DateTime pricingDate)
|
|
{
|
|
Prices prices=GetDailyPrices(symbol,pricingDate,pricingDate);
|
|
if(null==prices||0==prices.Count)return null;
|
|
Price price=prices.FirstOrDefault();
|
|
if(!price.Date.Date.Equals(pricingDate.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'",symbol,pricingDate.ToShortDateString(),price.Date.Date.ToShortDateString()));
|
|
return null;
|
|
}
|
|
return price;
|
|
}
|
|
|
|
public static Prices GetDailyPrices(String symbol, DateTime startDate, DateTime endDate)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
if(startDate>endDate)
|
|
{
|
|
DateTime tempDate=startDate;
|
|
startDate=endDate;
|
|
endDate=tempDate;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[MarketDataHelper:GetDailyPrices]{0} start:{1} end:{2}",symbol,Utility.DateTimeToStringMMHDDHYYYY(startDate),Utility.DateTimeToStringMMHDDHYYYY(endDate)));
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
Prices prices=null;
|
|
sb.Append("https://query1.finance.yahoo.com/v7/finance/chart/").Append(symbol).Append("?period1=").Append(Utility.DateToUnixDate(startDate)).Append("&period2=").Append(Utility.DateToUnixDate(endDate)).Append("&interval=1d&events=history");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Query:{0}",strRequest));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetDailyPrices");
|
|
CookieCollection cookieCollection=new CookieCollection();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3B(strRequest,cookieCollection,webProxy);
|
|
if(!httpNetResponse.Success) return null;
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
prices=GetPricesFromJSONString(json,symbol);
|
|
return prices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
private static Prices GetPricesFromJSONString(JObject json,String symbol)
|
|
{
|
|
Prices prices=new Prices();
|
|
|
|
try
|
|
{
|
|
JToken token=null;
|
|
int items=0;
|
|
|
|
try{token=json["chart"]["result"][0]["timestamp"];}catch(Exception){;}
|
|
if(null==token)
|
|
{
|
|
string instrumentType=null;
|
|
try{instrumentType=json["chart"]["result"][0]["meta"]["instrumentType"].ToString();}catch(Exception){;}
|
|
if(null==instrumentType||!instrumentType.Equals("MUTUALFUND"))return prices;
|
|
String strPrice=null;
|
|
String strDate=null;
|
|
try{strPrice=json["chart"]["result"][0]["meta"]["regularMarketPrice"].ToString();}catch(Exception){;}
|
|
try{strDate=json["chart"]["result"][0]["meta"]["currentTradingPeriod"]["regular"]["start"].ToString();}catch(Exception){;}
|
|
if(null==strPrice||null==strDate)return prices;
|
|
Price price=new Price();
|
|
price.Symbol=symbol;
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
price.High=price.Low=price.Close=price.Open=price.AdjClose=FeedParser.ParseValue(strPrice);
|
|
DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strDate));
|
|
price.Date=dateTimeOffset.DateTime;
|
|
price.Volume=0L;
|
|
prices.Add(price);
|
|
return prices;
|
|
}
|
|
try{items=token.Children().Count();}catch(Exception){;}
|
|
if(0==items)return prices;
|
|
for(int index=0;index<items;index++)
|
|
{
|
|
String strTradeDate=(json["chart"]["result"][0]["timestamp"][index]).ToString();
|
|
String strHigh=null;
|
|
String strOpen=null;
|
|
String strLow=null;
|
|
String strClose=null;
|
|
String strVolume=null;
|
|
String strAdjClose=null;
|
|
|
|
try{strHigh=(json["chart"]["result"][0]["indicators"]["quote"][0]["high"][index]).ToString();}catch(Exception){;}
|
|
try{strOpen=(json["chart"]["result"][0]["indicators"]["quote"][0]["open"][index]).ToString();}catch(Exception){;}
|
|
try{strLow=(json["chart"]["result"][0]["indicators"]["quote"][0]["low"][index]).ToString();}catch(Exception){;}
|
|
try{strClose=(json["chart"]["result"][0]["indicators"]["quote"][0]["close"][index]).ToString();}catch(Exception){;}
|
|
try{strVolume=(json["chart"]["result"][0]["indicators"]["quote"][0]["volume"][index]).ToString();}catch(Exception){;}
|
|
try{strAdjClose=(json["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"][index]).ToString();}catch(Exception){;}
|
|
|
|
DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strTradeDate));
|
|
Price price=new Price();
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
price.Symbol=symbol;
|
|
price.Date=dateTimeOffset.DateTime;
|
|
price.High=FeedParser.ParseValue(strHigh);
|
|
price.Open=FeedParser.ParseValue(strOpen);
|
|
price.Close=FeedParser.ParseValue(strClose);
|
|
price.Low=FeedParser.ParseValue(strLow);
|
|
price.Volume=(long)FeedParser.ParseValue(strVolume);
|
|
price.AdjClose=FeedParser.ParseValue(strAdjClose);
|
|
|
|
if(price.Open<=0.05&&price.High<=0.05&&price.Low<=0.05&&price.Close<=0.05&&price.AdjClose<=0.05)continue;
|
|
if(double.IsNaN(price.Close)||price.Close==0.00)continue;
|
|
if(price.Volume<0)continue;
|
|
if(price.Open==0.00||double.IsNaN(price.Open))price.Open=price.Close;
|
|
if(price.High==0.00||double.IsNaN(price.High))price.High=price.Close;
|
|
if(price.Low==0.00||double.IsNaN(price.Low))price.Low=price.Close;
|
|
if(price.Close<price.Low||price.Close>price.High)price.Close=(price.High+price.Low)/2.00;
|
|
|
|
prices.Add(price);
|
|
|
|
}
|
|
return prices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return prices;
|
|
}
|
|
}
|
|
|
|
// ******************************************************************************************************************************************************************************************************************
|
|
// *************************************************************************** E N D Y A H O O H I S T O R I C A L P R I C I N G *******************************************************************************
|
|
// ******************************************************************************************************************************************************************************************************************
|
|
|
|
|
|
// This function attempts to locate the quote data within the raw data returned from Yahoo. It should be interpreted as intraday data because
|
|
// the market date cannot be located within the raw data response.
|
|
public static Price BuildPriceFromResponse(String httpNetResponse,Price price)
|
|
{
|
|
String strResponse=Utility.KeepAfter(httpNetResponse,"/* -- Data -- */");
|
|
String symbol=price.Symbol.ToUpper();
|
|
bool isMutualFund=false;
|
|
String findPattern1="\"quoteType\":\"MUTUALFUND\",\"symbol\":"+"\""+symbol+"\"";
|
|
String findPattern2="\"quoteType\":\"EQUITY\",\"symbol\":"+"\""+symbol+"\"";
|
|
String findPattern3="\"quoteType\":\"ETF\",\"symbol\":"+"\""+symbol+"\"";
|
|
String findPattern4="\"quoteType\":\"INDEX\",\"symbol\":"+"\""+symbol+"\"";
|
|
strResponse=Utility.KeepAfter(httpNetResponse,findPattern1);
|
|
if(null!=strResponse)isMutualFund=true;
|
|
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern2);
|
|
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern3);
|
|
if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern4);
|
|
if(null==strResponse)return null;
|
|
strResponse=Utility.KeepBefore(strResponse,"MOST_ACTIVE_TITLE");
|
|
price.Open=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketOpen\":{\"raw\":",','));
|
|
price.High=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayHigh\":{\"raw\":",','));
|
|
price.Low=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayLow\":{\"raw\":",','));
|
|
price.Close=FeedParser.ParseValue(Utility.FindFirst(Utility.KeepAfterLast(strResponse,"\"regularMarketDayLow\":{\"raw\":"),"\"regularMarketPrice\":{\"raw\":",','));
|
|
price.Volume=FeedParser.ParseValueLong(Utility.FindFirst(strResponse,"\"regularMarketVolume\":{\"raw\":",','));
|
|
if(double.NaN.Equals(price.Close))price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPrice\":{\"raw\":",','));
|
|
if(double.IsNaN(price.Close)) // try to find raw quote data
|
|
{
|
|
String strQuote=Utility.KeepAfter(httpNetResponse,"<!-- react-text: 36");
|
|
if(null==strQuote&&httpNetResponse.Contains("<span data-reactid=\"38\">At close:"))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***Retrieving price from BigCharts for {0}***",symbol));
|
|
return GetPriceAsOf(symbol,DateTime.Now); // if the market is closed then get the price from BigCharts
|
|
}
|
|
if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,"-->",'<'));
|
|
else if(null==strQuote&&httpNetResponse.Contains("data-reactid=\"38\">As of"))
|
|
{
|
|
strQuote=Utility.KeepAfter(httpNetResponse,"class=\"Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)\"");
|
|
if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,">",'<'));
|
|
}
|
|
if(!double.IsNaN(price.Close))MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Found quote data for {0}->{1}",symbol,Utility.FormatCurrency(price.Close)));
|
|
}
|
|
if(double.IsNaN(price.Close))
|
|
{
|
|
double bid=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"bid\":{\"raw\":",','));
|
|
double ask=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"ask\":{\"raw\":",','));
|
|
if(!double.IsNaN(bid)&&!double.IsNaN(ask))price.Close=(bid+ask)/2.00;
|
|
}
|
|
if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
|
|
{
|
|
price.Close=price.AdjClose=(price.High+price.Low)/2.00;
|
|
}
|
|
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
|
|
{
|
|
// if this is a mutual fund then the regularMarketPreviousClose may actually be the close from two days ago and not last night. Goto BigCharts in this case
|
|
if(isMutualFund)price=GetPriceAsOf(symbol,DateTime.Now);
|
|
else price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPreviousClose\":{\"raw\":",','));
|
|
if(null==price||double.IsNaN(price.Close))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",symbol));
|
|
return null;
|
|
}
|
|
price.Open=price.High=price.Low=price.AdjClose=price.Close;
|
|
return price;
|
|
}
|
|
if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
|
|
{ // if the close price is garbage set it to the midpoint of the high and the low
|
|
if(price.Close<price.Low||price.Close>price.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
|
|
}
|
|
price.AdjClose=price.Close;
|
|
if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No open,high,low,close price for {0}",price.Symbol));
|
|
return null;
|
|
}
|
|
if(double.IsNaN(price.Open))price.Open=0.00;
|
|
if(double.IsNaN(price.Low))price.Low=0.00;
|
|
if(double.IsNaN(price.High))price.High=0.00;
|
|
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
|
|
{
|
|
price.Open=price.High=price.Low=price.Close;
|
|
}
|
|
return price;
|
|
}
|
|
// ****************************************************************************************************************************************************************************************
|
|
// **************************************************************************** E N D D A T A F E E D S *********************************************************************************
|
|
// ****************************************************************************************************************************************************************************************
|
|
// ****************************************************************************************************************************************************************************************
|
|
// **************************************************************************** H E L P E R M E T H O D S *******************************************************************************
|
|
// ****************************************************************************************************************************************************************************************
|
|
private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName)
|
|
{
|
|
StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt");
|
|
for (int row = 0; row < rows.Count; row++)
|
|
{
|
|
streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml);
|
|
}
|
|
streamWriter.Flush();
|
|
streamWriter.Close();
|
|
}
|
|
private static void WriteToDisk(String responseString, String strPathFileName)
|
|
{
|
|
StreamWriter streamWriter = File.CreateText(strPathFileName);
|
|
streamWriter.WriteLine(responseString);
|
|
streamWriter.Flush();
|
|
streamWriter.Close();
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
|
|
// 2017 Yahoo Finance Modified Format of all feeds. Most of the HTMLDocument parser code is broken because the response strings are no longer in tabular format.
|
|
// It's difficult to make sense of the new response string outside of noting that it remains in aa tag format. The following code is an attempt to make sense of it all.
|
|
|
|
public static List<String> LocateJSONItems(String strInput,String item,bool isHeading=false)
|
|
{
|
|
try
|
|
{
|
|
if(null==item||null==strInput)return null;
|
|
int startPos=strInput.IndexOf(item);
|
|
if(-1==startPos)return null;
|
|
strInput=strInput.Substring(startPos-1);
|
|
if(isHeading)
|
|
{
|
|
strInput=strInput.Substring(item.Length);
|
|
startPos=strInput.IndexOf(item);
|
|
if(-1==startPos)return null;
|
|
strInput=strInput.Substring(startPos-1);
|
|
}
|
|
strInput=Utility.BetweenString(strInput,"[","]");
|
|
if(null==strInput)return null;
|
|
String[] itemsArray=strInput.Split(',');
|
|
return itemsArray.ToList();
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<KeyValue> LocateJSONKeyValuePairs(String strInput, String item1,String item2)
|
|
{
|
|
try
|
|
{
|
|
List<KeyValue> items=new List<KeyValue>();
|
|
if (null == item1 || null == strInput) return null;
|
|
int startPos = strInput.IndexOf(item1);
|
|
while(true)
|
|
{
|
|
KeyValue keyValue=new KeyValue();
|
|
if (-1 == startPos||null==strInput) break;
|
|
String strItem = strInput.Substring(startPos);
|
|
strItem = Utility.BetweenString(strItem, "\"", "\"");
|
|
String strValue = strInput.Substring((startPos+strItem.Length+4)-1);
|
|
strValue = Utility.BetweenString(strValue, "\"", "\"");
|
|
keyValue.Key=strValue;
|
|
try{strInput=strInput.Substring((startPos+strItem.Length+4+strValue.Length+4)-1);}
|
|
catch(Exception){break;}
|
|
if(null==strInput)break;
|
|
startPos = strInput.IndexOf(item2);
|
|
if(-1==startPos)break;
|
|
strItem = strInput.Substring(startPos);
|
|
strItem = Utility.BetweenString(strItem, "\"", "\"");
|
|
strValue = strInput.Substring((startPos + strItem.Length + 4) - 1);
|
|
strValue = Utility.BetweenString(strValue, "\"", "\"");
|
|
keyValue.Value=strValue;
|
|
try { strInput = strInput.Substring((startPos + strValue.Length + 4) - 1); }
|
|
catch (Exception) { break; }
|
|
if(null==strInput)break;
|
|
startPos = strInput.IndexOf(item1);
|
|
items.Add(keyValue);
|
|
}
|
|
return items;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
public static List<List<String>> LocateJSONItems(String strInput)
|
|
{
|
|
List<List<String>> items=new List<List<String>>();
|
|
try
|
|
{
|
|
if(null==strInput)return null;
|
|
strInput=Utility.BetweenString(strInput,"[","]");
|
|
if(null==strInput)return null;
|
|
String[] itemsArray=strInput.Split(',');
|
|
if(null==itemsArray||0==itemsArray.Length)return null;
|
|
for(int index=0;index<itemsArray.Length;index++)
|
|
{
|
|
String item=itemsArray[index];
|
|
item=item.Replace("{",null);
|
|
item=item.Replace("}",null);
|
|
item=item.Replace("\"",null);
|
|
String[] subItemArray=item.Split(':');
|
|
if(null==subItemArray||2!=subItemArray.Length)continue;
|
|
if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
|
|
if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
|
|
items.Add(subItemArray.ToList());
|
|
}
|
|
return items;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
|
|
{
|
|
DateTime referenceDate=DateTime.Now;
|
|
string[] items=relativeDate.Split(' ');
|
|
if(items.Length<2)return Utility.Epoch;
|
|
if("days".Equals(items[1])||"day".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0,0,0);
|
|
referenceDate-=timeSpan;
|
|
}
|
|
else if("hours".Equals(items[1])||"hour".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
|
|
referenceDate-=timeSpan;
|
|
}
|
|
else if("mins".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
|
|
referenceDate-=timeSpan;
|
|
}
|
|
else if( (new string[]{"Jul","Aug","Sep","Oct","Nov","Dec","Jan","Feb","Mar","Apr","May","Jun"}).Any(x=>x.Equals(items[0])))
|
|
{
|
|
DateTime now=DateTime.Now;
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year);
|
|
referenceDate=Utility.ParseDate(sb.ToString());
|
|
if(referenceDate>now)
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1],'0',2)).Append(" ").Append(referenceDate.Year-1);
|
|
referenceDate=Utility.ParseDate(sb.ToString());
|
|
}
|
|
}
|
|
else referenceDate=Utility.Epoch;
|
|
if(referenceDate>DateTime.Now)referenceDate=Utility.Epoch;
|
|
return referenceDate;
|
|
}
|
|
}
|
|
}
|