5725 lines
282 KiB
C#
Executable File
5725 lines
282 KiB
C#
Executable File
using System;
|
|
using System.IO;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
using System.Linq;
|
|
using System.Data;
|
|
using System.Threading;
|
|
using System.Net;
|
|
using System.Reflection;
|
|
using System.Xml.Linq;
|
|
using Newtonsoft.Json.Linq;
|
|
using HtmlAgilityPack;
|
|
using MarketData.MarketDataModel;
|
|
using MarketData.Utils;
|
|
using MarketData.CSVHelper;
|
|
using MarketData.DataAccess;
|
|
using MarketData.Integration;
|
|
using System.Globalization;
|
|
using MarketDataLib.Utility;
|
|
using MarketData.Configuration;
|
|
using System.Text.Json;
|
|
|
|
// Zacks Rank - Zacks
|
|
// Splits - EODDATA
|
|
// GDP Data - WorldBank
|
|
// Employment Data - WorldBank
|
|
// Dividend History - NASDAQ
|
|
// Options - NASDAQ
|
|
// Analyst Ratings - Briefing.com
|
|
// SEC Filings - SEC.GOV (EDGAR)
|
|
// Yield Curve - Treasury.Gov
|
|
// ETF Holdings - Yahoo Finance
|
|
// Insider Transactions - Insider Tracking.com
|
|
// Company Profile - MorningStar + Reuters
|
|
// Headlines - Seeking Alpha + NASDAQ
|
|
// CIK Codes - SEC.GOV
|
|
// Analyst Price Target - Yahoo Finance , Market Beat as backup
|
|
// Historical - MorningStar
|
|
// Income Statement - NASDAQ
|
|
// Balance Sheet - NASDAQ
|
|
// Statement of Cashflows - MorningStar
|
|
// Fundamentals - 2 Sources....Yahoo Finance & FINVIZ
|
|
// a) Key Statistics
|
|
// b) Financials
|
|
// Intraday Pricing - Yahoo Finance + RobinHood
|
|
// Historical Pricing - Yahoo Finance
|
|
// CompanyDescription - Reuters
|
|
// Premarket - CNN
|
|
|
|
namespace MarketData.Helper
|
|
{
|
|
public class MarketDataHelper
|
|
{
|
|
public static int DEFAULT_TIMEOUT_MS=30000;
|
|
public static String SEC_BASE_URL="https://www.sec.gov";
|
|
public static String SAL_VERSION="4.71.0"; // This is for morningstar and may have to be changed periodically until I find a way to get automatically
|
|
|
|
private MarketDataHelper()
|
|
{
|
|
}
|
|
|
|
public static bool GetWorldTime()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.Append("https://www.timeanddate.com/worldclock/");
|
|
String strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime:{0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4A(strRequest, null, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return false;
|
|
}
|
|
|
|
if(String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No data returned for request {0}",strRequest));
|
|
return false;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetWorldTime Received {0} bytes of data.",httpNetResponse.ResponseString.Length));
|
|
return true;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetWorldTime: Done, took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************************ P R E M A R K E T D A T A ***********************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static PremarketElements GetPremarketData()
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
PremarketElements premarketElements=new PremarketElements();
|
|
DateTime timestamp=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest=null;
|
|
|
|
sb.Append("https://production.dataviz.cnn.io/markets/futures/summary");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request {0} ",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. Failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
if(null==items || 0==items.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData Request '{0}'. No Data",strRequest));
|
|
return premarketElements;
|
|
}
|
|
for(int index=0;index<items.Count;index++)
|
|
{
|
|
List<String> pair=items[index];
|
|
if(pair[1].Equals("S&P 500"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="S&P";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
else if(pair[1].Equals("NASDAQ"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="NASDAQ";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
else if(pair[1].Equals("DOW"))
|
|
{
|
|
PremarketElement premarketElement=new PremarketElement();
|
|
premarketElement.Market="DOW";
|
|
premarketElement.ChangeValue=FeedParser.ParseValue((items[index+2])[1]);
|
|
premarketElement.ChangePercent=FeedParser.ParseValue((items[index+3])[1])*100.00;
|
|
premarketElement.Timestamp=timestamp;
|
|
premarketElements.Add(premarketElement);
|
|
}
|
|
}
|
|
return premarketElements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetPremarketData: Done, took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S *******************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
/// <summary>
|
|
/// This retrieves earnings announcements from Zacks Research and is used during the nightly process.
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <returns></returns>
|
|
public static EarningsAnnouncements GetEarningsAnnouncements(String symbol)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
HttpNetResponse httpNetResponse = null;
|
|
EarningsAnnouncements earningsAnnouncements = new EarningsAnnouncements();
|
|
MemoryStream memoryStream = null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.Append("https://www.zacks.com/stock/research/").Append(symbol).Append("/earnings-announcements");
|
|
String strRequest = sb.ToString();
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
String responseString = httpNetResponse.ResponseString;
|
|
// get next earnings date
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"key-expected-earnings-data-module\"]");
|
|
if (null != table && 0 != table.Count)
|
|
{
|
|
HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
|
|
String[] requiredElements = { "Report Date", "Period Ending", "Zacks Consensus Estimate", "Earnings ESP" };
|
|
if (2 == rows.Count)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[0].SelectNodes(".//th");
|
|
if (null != headerColumns && 0 != headerColumns.Count && HeaderContains(headerColumns, requiredElements))
|
|
{
|
|
HtmlNodeCollection dataColumns = rows[1].ChildNodes;
|
|
String[] reportElements = dataColumns[1].InnerText.Split(' ');
|
|
String reportDate = reportElements[0];
|
|
if (!"NA".Equals(reportDate))
|
|
{
|
|
String[] formats = new[] { "MM/yyyy", "M/yyyy" };
|
|
String periodEnding = dataColumns[3].InnerText;
|
|
EarningsAnnouncement earningsAnnouncement = new EarningsAnnouncement();
|
|
earningsAnnouncement.Symbol = symbol;
|
|
earningsAnnouncement.Date = Utility.ParseDate(reportDate);
|
|
earningsAnnouncement.PeriodEnding = DateTime.ParseExact(periodEnding, formats, new System.Globalization.CultureInfo("en-US"), DateTimeStyles.AssumeLocal);
|
|
if (reportElements[1].Equals("(AMC)")) earningsAnnouncement.Time = "After Close";
|
|
else if (reportElements[1].Equals("(NONE)")) earningsAnnouncement.Time = "Before Open";
|
|
else earningsAnnouncement.Time = reportElements[1];
|
|
earningsAnnouncements.Add(earningsAnnouncement);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// get historical
|
|
responseString = Utility.KeepAfter(responseString, "earnings_announcements_earnings_table");
|
|
if (null == responseString) return null;
|
|
responseString = Utility.KeepBefore(responseString, "] ]");
|
|
if (null == responseString) return null;
|
|
responseString = responseString + "]";
|
|
responseString = Utility.KeepAfter(responseString, "[ [");
|
|
if (null == responseString) return null;
|
|
responseString = "[" + responseString;
|
|
responseString = Utility.RemoveDivs(responseString);
|
|
if (null == responseString) return null;
|
|
String[] items = responseString.Split(']');
|
|
for (int index = 0; index < items.Length; index++)
|
|
{
|
|
EarningsAnnouncement earningsAnnouncement = new EarningsAnnouncement();
|
|
String itemText = items[index];
|
|
if (null == itemText || 0 == itemText.Length) continue;
|
|
itemText = Utility.BetweenString(itemText, "[", null);
|
|
itemText = itemText.Replace("\"", null);
|
|
String[] columnData = itemText.Split(',');
|
|
for (int columnIndex = 0; columnIndex < columnData.Length; columnIndex++) columnData[columnIndex] = columnData[columnIndex].Trim();
|
|
if (null == columnData || 7 != columnData.Length) continue;
|
|
earningsAnnouncement.Symbol = symbol;
|
|
earningsAnnouncement.Date = Utility.ParseDate(columnData[0]);
|
|
|
|
String[] periodEndingItems = columnData[1].Split('/');
|
|
if (null != periodEndingItems && 2 == periodEndingItems.Length)
|
|
{
|
|
int month = (int)FeedParser.ParseValueLong(periodEndingItems[0]);
|
|
int year = (int)FeedParser.ParseValueLong(periodEndingItems[1]);
|
|
earningsAnnouncement.PeriodEnding = new DateTime(year, month, DateTime.DaysInMonth(year, month));
|
|
}
|
|
earningsAnnouncement.Estimate = FeedParser.ParseValue(columnData[2]);
|
|
earningsAnnouncement.Reported = FeedParser.ParseValue(columnData[3]);
|
|
earningsAnnouncement.Surprise = FeedParser.ParseValue(columnData[4]);
|
|
earningsAnnouncement.SurprisePct = FeedParser.ParseValue(columnData[5]);
|
|
earningsAnnouncement.Time = columnData[6];
|
|
earningsAnnouncements.Add(earningsAnnouncement);
|
|
}
|
|
return earningsAnnouncements;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
if (null != memoryStream) memoryStream.Dispose();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("MarketData.GetEarningsAnnouncements: Done, took {0}(ms)", profiler.End()));
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// ************************************************************************** Z A C K S R A N K ******************************************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static ZacksRank GetZacksRank(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.zacks.com/stock/quote/");
|
|
sb.Append(symbol);
|
|
String strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
if(httpNetResponse.StatusCode.Equals("Forbidden"))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
|
|
try{Thread.Sleep(15000);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
else return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"rank_view\"]");
|
|
if(null==table||0==table.Count)return null;
|
|
String rank=Utility.BetweenString(table[0].InnerHtml,null,"<").Trim();
|
|
if(null==rank||"".Equals(rank))return null;
|
|
ZacksRank zacksRank=new ZacksRank();
|
|
zacksRank.Symbol=symbol;
|
|
zacksRank.Date=DateTime.Now.Date;
|
|
zacksRank.Rank=rank;
|
|
return zacksRank;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** S P L I T S F R O M E O D D A T A **********************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Splits GetSplits()
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US");
|
|
String format="dd MMM yyyy";
|
|
|
|
try
|
|
{
|
|
Splits splits = new Splits();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("http://eoddata.com/splits.aspx");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Getting splits {0}", strRequest));
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
using MemoryStream memoryStream = new MemoryStream(streamBytes);
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"table-standard\"]");
|
|
if (null == divSection || 0 == divSection.Count) return null;
|
|
HtmlNodeCollection rows = divSection[0].SelectNodes(".//tr");
|
|
for (int index = 0; index < rows.Count; index++)
|
|
{
|
|
Split split = new Split();
|
|
HtmlNodeCollection headerColumns = rows[index].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[index].SelectNodes(".//td");
|
|
if (dataColumns.Count < 4) continue;
|
|
if (null != dataColumns[0]) split.Exchange = dataColumns[0].InnerText.Trim().ToUpper(); // exchange
|
|
if (null != dataColumns[1]) split.Symbol = dataColumns[1].InnerText.Trim().ToUpper(); // symbol
|
|
if (null != dataColumns[2]) split.EffectiveDate = DateTime.ParseExact(dataColumns[2].InnerText.Trim().ToUpper(),
|
|
format, cultureInfo, DateTimeStyles.AssumeLocal);
|
|
if (null != dataColumns[3]) split.StrRatio = dataColumns[3].InnerText.Trim().ToUpper(); // ratio
|
|
splits.Add(split);
|
|
}
|
|
return splits;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** W O R L D B A N K F E E D *************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static EconomicIndicators GetUnemploymentPerCapita(bool debug = false)
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
String currentWorkingDirectory = Directory.GetCurrentDirectory();
|
|
String strExtractFolder = currentWorkingDirectory + "/" + "extracts";
|
|
String strFileName = "API_SL.UEM.TOTL.ZS_DS2_en_csv_v2_254884.zip";
|
|
String strPathFileName = currentWorkingDirectory + "/" + strFileName;
|
|
sb.Append("http://api.worldbank.org/v2/en/indicator/SL.UEM.TOTL.ZS?downloadformat=csv");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Downloading {0} from {1}", strFileName, strRequest));
|
|
httpNetResponse = HttpNetRequest.GetRequestStreamZIP(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
if (File.Exists(strPathFileName)) File.Delete(strPathFileName);
|
|
FileStream outStream = new FileStream(strPathFileName, FileMode.Create);
|
|
byte[] streamBytes = httpNetResponse.ResponseStream.GetBuffer();
|
|
outStream.Write(streamBytes, 0, streamBytes.Length);
|
|
outStream.Flush();
|
|
outStream.Close();
|
|
outStream.Dispose();
|
|
return EconomicIndicators.FromZipFile(strPathFileName, strExtractFolder, debug);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves GDP Data from WorlBank as zip file, decompresses the zip file, reads and imports the CSV contents and returns EconomicIndicators
|
|
/// </summary>
|
|
/// <param name="debug"></param>
|
|
/// <returns></returns>
|
|
public static EconomicIndicators GetGDPPerCapita(bool debug = false)
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
String currentWorkingDirectory = Directory.GetCurrentDirectory();
|
|
String strExtractFolder = currentWorkingDirectory + "/" + "extracts";
|
|
String strFileName = "API_NY.GDP.MKTP.CD_DS2_V2_USD.zip";
|
|
String strPathFileName = currentWorkingDirectory + "/" + strFileName;
|
|
sb.Append("http://api.worldbank.org/v2/en/indicator/NY.GDP.MKTP.CD?downloadformat=csv");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Downloading {0} from {1}", strFileName, strRequest));
|
|
httpNetResponse = HttpNetRequest.GetRequestStreamZIP(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
if (File.Exists(strPathFileName)) File.Delete(strPathFileName);
|
|
FileStream outStream = new FileStream(strPathFileName, FileMode.Create);
|
|
byte[] streamBytes = httpNetResponse.ResponseStream.GetBuffer();
|
|
outStream.Write(streamBytes, 0, streamBytes.Length);
|
|
outStream.Flush();
|
|
outStream.Close();
|
|
outStream.Dispose();
|
|
return EconomicIndicators.FromZipFile(strPathFileName, strExtractFolder, debug);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** D I V I D E N D H I S T O R Y N A S D A Q ***********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static DividendHistory GetDividendHistory(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
DividendHistory dividendHistory = new DividendHistory();
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
CookieCollection cookieCollection = new CookieCollection();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetDividendHistory");
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf");
|
|
else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest));
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
if(httpNetResponse.StatusCode.Equals(System.Net.HttpStatusCode.Forbidden))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Retrying request:{0}, after 15000(ms)",httpNetResponse.Request));
|
|
try{Thread.Sleep(15000);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestStreamCSV(strRequest, cookieCollection, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
else return null;
|
|
}
|
|
|
|
int groupBy=6;
|
|
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
if(null==items || 0==items.Count || items.Count<groupBy)return dividendHistory;
|
|
for(int index=0;index<items.Count;index+=groupBy)
|
|
{
|
|
if(items.Count<index+groupBy)break;
|
|
String strExEffDate = items[index][1];
|
|
String strType = items[index+1][1];
|
|
String strCashAmount = items[index+2][1];
|
|
String strDeclarationDate = items[index+3][1];
|
|
String strRecordDate = items[index+4][1];
|
|
String strPaymentDate = items[index+5][1];
|
|
|
|
try
|
|
{
|
|
if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
|
|
if("EX-DIVIDEND DATE".Equals(strExEffDate, StringComparison.InvariantCultureIgnoreCase))
|
|
{
|
|
strExEffDate = items[index+1][1];
|
|
strType = items[index+4][1];
|
|
strCashAmount = items[index+5][1];
|
|
strDeclarationDate = items[index+1][1];
|
|
strRecordDate = items[index+1][1];
|
|
strPaymentDate = items[index+1][1];
|
|
}
|
|
if("--".Equals(strExEffDate)||"N/A".Equals(strExEffDate)) continue;
|
|
DividendHistoryItem dividendHistoryItem = new DividendHistoryItem();
|
|
dividendHistoryItem.Modified=DateTime.Now;
|
|
dividendHistoryItem.Symbol = symbol.ToUpper();
|
|
try{dividendHistoryItem.DivExDate = Utility.ParseDate(strExEffDate);}
|
|
catch(Exception exception){MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception parsing value '{0}', {1}",strExEffDate,exception.ToString()));continue;}
|
|
dividendHistoryItem.DividendType = strType;
|
|
if (!"--".Equals(strCashAmount)&&!"".Equals(strCashAmount)) dividendHistoryItem.CashAmount = FeedParser.ParseValue(strCashAmount);
|
|
else dividendHistoryItem.CashAmount = null;
|
|
if (!"--".Equals(strDeclarationDate)&&!"N/A".Equals(strDeclarationDate)) dividendHistoryItem.DeclarationDate = Utility.ParseDate(strDeclarationDate);
|
|
else dividendHistoryItem.DeclarationDate=null;
|
|
if (!"--".Equals(strRecordDate)&&!"N/A".Equals(strRecordDate)) dividendHistoryItem.RecordDate = Utility.ParseDate(strRecordDate);
|
|
if (!"--".Equals(strPaymentDate)&&!"N/A".Equals(strPaymentDate)) dividendHistoryItem.PaymentDate = Utility.ParseDate(strPaymentDate);
|
|
else dividendHistoryItem.PaymentDate = null;
|
|
|
|
if(null==dividendHistoryItem.CashAmount || double.IsNaN(dividendHistoryItem.CashAmount.Value))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Encountered invalid CashAmount in DividendHistory record for {0} ",symbol));
|
|
continue;
|
|
}
|
|
if(dividendHistoryItem.DividendType.Equals("ANNUAL DIVIDEND"))dividendHistoryItem.DividendType="Cash";
|
|
dividendHistory.Add(dividendHistoryItem);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetDividendHistory: Exception:{0}",exception.ToString()));
|
|
continue;
|
|
}
|
|
}
|
|
return dividendHistory;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return dividendHistory;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** A N A L Y S T R A T I N G S - B R I E F I N G . C O M *************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Upgrades/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/downgrades/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Reiterated/2014/08/04
|
|
//http://www.briefing.com/Investor/Calendars/Upgrades-Downgrades/Initiated/2014/08/04
|
|
// The feed no longer accepts a date parameter
|
|
public static AnalystRatings GetLatestAnalystRatings()
|
|
{
|
|
AnalystRatings analystRatings = GetLatestAnalystRatingsBriefing();
|
|
return analystRatings;
|
|
}
|
|
|
|
public static AnalystRatings GetLatestAnalystRatingsBriefing()
|
|
{
|
|
Dictionary<String, String> uniqueSymbols = new Dictionary<String, String>();
|
|
AnalystRatings analystRatings = new AnalystRatings();
|
|
AnalystRatings ratings = null;
|
|
|
|
ratings = GetLatestAnalystRatingsBriefing("Upgrades");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Downgrades");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Reiterated");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
ratings = GetLatestAnalystRatingsBriefing("Initiated");
|
|
if (null != ratings)
|
|
{
|
|
foreach (AnalystRating analystRating in ratings)
|
|
{
|
|
if (!uniqueSymbols.ContainsKey(analystRating.Symbol))
|
|
{
|
|
analystRatings.Add(analystRating);
|
|
uniqueSymbols.Add(analystRating.Symbol, analystRating.Symbol);
|
|
}
|
|
}
|
|
}
|
|
return analystRatings;
|
|
}
|
|
|
|
private static AnalystRatings GetLatestAnalystRatingsBriefing(String type)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse = null;
|
|
AnalystRatings analystRatings = new AnalystRatings();
|
|
try
|
|
{
|
|
String strRequest = null;
|
|
if ("Upgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/upgrade.htm";
|
|
else if ("Downgrades".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/downgrade.htm";
|
|
else if ("Reiterated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/reiterated.htm";
|
|
else if ("Initiated".Equals(type)) strRequest = "https://www.briefing.com/Inv/content/Calendar/Updown/initiated.htm";
|
|
else return null;
|
|
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if (!httpNetResponse.Success || null == httpNetResponse.ResponseString)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSectionDate = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calDATE\"]");
|
|
HtmlNodeCollection divSectionHeader = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calHDR2\"]");
|
|
HtmlNodeCollection divSections = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"calEVENT\"]");
|
|
|
|
if (null == divSectionDate || 0 == divSectionDate.Count) return null;
|
|
if (null == divSectionHeader || 0 == divSectionHeader.Count) return null;
|
|
if (null == divSections || 0 == divSections.Count) return null;
|
|
|
|
DateTime ratingsDate = Utility.ParseDate(divSectionDate[0].InnerText);
|
|
String ratingsType = divSectionHeader[0].InnerText;
|
|
if (!ratingsType.StartsWith(type.ToUpper())) return null;
|
|
|
|
// calDATA
|
|
for (int index = 0; index < divSections.Count; index++)
|
|
{
|
|
|
|
List<String> items = Sections.GetSections(divSections[index].InnerHtml);
|
|
if (null == items || 0 == items.Count) continue;
|
|
|
|
AnalystRating analystRating = new AnalystRating();
|
|
analystRating.CompanyName = items.Count > 0 ? items[0] : null;
|
|
analystRating.Date = ratingsDate;
|
|
analystRating.Symbol = items.Count > 4 ? items[4] : null;
|
|
analystRating.Type = type;
|
|
if (items.Count > 10 && items[10].StartsWith("Brokerage Firm")) analystRating.BrokerageFirm = items.Count > 12 ? items[12] : null;
|
|
if (items.Count > 22 && items[22].StartsWith("Ratings Change")) analystRating.RatingsChange = items.Count > 24 ? items[24] : null;
|
|
if (null != analystRating.RatingsChange & analystRating.RatingsChange.Contains("»")) analystRating.RatingsChange = analystRating.RatingsChange.Replace("»", "->");
|
|
if (items.Count > 28 && items[28].StartsWith("Price Tgt"))
|
|
{
|
|
String priceTarget = items.Count > 31 ? items[31] : null;
|
|
if (null != priceTarget && priceTarget.Contains("»")) priceTarget = priceTarget.Replace("»", "-");
|
|
if (priceTarget.Contains("-"))
|
|
{
|
|
String[] prices = priceTarget.Split('-');
|
|
priceTarget = prices[1];
|
|
}
|
|
analystRating.PriceTarget = FeedParser.ParseValue(priceTarget);
|
|
}
|
|
analystRatings.Add(analystRating);
|
|
}
|
|
return analystRatings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return analystRatings;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************************
|
|
// ****************************************************************************** A N A L Y S T R A T I N G S M A R K E T B E A T *******************************************************
|
|
// ******************************************************************************************************************************************************************************************
|
|
public static AnalystRatings GetAnalystRatingsMarketBeat(String symbol)
|
|
{
|
|
AnalystRatings analystRatings=new AnalystRatings();
|
|
String nasdaq="NASDAQ";
|
|
String nyse="NYSE";
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
HtmlNodeCollection ratingsTable=null;
|
|
ratingsTable=GetRatingsTableOnMarketBeat(symbol,nyse);
|
|
if(null==ratingsTable) ratingsTable=GetRatingsTableOnMarketBeat(symbol,nasdaq);
|
|
if(null==ratingsTable)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No ratings for {0} on MarketBeat",symbol));
|
|
return null;
|
|
}
|
|
HtmlNodeCollection header=ratingsTable[0].SelectNodes(".//th");
|
|
string[] requiredElements=new string[]{"Date","Brokerage","Action","Rating","Price Target","Details"};
|
|
if(!HeaderContains(header,requiredElements))return null;
|
|
HtmlNodeCollection data=ratingsTable[0].SelectNodes(".//tr//td");
|
|
if(null==data)return analystRatings;
|
|
for(int index=0,rowIndex=0;index<data.Count&&index+(header.Count-1)<data.Count;index+=header.Count,rowIndex++)
|
|
{
|
|
String strDate=GetElementData(header,data,rowIndex,"Date");
|
|
if(null==strDate || "".Equals(strDate))break;
|
|
String strBrokerage=GetElementData(header,data,rowIndex,"Brokerage");
|
|
if(null!=strBrokerage)strBrokerage=strBrokerage.Trim();
|
|
String strAction=GetElementData(header,data,rowIndex,"Action");
|
|
String strRating=GetElementData(header,data,rowIndex,"Rating");
|
|
String strPriceTarget=GetElementData(header,data,rowIndex,"Price Target");
|
|
AnalystRating analystRating=new AnalystRating();
|
|
analystRating.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;
|
|
analystRating.Date=Utility.ParseDate(strDate);
|
|
analystRating.Symbol=symbol;
|
|
analystRating.Type=strAction;
|
|
analystRating.BrokerageFirm=strBrokerage;
|
|
if(null!=strRating&strRating.Contains("➝")) strRating=strRating.Replace("➝","->");
|
|
analystRating.RatingsChange=strRating;
|
|
if(null!=strPriceTarget)
|
|
{
|
|
|
|
if(strPriceTarget.Contains("➝")) strPriceTarget=strPriceTarget.Replace("➝","-");
|
|
if(strPriceTarget.Contains("-"))
|
|
{
|
|
String[] prices=strPriceTarget.Split('-');
|
|
strPriceTarget=prices[1];
|
|
}
|
|
analystRating.PriceTarget=FeedParser.ParseValue(strPriceTarget);
|
|
}
|
|
if(analystRating.Type.Equals("Boost Price Target")) analystRating.Type="Upgrades";
|
|
if(analystRating.Type.Equals("Upgrade")) analystRating.Type="Upgrades";
|
|
if(analystRating.Type.Equals("Downgrade")) analystRating.Type="Downgrades";
|
|
if(analystRating.Type.StartsWith("Reiterated")) analystRating.Type="Reiterated";
|
|
if(analystRating.Type.StartsWith("Initiated")) analystRating.Type="Initiated";
|
|
if(analystRating.Type.StartsWith("Set Price Target"))analystRating.Type="Initiated";
|
|
if(!analystRating.Type.Equals("Upgrades")&&!analystRating.Type.Equals("Downgrades")&&!analystRating.Type.Equals("Reiterated")&&!analystRating.Type.Equals("Initiated")) continue;
|
|
analystRatings.Add(analystRating);
|
|
}
|
|
return new AnalystRatings(analystRatings.Distinct().ToList());
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return analystRatings;
|
|
}
|
|
finally
|
|
{
|
|
}
|
|
}
|
|
|
|
private static bool HeaderContains(HtmlNodeCollection header,String[] requiredElements)
|
|
{
|
|
List<String> headerElements=new List<String>();
|
|
if(null==header)return false;
|
|
for(int index=0;index<header.Count;index++)
|
|
{
|
|
headerElements.Add(header[index].InnerText);
|
|
}
|
|
List<String> exceptions=requiredElements.Except(headerElements).ToList();
|
|
return exceptions.Count()>0?false:true;
|
|
}
|
|
|
|
private static int HeaderIndexOf(HtmlNodeCollection header,String identifier)
|
|
{
|
|
if(null==header)return -1;
|
|
for(int index=0;index<header.Count;index++)
|
|
{
|
|
if(header[index].InnerText.Equals(identifier))return index;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
private static String GetElementData(HtmlNodeCollection header,HtmlNodeCollection data,int row,String identifier)
|
|
{
|
|
int index=HeaderIndexOf(header,identifier);
|
|
if(-1==index)return null;
|
|
int indexer=(header.Count*row)+index;
|
|
List<String> sections=Sections.GetSections(data[indexer].InnerHtml);
|
|
if(null==sections || 0==sections.Count)return data[indexer].InnerText;
|
|
return sections[0];
|
|
}
|
|
|
|
/// <summary>
|
|
/// This method looks for the ratings rating in the html from the MarketBeat query
|
|
/// I made a change to the url on 11/4/2025 changed /price-target to /forecast
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <param name="exchange"></param>
|
|
/// <returns></returns>
|
|
private static HtmlNodeCollection GetRatingsTableOnMarketBeat(String symbol,String exchange)
|
|
{
|
|
MemoryStream memoryStream=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
String strRequest=null;
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/forecast");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3(strRequest, "www.marketbeat.com");
|
|
if(!httpNetResponse.Success||String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream=new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
|
|
HtmlNodeCollection ratingsTable=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"scroll-table sort-table\"]");
|
|
if(null==ratingsTable||0==ratingsTable.Count) return null;
|
|
return ratingsTable;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** C I K C O D E S - S E C . G O V ***********************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static String GetCIK(String symbol)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?CIK=").Append(symbol.ToUpper()).Append("&Find=Search&owner=exclude&action=getcompany");
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCIK");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"companyInfo\"]"); // yfnc_tablehead1,yfnc_modtitle1,yfnc_tabledata1
|
|
if (null == rows || 1!=rows.Count)
|
|
{
|
|
if (null == rows) MDTrace.WriteLine(LogLevel.DEBUG,"[GetCIK] Received no rows for '" + symbol + "'");
|
|
else MDTrace.WriteLine(LogLevel.DEBUG,"Expected 1 row, got " + rows.Count + " for '" + symbol + "'");
|
|
return null;
|
|
}
|
|
HtmlNode htmlNodeTitle = rows[0];
|
|
HtmlNodeCollection nodes = htmlNodeTitle.SelectNodes(".//a");
|
|
if (null == nodes || 0==nodes.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Expected <a> tag for symbol '" + symbol + "'");
|
|
return null;
|
|
}
|
|
String cik = null;
|
|
String[] strings = nodes[0].InnerText.Split(' ');
|
|
if (strings.Length >= 1) cik = strings[0];
|
|
return cik;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************ S E C F I L I N G S - F O R M 4 & F O R M 5 - F O R I N S I D E R T R A N S A C T I O N S S E C . G O V ***************************S
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static InsiderTransactions GetInsiderTransactions(String symbol,int timePeriodDays=5)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
|
|
DateTime minFilingDate=DateTime.Now;
|
|
int maxFilings=80;
|
|
bool continuationFlag=true;
|
|
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} dayCount:{1}",symbol,timePeriodDays));
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
String cik=PricingDA.GetCIKForSymbol(symbol);
|
|
if(null==cik)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: No CIK for symbol {0}",symbol));
|
|
return null;
|
|
}
|
|
minFilingDate=dateGenerator.DaysAddActual(minFilingDate,Math.Abs(timePeriodDays)*-1);
|
|
SECFilings secFilings = new SECFilings();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactions: symbol:{0} .Fetching {1} SECFilingDocuments ",symbol,rows.Count));
|
|
for (int row = 0; row < rows.Count && continuationFlag; row ++)
|
|
{
|
|
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
|
|
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
|
|
{
|
|
SECFiling innerFiling = innerCollection[index];
|
|
if(innerFiling.FilingDate<minFilingDate)
|
|
{
|
|
continuationFlag=false;
|
|
break;
|
|
}
|
|
if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
|
|
secFilings.Add(innerFiling);
|
|
}
|
|
}
|
|
return InsiderTransactionsParser.GetInstance().Parse(secFilings);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get InsiderTransactions for years greater than or equal to specified year.
|
|
/// For example. If 2023 is specified then get all for years>=2023
|
|
/// </summary>
|
|
/// <param name="symbol">The symbol.</param>
|
|
/// <param name="yearGreaterThanEqualTo">the new y-coordinate.</param>
|
|
public static InsiderTransactions GetInsiderTransactionsYear(String symbol,int yearGreaterThanEqualTo)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
String[] descriptionStartsWith=new String[]{"Form 4","Form 5"};
|
|
int maxFilings=120;
|
|
bool continuationFlag=true;
|
|
int TIMEOUT_MS_BETWEEN_REQUESTS=1000;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}",symbol,yearGreaterThanEqualTo));
|
|
DateGenerator dateGenerator=new DateGenerator();
|
|
String cik=PricingDA.GetCIKForSymbol(symbol);
|
|
if(null==cik)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: No CIK for symbol {0}",symbol));
|
|
return null;
|
|
}
|
|
SECFilings secFilings = new SECFilings();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetInsiderTransactions");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetInsiderTransactionsYear: symbol:{0} year:{1}. Fetching {2} SECFilingDocuments ",symbol,yearGreaterThanEqualTo,rows.Count));
|
|
for (int row = 0; row < rows.Count && continuationFlag; row ++)
|
|
{
|
|
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_REQUESTS);}catch(Exception){;}
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
|
|
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
|
|
{
|
|
SECFiling innerFiling = innerCollection[index];
|
|
if(innerFiling.FilingDate.Year<yearGreaterThanEqualTo)
|
|
{
|
|
continuationFlag=false;
|
|
break;
|
|
}
|
|
if(!descriptionStartsWith.Any(x=>innerFiling.Description.StartsWith(x)))continue;
|
|
secFilings.Add(innerFiling);
|
|
}
|
|
}
|
|
return InsiderTransactionsParser.GetInstance().Parse(secFilings);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** S E C F I L I N G S - S E C . G O V ********************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static SECFilings GetSECFilings(String symbol,int maxFilings=80)
|
|
{
|
|
try
|
|
{
|
|
String cik=PricingDA.GetCIKForSymbol(symbol);
|
|
if(null==cik)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetSECFilings: No CIK for symbol {0}",symbol));
|
|
return null;
|
|
}
|
|
return GetSECFilings(symbol,cik,maxFilings);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null; }
|
|
}
|
|
|
|
public static SECFilings GetSECFilings(String symbol,String cik,int maxFilings=80)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
|
|
SECFilings secFilings = new SECFilings();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append(SEC_BASE_URL).Append("/cgi-bin/browse-edgar?action=getcompany&CIK=").Append(cik).Append("&type=&dateb=&owner=include&count="+maxFilings.ToString());
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilings");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilings] Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile2\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
for (int row = 0; row < rows.Count; row ++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
HtmlNodeCollection nodes = dataColumns[1].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingUrl=SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
try { Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS); }catch(Exception) { ;}
|
|
SECFilings innerCollection = GetSECFilingDocuments(symbol, DateTime.Parse(dataColumns[3].InnerText), secFilingUrl);
|
|
for (int index = 0; null!=innerCollection && index < innerCollection.Count; index++)
|
|
{
|
|
SECFiling innerFiling = innerCollection[index];
|
|
secFilings.Add(innerFiling);
|
|
}
|
|
}
|
|
return secFilings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"[GetSECFilings] Retrieved {secFilings.Count} SECFilings for {symbol}");
|
|
}
|
|
}
|
|
// Get the filing document in XML format
|
|
private static SECFilings GetSECFilingDocuments(String symbol,DateTime filingDate,String secFilingDocumentUrl)
|
|
{
|
|
Profiler profiler = new Profiler();
|
|
MemoryStream memoryStream = null;
|
|
SECFilings secFilings = new SECFilings();
|
|
String secFilingDocument = null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
int TIMEOUT_MS_BETWEEN_DOCUMENTS=1000;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Retrieving documents for {0} at {1}",symbol,secFilingDocumentUrl));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetSECFilingDocuments");
|
|
if(null == webProxy)MDTrace.WriteLine(LogLevel.DEBUG,"[GetSECFilingDocuments] Not using web proxy.");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentUrl,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]Request Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"secNum\"]");
|
|
String secAccessionNumber = secNodes[0].InnerText;
|
|
secAccessionNumber = secAccessionNumber.Replace("\n","");
|
|
secAccessionNumber = secAccessionNumber.Trim();
|
|
if (secAccessionNumber.Contains("SEC Accession No.")) secAccessionNumber = secAccessionNumber.Substring(18, secAccessionNumber.Length - 18);
|
|
else return null;
|
|
secNodes = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"formName\"]");
|
|
String formName = secNodes[0].InnerText;
|
|
formName = formName.Replace("\n", "");
|
|
formName = formName.Trim();
|
|
HtmlNodeCollection tables = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"tableFile\"]");
|
|
if (null == tables || tables.Count < 1) return null;
|
|
HtmlNodeCollection rows = tables[0].SelectNodes(".//tr");
|
|
httpNetResponse.Dispose();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. {0} Examining form {1} with {2} entries",symbol,formName,rows.Count));
|
|
// go through the rows in the table file, searching for the different submission form content
|
|
int sequence = 0;
|
|
for (int row = 0; row < rows.Count; row++)
|
|
{
|
|
HtmlNodeCollection headerColumns = rows[row].SelectNodes(".//th");
|
|
if (null != headerColumns && headerColumns.Count > 0) continue;
|
|
HtmlNodeCollection dataColumns = rows[row].SelectNodes(".//td");
|
|
if (null == dataColumns || 5 != dataColumns.Count) continue;
|
|
String document = dataColumns[2].InnerText.Trim();
|
|
String type = dataColumns[3].InnerText.Trim();
|
|
if(null!=type && type.Equals("GRAPHIC"))continue;
|
|
if (document.Contains(".htm"))
|
|
{
|
|
|
|
HtmlNodeCollection nodes = dataColumns[2].SelectNodes(".//a");
|
|
if (null == nodes || 0 == nodes.Count) continue;
|
|
String secFilingDocumentXmlUrl = SEC_BASE_URL + nodes[0].GetAttributeValue("href", "unknown");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Requesting:{1}",symbol,secFilingDocumentXmlUrl));
|
|
try{Thread.Sleep(TIMEOUT_MS_BETWEEN_DOCUMENTS);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(secFilingDocumentXmlUrl,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Symbol:{0} Request:{1} failed with status {2}",symbol,httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
continue;
|
|
}
|
|
secFilingDocument = httpNetResponse.ResponseString;
|
|
if (null == secFilingDocument) continue;
|
|
SECFiling secFiling = new SECFiling();
|
|
secFiling.SECAccessionNumber = secAccessionNumber;
|
|
secFiling.Sequence = sequence++;
|
|
secFiling.Symbol = symbol;
|
|
secFiling.Form = type;
|
|
secFiling.Description = formName;
|
|
secFiling.FilingDate = filingDate;
|
|
secFiling.FileNumber = dataColumns[4].InnerText.Trim();
|
|
secFiling.SecFilingUrl = secFilingDocumentXmlUrl;
|
|
secFiling.FormText = secFilingDocument;
|
|
secFiling.FormText = secFiling.FormText.Replace("\n","");
|
|
if (null == secFiling.FormText) continue;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[GetSECFilingDocuments]. Got {0} for {1} filed on {2}",secFiling.Description,symbol,secFiling.FilingDate.ToShortDateString()));
|
|
secFilings.Add(secFiling);
|
|
httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
return secFilings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream = null;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,string.Format("[GetSECFilingDocuments] Done, retrieved {0} documents for {1} in {2}(ms)",secFilings.Count,symbol,profiler.End()));
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** Y I E L D C U R V E - T R E A S U R Y . G O V **************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static YieldCurve GetYieldCurve(int year)
|
|
{
|
|
YieldCurve yieldCurve = new YieldCurve();
|
|
HttpNetResponse httpNetResponse=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://home.treasury.gov/resource-center/data-chart-center/interest-rates/pages/xml?data=daily_treasury_yield_curve&field_tdr_date_value=").Append(year);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
XElement po = XElement.Parse(httpNetResponse.ResponseString);
|
|
IEnumerable<XElement> childElements = from el in po.Elements() where el.Name.LocalName.Equals("entry") select el;
|
|
foreach (XElement element in childElements)
|
|
{
|
|
IEnumerable<XElement> childElementsL1 = from el in element.Elements() select el;
|
|
foreach (XElement elementL1 in childElementsL1)
|
|
{
|
|
if(elementL1.Name.LocalName.Equals("content"))
|
|
{
|
|
IEnumerable<XElement> elementsL2=elementL1.Elements();
|
|
if(!elementsL2.FirstOrDefault().Name.LocalName.Equals("properties"))continue;
|
|
IEnumerable<XElement> elementsL3=elementsL2.FirstOrDefault().Elements();
|
|
YieldCurveData yieldCurveData = new YieldCurveData();
|
|
yieldCurveData.Date=Utility.Epoch;
|
|
foreach (XElement elementL3 in elementsL3)
|
|
{
|
|
if(elementL3.Name.LocalName.EndsWith("NEW_DATE")) yieldCurveData.Date=Utility.ParseDate(Utility.RemoveAfter(elementL3.Value,'T'));
|
|
else if(elementL3.Name.LocalName.EndsWith("1MONTH"))yieldCurveData.Mo1=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("2MONTH"))yieldCurveData.Mo2=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("3MONTH"))yieldCurveData.Mo3=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("6MONTH"))yieldCurveData.Mo6=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("1YEAR"))yieldCurveData.Yr1=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("2YEAR"))yieldCurveData.Yr2=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("3YEAR"))yieldCurveData.Yr3=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("5YEAR"))yieldCurveData.Yr5=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("7YEAR"))yieldCurveData.Yr7=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("10YEAR"))yieldCurveData.Yr10=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("20YEAR"))yieldCurveData.Yr20=FeedParser.ParseValue(elementL3.Value);
|
|
else if(elementL3.Name.LocalName.EndsWith("30YEAR"))yieldCurveData.Yr30=FeedParser.ParseValue(elementL3.Value);
|
|
}
|
|
if(yieldCurveData.IsValid())yieldCurve.Add(yieldCurveData);
|
|
}
|
|
}
|
|
}
|
|
yieldCurve=new YieldCurve(yieldCurve.Distinct().ToList());
|
|
return yieldCurve;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return yieldCurve;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static ETFHoldings GetETFHoldings(String etfSymbol)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
ETFHoldings etfHoldings = new ETFHoldings();
|
|
HttpNetResponse httpNetResponse=null;
|
|
DateTime modified=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
etfSymbol = etfSymbol.ToUpper();
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings");
|
|
strRequest = sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV7(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
|
|
if(null == sections || 1!=sections.Count)
|
|
{
|
|
etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString);
|
|
if(null==etfHoldings)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret the response string.");
|
|
return null;
|
|
}
|
|
return etfHoldings;
|
|
}
|
|
String marker="<table class=\"W(100%) M(0) BdB Bdc($seperatorColor)";
|
|
if(!sections[0].StartsWith(marker))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings: Expected marker {0}",marker));
|
|
return null;
|
|
}
|
|
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(sections[0]);
|
|
memoryStream=new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes(".//td");
|
|
if(null==rows || 0==rows.Count)return null;
|
|
for(int rowIndex=0;rowIndex<rows.Count;rowIndex+=3)
|
|
{
|
|
HtmlNode node = rows[rowIndex];
|
|
ETFHolding etfHolding = new ETFHolding();
|
|
etfHolding.ETFSymbol = etfSymbol;
|
|
etfHolding.HoldingSymbolShareClass = null;
|
|
etfHolding.HoldingCompanyName = rows[rowIndex].InnerText;
|
|
etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = rows[rowIndex+1].InnerText;
|
|
if(null == etfHolding.HoldingSymbol ||
|
|
"N/A".Equals(etfHolding.HoldingSymbol) ||
|
|
"".Equals(etfHolding.HoldingSymbol))continue;
|
|
etfHolding.PercentOfAssets = FeedParser.ParseValue(rows[rowIndex+2].InnerText);
|
|
etfHolding.Modified = modified;
|
|
etfHoldings.Add(etfHolding);
|
|
}
|
|
return etfHoldings;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != memoryStream) memoryStream.Close();
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// <section data-testid="top-holdings" class="yf-1hj9jti">
|
|
private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString)
|
|
{
|
|
ETFHoldings etfHoldings = new ETFHoldings();
|
|
try
|
|
{
|
|
DateTime modified = DateTime.Now;
|
|
int groupBy=3;
|
|
List<String> sections = Sections.GetAllItemsInSections(responseString,"section");
|
|
if(null==sections || 0==sections.Count)return null;
|
|
String sectionItem=sections.Where(x => x.Contains("data-testid=\"top-holdings\"")).FirstOrDefault();
|
|
if(String.IsNullOrEmpty(sectionItem))return null;
|
|
List<String> spans =Sections.GetAllItemsInSections(sectionItem,"span");
|
|
if(null==spans || 0==spans.Count || spans.Count<groupBy)return null;
|
|
for(int index=0;index<spans.Count;index+=groupBy)
|
|
{
|
|
String strElement = spans[index];
|
|
if(0==index)
|
|
{
|
|
String symbolNameHeading=Utility.BetweenString(spans[index],">","<");
|
|
String companyNameHeading=Utility.BetweenString(spans[index+1],">","<");
|
|
String percentOfAssetsHeading=Utility.BetweenString(spans[index+2],">","<");
|
|
if(!symbolNameHeading.Equals("Symbol") || !companyNameHeading.Equals("Company") || !percentOfAssetsHeading.Equals("% Assets"))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Unexpected heading.");
|
|
return null;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ETFHolding etfHolding = new ETFHolding();
|
|
if(index+2>=spans.Count())continue;
|
|
List<String> subSectionsSymbol = Sections.GetSections(spans[index]);
|
|
List<String> subSectionsCompanyName = Sections.GetSections(spans[index+1]);
|
|
List<String> subSectionsPercentOfAssets = Sections.GetSections(spans[index+2]);
|
|
|
|
etfHolding.ETFSymbol = etfSymbol;
|
|
etfHolding.HoldingSymbolShareClass = null;
|
|
|
|
if(null!=subSectionsSymbol && subSectionsSymbol.Count>0)
|
|
{
|
|
etfHolding.HoldingSymbol = subSectionsSymbol.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault();
|
|
}
|
|
|
|
if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"ETFHolding Symbol was not provided in the response for {etfSymbol}");
|
|
break; // we always expect to have a symbol, otherewise something is wrong with the collection.
|
|
}
|
|
|
|
if(null!=subSectionsCompanyName && subSectionsCompanyName.Count>0)
|
|
{
|
|
etfHolding.HoldingCompanyName = subSectionsCompanyName.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault();
|
|
}
|
|
|
|
etfHolding.PercentOfAssets = FeedParser.ParseValue(subSectionsPercentOfAssets.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault());
|
|
|
|
if(double.IsNaN(etfHolding.PercentOfAssets))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"ETFHolding Unable to parse PercentOfAssets for Symbol {etfSymbol}.");
|
|
}
|
|
|
|
etfHolding.Modified = modified;
|
|
etfHolding.HoldingSymbol=etfHolding.HoldingSymbol.Trim().ToUpper();
|
|
etfHoldings.Add(etfHolding);
|
|
}
|
|
}
|
|
return 0==etfHoldings.Count?null:etfHoldings;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Symbol:{0} Exception:{1}",etfSymbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
// ******************************************************************************************************************************************************************************
|
|
// **************************************************************************C O M P A N Y P R O F I L E **********************************************************************
|
|
// ***************************************************************** P R O F I L E : M O R N I N G S T A R *******************************************************************
|
|
// ***************************************************************** D E S C R I P T I O N : R E U T E R S ********************************************************************
|
|
// ******************************************************************************************************************************************************************************
|
|
public static CompanyProfile GetCompanyProfile(String symbol)
|
|
{
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnys";
|
|
|
|
CompanyProfile companyProfile = null;
|
|
companyProfile = GetCompanyProfileMorningStar(symbol,nasdaq);
|
|
if (null == companyProfile)companyProfile = GetCompanyProfileMorningStar(symbol, nyse);
|
|
if (null == companyProfile)companyProfile = GetCompanyProfileYahoo(symbol);
|
|
return companyProfile;
|
|
}
|
|
|
|
// GetCompanyProfile - DataSource Yahoo Finance
|
|
public static CompanyProfile GetCompanyProfileYahoo(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
|
|
WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo");
|
|
// sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile");
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/profile/");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
// Locate Industry
|
|
String strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry", 4);
|
|
if(strIndustry == null)strIndustry = Sections.LocateItem(httpNetResponse.ResponseString, "Industry:??", 2);
|
|
|
|
// Locate Sector
|
|
String strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector", 4);
|
|
if (null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector(s)", 4);
|
|
if(null == strSector) strSector = Sections.LocateItem(httpNetResponse.ResponseString, "Sector:??", 3);
|
|
|
|
if (null != strIndustry && strIndustry.Contains(Constants.CONST_QUESTION)) strIndustry = strIndustry.Replace(Constants.CONST_QUESTION, " - ");
|
|
if (null != strSector && strSector.Contains(Constants.CONST_QUESTION)) strSector = strSector.Replace(Constants.CONST_QUESTION, " - ");
|
|
|
|
// Yahoo changed this... again.
|
|
String strDescription = ExtractLongBusinessSummary(httpNetResponse.ResponseString);
|
|
|
|
if(null!=strDescription && strDescription.Equals("Description Information Not Available"))
|
|
{
|
|
strDescription = null;
|
|
}
|
|
|
|
if(null == strIndustry || null == strSector || null == strDescription)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
CompanyProfile companyProfile = new CompanyProfile();
|
|
companyProfile.Symbol = symbol;
|
|
companyProfile.Sector = strSector;
|
|
companyProfile.Industry = strIndustry;
|
|
companyProfile.Description = strDescription;
|
|
if (null != companyProfile.Sector) companyProfile.Sector = companyProfile.Sector.Trim();
|
|
if (null != companyProfile.Industry) companyProfile.Industry = companyProfile.Industry.Trim();
|
|
if (null != companyProfile.Description) companyProfile.Description = Utility.RemoveHtml(companyProfile.Description.Trim());
|
|
return companyProfile;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieve the Company Description field in the Yahoo Company Profile Data
|
|
/// </summary>
|
|
/// <param name="html"></param>
|
|
/// <returns></returns>
|
|
public static string ExtractLongBusinessSummary(string html)
|
|
{
|
|
if(string.IsNullOrEmpty(html))return null;
|
|
// locate the script containing the assetProfile API response
|
|
int start = html.IndexOf("modules=assetProfile");
|
|
if (start < 0) return null;
|
|
|
|
// move back to start of script tag
|
|
start = html.LastIndexOf("<script", start);
|
|
if (start < 0) return null;
|
|
|
|
int jsonStart = html.IndexOf(">", start);
|
|
if (jsonStart < 0) return null;
|
|
jsonStart++;
|
|
|
|
int jsonEnd = html.IndexOf("</script>", jsonStart);
|
|
if (jsonEnd < 0) return null;
|
|
|
|
string outerJson = html.Substring(jsonStart, jsonEnd - jsonStart);
|
|
|
|
// parse outer JSON
|
|
JObject outer = Newtonsoft.Json.Linq.JObject.Parse(outerJson);
|
|
|
|
// body is escaped JSON
|
|
string bodyJson = outer["body"]?.ToString();
|
|
if (bodyJson == null) return null;
|
|
|
|
// parse inner JSON
|
|
JObject inner = Newtonsoft.Json.Linq.JObject.Parse(bodyJson);
|
|
|
|
return inner["quoteSummary"]?["result"]?[0]?["assetProfile"]?["longBusinessSummary"]?.ToString();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieve company profile information from MorningStar
|
|
/// </summary>
|
|
/// <param name="symbol">The Symbol</param>
|
|
/// <param name="exchange">XNAS or XNYS</param>
|
|
/// <returns></returns>
|
|
public static CompanyProfile GetCompanyProfileMorningStar(String symbol,String exchange)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
if(null==exchange)return default;
|
|
exchange=exchange.ToLower();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyProfileMorningStar");
|
|
sb.Append(String.Format("https://www.morningstar.com/stocks/{0}/{1}/quote",exchange,symbol));
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 15000, webProxy);
|
|
httpNetResponse = HttpNetRequest.GetRequestV6(strRequest, 15000, webProxy);
|
|
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
String strSector=null;
|
|
String strIndustry=null;
|
|
String strDescription=null;
|
|
int itemIndex=0;
|
|
List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);
|
|
|
|
bool found=Sections.FindInSections(sections,"Sector",0,ref itemIndex);
|
|
if(found)strSector=sections[itemIndex+3];
|
|
|
|
found=Sections.FindInSections(sections,"Industry",itemIndex,ref itemIndex);
|
|
if(found)strIndustry=sections[itemIndex+3];
|
|
|
|
found=Sections.FindInSections(sections,"Business Description",itemIndex,ref itemIndex);
|
|
if(found)strDescription=sections[itemIndex+2];
|
|
else
|
|
{
|
|
itemIndex=0;
|
|
found=Sections.FindInSections(sections,"Company Profile",itemIndex,ref itemIndex);
|
|
if(found)strDescription = Sections.GetFirstNonEmptyItemInSection(sections, itemIndex+1);
|
|
}
|
|
|
|
if(null==strIndustry && null==strSector)return null;
|
|
|
|
strSector=strSector.Trim();
|
|
strIndustry=strIndustry.Trim();
|
|
CompanyProfile companyProfile = new CompanyProfile();
|
|
companyProfile.Symbol = symbol;
|
|
companyProfile.Sector = strSector;
|
|
companyProfile.Industry = strIndustry;
|
|
companyProfile.Description=strDescription;
|
|
if(null!=companyProfile.Sector)companyProfile.Sector=companyProfile.Sector.Trim();
|
|
if(null!=companyProfile.Industry)companyProfile.Industry=companyProfile.Industry.Trim();
|
|
if(null!=strDescription)companyProfile.Description=companyProfile.Description.Trim();
|
|
|
|
if(null!=companyProfile.Industry && companyProfile.Industry.Contains(Constants.CONST_QUESTION))
|
|
{
|
|
companyProfile.Industry=companyProfile.Industry.Replace(Constants.CONST_QUESTION," - ");
|
|
}
|
|
if(null!=companyProfile.Sector && companyProfile.Sector.Contains(Constants.CONST_QUESTION))
|
|
{
|
|
companyProfile.Sector=companyProfile.Sector.Replace(Constants.CONST_QUESTION," - ");
|
|
}
|
|
if((null!=companyProfile.Sector && companyProfile.Sector.Equals(Constants.CONST_QUESTION))
|
|
|| (null!=companyProfile.Industry && companyProfile.Industry.Equals(Constants.CONST_QUESTION))
|
|
|| (null!=companyProfile.Description && companyProfile.Description.Equals(Constants.CONST_QUESTION)))return null;
|
|
return companyProfile;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** H E A D L I N E S - S E E K I N G A L P H A ***********************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol)
|
|
{
|
|
Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol);
|
|
return headlines;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Retrieves headlines from seeking alpha. If ignoreMarketDate is TRUE then the results will not be
|
|
/// filtered by the current market date .
|
|
/// If this feed fails then it may be necessary to update the cookie data. I recently added the session_id cookie after a failure
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <param name="ignoreMarketDate"></param>
|
|
/// <returns></returns>
|
|
public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
Headlines headlines=new Headlines();
|
|
|
|
try
|
|
{
|
|
String domain = "seekingalpha.com";
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
|
|
// Visit the home page
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV3");
|
|
sb.Append("https://www.").Append(domain);
|
|
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(sb.ToString(),domain,30000,webProxy,false);
|
|
if(!httpNetResponse.Success)return null;
|
|
CookieCollection cookieCollection = httpNetResponse.CookieCollection;
|
|
|
|
sb=new StringBuilder();
|
|
DateTime marketDate=PremarketDA.GetLatestMarketDate();
|
|
if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
|
|
|
|
#region Create Cookies
|
|
|
|
StringBuilder lastVisitedPage = new StringBuilder();
|
|
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
|
|
lastVisitedPage.Append(symbol);
|
|
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
|
|
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE", lastVisitedPage.ToString()) { Domain = domain });
|
|
cookieCollection.Add(new Cookie("_ga", "GA1.1.561570036.1762275562") { Domain = domain });
|
|
cookieCollection.Add(new Cookie("_ga_KGRFF2R2C5","GS2.1.s1762281227$o2$g1$t1762281245$j42$l0$h0"){ Domain = domain });
|
|
cookieCollection.Add(new Cookie("_gcl_au","1.1.153815328.1762275562"){ Domain = domain });
|
|
cookieCollection.Add(new Cookie("_pxvid","974e14d5-b99f-11f0-a87f-9f37e24a2165"){ Domain = domain });
|
|
cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain });
|
|
cookieCollection.Add(new Cookie("pxcts", "974e2072-b99f-11f0-a881-8ccc46784a0f") { Domain = domain });
|
|
cookieCollection.Add(new Cookie("session_id", "15023729-4bd9-421f-8721-e849f65fb8b6") { Domain = domain });
|
|
|
|
#endregion
|
|
sb.Append("https://seekingalpha.com/symbol/").Append(symbol).Append("/news");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
|
|
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection);
|
|
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
|
|
if(null==keyValuePairs)return null;
|
|
|
|
foreach (KeyValue keyValue in keyValuePairs)
|
|
{
|
|
Headline headline=null;
|
|
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
|
|
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
|
|
headline.Entry=headline.Entry.Replace("\\"," ");
|
|
headline.Entry=headline.Entry.Trim();
|
|
headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
|
|
headline.Entry=Uri.UnescapeDataString(headline.Entry);
|
|
headline.Source = "Seeking Alpha";
|
|
headlines.Add(headline);
|
|
}
|
|
|
|
if(!ignoreMarketDate)
|
|
{
|
|
headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
|
|
}
|
|
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************
|
|
//***************************************************************** H E A D L I N E S - M A R K E T W A T C H *************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
|
|
public static Headlines GetCompanyHeadlinesMarketWatch(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
Headlines headlines=new Headlines();
|
|
DateTime modified=DateTime.Now;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
MemoryStream memoryStream=null;
|
|
CookieCollection cookieCollection=new CookieCollection();
|
|
httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.marketwatch.com",cookieCollection);
|
|
Thread.Sleep(500);
|
|
sb.Append("https://www.marketwatch.com/investing/stock/").Append(symbol).Append("?mod=search_symbol");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesMarketWatch");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,DEFAULT_TIMEOUT_MS,webProxy,true,cookieCollection);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream=new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument=new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection divSections=htmlDocument.DocumentNode.SelectNodes("//*[@class=\"article__content\"]");
|
|
|
|
if(null == divSections)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for request {0} ",strRequest));
|
|
return headlines;
|
|
}
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
|
|
|
|
for(int nodeIndex=0;nodeIndex<divSections.Count;nodeIndex++)
|
|
{
|
|
HtmlNode node=divSections[nodeIndex];
|
|
|
|
byte[] bytes=Encoding.ASCII.GetBytes(node.InnerHtml);
|
|
memoryStream=new MemoryStream(bytes);
|
|
HtmlDocument document=new HtmlDocument();
|
|
document.Load(memoryStream);
|
|
HtmlNode htmlNodeHeadline = document.DocumentNode.SelectSingleNode("//*[@class=\"article__headline\"]");
|
|
HtmlNode htmlNodeTimestamp = document.DocumentNode.SelectSingleNode("//*[@class=\"article__timestamp\"]");
|
|
if(null==htmlNodeHeadline || null==htmlNodeTimestamp)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
List<String> sections=Sections.GetSections(htmlNodeHeadline.InnerHtml);
|
|
if(null == sections || 0==sections.Count)continue;
|
|
String strHeadline = Utility.RemoveHtml(sections[sections.Count-1]);
|
|
String strTimestamp = htmlNodeTimestamp.InnerText;
|
|
|
|
if(String.IsNullOrEmpty(strHeadline) || String.IsNullOrEmpty(strTimestamp))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
Headline headline= new Headline();
|
|
headline.Symbol=symbol;
|
|
headline.Date=FeedParser.ParseValueDateTimeMonthFormatTZ(strTimestamp);
|
|
headline.Entry=strHeadline.Trim();
|
|
headline.Source="MarketWatch";
|
|
headline.Modified=modified;
|
|
headline.CompanyName=null!=companyProfile?companyProfile.CompanyName:null;
|
|
|
|
if(Utility.IsEpoch(headline.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Invalid date for symbol {0}",symbol));
|
|
continue;
|
|
}
|
|
headlines.Add(headline);
|
|
}
|
|
headlines=new Headlines(headlines.Distinct().ToList());
|
|
return headlines;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
//************************************************************************ H E A D L I N E S - N A S D A Q ******************************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static Headlines GetCompanyHeadlinesNASDAQ(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
Headlines headlines=new Headlines();
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
symbol=symbol.ToUpper();
|
|
sb.Append("https://api.nasdaq.com/api/news/topic/articlebysymbol?q=").Append(symbol).Append("|etf&offset=0&limit=8&fallback=false");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
List<List<String>> items=LocateJSONItemsTags(httpNetResponse.ResponseString);
|
|
if(null==items || items.Count<7)return null;
|
|
int index=0;
|
|
for(;index<items.Count;index++)
|
|
{
|
|
if((items[index])[0].Equals("title"))break;
|
|
}
|
|
for(;index<items.Count;index+=7)
|
|
{
|
|
if(!(items[index])[0].Equals("title"))continue;
|
|
|
|
DateTime referenceDate = Utility.Epoch;
|
|
if((items[index+3])[0].Equals("ago"))
|
|
{
|
|
referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
|
|
}
|
|
else if((items[index+2])[0].Equals("ago"))
|
|
{
|
|
referenceDate=ProcessNASDAQRelativeDate((items[index+2])[1]);
|
|
}
|
|
else continue;
|
|
if(Utility.IsEpoch(referenceDate))continue;
|
|
Headline headline=new Headline();
|
|
headline.Source="NASDAQ";
|
|
headline.Symbol=symbol;
|
|
headline.Date=referenceDate;
|
|
headline.Entry=Uri.UnescapeDataString((items[index])[1]);
|
|
headline.Entry=Utility.RemoveHtml(headline.Entry);
|
|
headlines.Add(headline);
|
|
}
|
|
return headlines;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
// ************************************ C O N S U M E R P R I C E I N D E X - B U R E A U O F L A B O R S T I S T I C S ************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
|
|
public static PriceIndices GetConsumerPriceIndices()
|
|
{
|
|
String indexCode=null;
|
|
String indexName=null;
|
|
String indexSource="BLS";
|
|
HttpNetResponse httpNetResponse=null;
|
|
PriceIndices priceIndices=null;
|
|
MemoryStream memoryStream=null;
|
|
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://www.bls.gov/regions/mid-atlantic/data/consumerpriceindexhistorical_us_table.htm");
|
|
String strRequest=sb.ToString();
|
|
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetConsumerPriceIndices");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetConsumerPriceIndices: {0}", strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"www.bls.gov", 300000,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"ro3fxusu_cms\"]");
|
|
HtmlNodeCollection rowData=table[0].SelectNodes(".//tr");
|
|
|
|
HtmlNodeCollection titles=table[0].SelectNodes("//*[@class=\"tableTitle\"]");
|
|
if(titles.Count<1)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate title."));
|
|
return null;
|
|
}
|
|
String titleInfo=titles[0].InnerText;
|
|
indexName=Utility.BetweenString(titleInfo,null,"(").Trim();
|
|
indexCode=Utility.BetweenString(titleInfo,"(",")").Trim();
|
|
if(null==indexName || null==indexCode)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate indexName/indexCode."));
|
|
return null;
|
|
}
|
|
if(rowData.Count<3)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, insufficient row data."));
|
|
return null;
|
|
}
|
|
HtmlNode htmlNode=rowData[2].SelectNodes(".//th").FirstOrDefault();
|
|
if(null==htmlNode)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate proper header."));
|
|
return null;
|
|
}
|
|
if(!"Consumer Price Index".Equals(htmlNode.InnerText))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetConsumerPriceIndices failed, could not locate consumer price index."));
|
|
return null;
|
|
}
|
|
|
|
priceIndices=new PriceIndices();
|
|
for(int index=3;index<rowData.Count;index++)
|
|
{
|
|
HtmlNodeCollection headerElements=rowData[index].SelectNodes(".//th");
|
|
if(null==headerElements || 0==headerElements.Count)break;
|
|
HtmlNode headerNode=headerElements.FirstOrDefault();
|
|
if(null==headerNode ||"".Equals(headerNode.InnerText))break;
|
|
int year=int.Parse(headerNode.InnerText);
|
|
HtmlNodeCollection monthsDataCollection=rowData[index].SelectNodes(".//td");
|
|
for(int monthIndex=0;monthIndex<monthsDataCollection.Count;monthIndex++)
|
|
{
|
|
String strValue=monthsDataCollection[monthIndex].InnerText;
|
|
if(null==strValue || "".Equals(strValue))break;
|
|
double value=FeedParser.ParseValue(strValue);
|
|
if(double.IsNaN(value))break;
|
|
PriceIndex priceIndex=new PriceIndex();
|
|
priceIndex.AsOf=new DateTime(year,monthIndex+1,1);
|
|
priceIndex.Code=indexCode;
|
|
priceIndex.Name=indexName;
|
|
priceIndex.Value=value;
|
|
priceIndex.Source=indexSource;
|
|
priceIndices.Add(priceIndex);
|
|
}
|
|
}
|
|
return priceIndices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)
|
|
{
|
|
httpNetResponse.Dispose();
|
|
}
|
|
if(null!=memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
memoryStream=null;
|
|
}
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************
|
|
// ******************************************************** A N A L Y S T P R I C E T A R G E T - M A R K E T B E A T ***************************************************
|
|
// ***************************************************************************************************************************************************************************
|
|
public static AnalystPriceTarget GetAnalystPriceTarget(String symbol)
|
|
{
|
|
AnalystPriceTarget analystPriceTarget = GetAnalystPriceTargetMarketBeat(symbol);
|
|
return analystPriceTarget;
|
|
}
|
|
|
|
public static AnalystPriceTarget GetAnalystPriceTargetMarketBeat(String symbol)
|
|
{
|
|
AnalystPriceTarget analystPriceTarget = null;
|
|
String nyse="NYSE";
|
|
String nasdaq="NASDAQ";
|
|
|
|
try
|
|
{
|
|
HtmlNodeCollection table = GetMarketBeatAnalystPriceTargetTable(symbol, nasdaq);
|
|
if(null==table)table = GetMarketBeatAnalystPriceTargetTable(symbol, nyse);
|
|
if(null==table)return null;
|
|
|
|
HtmlNodeCollection rows=table[0].SelectNodes(".//tr");
|
|
|
|
if(null == rows || 0==rows.Count)return null;
|
|
|
|
analystPriceTarget=new AnalystPriceTarget();
|
|
analystPriceTarget.Symbol=symbol;
|
|
analystPriceTarget.Date=DateTime.Now.Date;
|
|
|
|
for(int rowIndex=0;rowIndex<rows.Count;rowIndex++)
|
|
{
|
|
HtmlNodeCollection data = rows[rowIndex].SelectNodes(".//td");
|
|
|
|
if(null == data || data.Count ==0 || !data[0].InnerText.Equals("Consensus Price Target"))continue;
|
|
|
|
analystPriceTarget.HighTargetPrice=
|
|
analystPriceTarget.LowTargetPrice=
|
|
analystPriceTarget.MeanTargetPrice=
|
|
analystPriceTarget.MedianTargetPrice=
|
|
FeedParser.ParseValue(data[1].InnerText);
|
|
|
|
break;
|
|
}
|
|
if(double.IsNaN(analystPriceTarget.HighTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.MeanTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.LowTargetPrice)&&
|
|
double.IsNaN(analystPriceTarget.MedianTargetPrice))return null;
|
|
return analystPriceTarget;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static HtmlNodeCollection GetMarketBeatAnalystPriceTargetTable(String symbol, String exchange)
|
|
{
|
|
MemoryStream memoryStream=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
symbol = symbol.ToUpper();
|
|
sb.Append("https://www.marketbeat.com/stocks/").Append(exchange).Append("/").Append(symbol).Append("/price-target");
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetMarketBeat");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("q: {0}", strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
memoryStream = null;
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@id=\"consensus-table\"]");
|
|
if(null==table || 1!=table.Count)
|
|
{
|
|
return null;
|
|
}
|
|
return table;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
memoryStream=null;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N *******************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
// If this breaks then it may be necessary to run the developer tools and point to the operatingPerformance page on MorningStar website.
|
|
// Set a filter in the network tab in the developer tools to "operatingPerformance". You should see the sal api information in the filter list.
|
|
// Match the salVersion to the latest sal version in the actual request.
|
|
// Also... Examine the GetRequestNoEncodingMStar(...) method. It may be necesary to update the RequestId and maybe some other information in there.
|
|
// Not sure how long the RequestId lives for. I updated it on 03/12/2024
|
|
// 02-07-2025 : Morningstar moved the ROA and the ROIC so I had to point to a new url to source that information
|
|
public static Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> GetHistoricalValues(String symbol)
|
|
{
|
|
Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection> values = new Dictionary<TimeSeriesElement.ElementType, TimeSeriesCollection>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnyse";
|
|
String nys="xnys";
|
|
|
|
String strDefinition = default;
|
|
String strColumnDefinition = default;
|
|
int TIMEOUT_BETWEEN_REQUESTS_MS=500;
|
|
|
|
String exchange=nasdaq;
|
|
String url="https://www.morningstar.com/api/v2/stocks/";
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
StringBuilder strReferer = new StringBuilder();
|
|
String strRequest;
|
|
sb=new StringBuilder();
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetHistoricalValues");
|
|
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
exchange=nys;
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
exchange=nyse;
|
|
sb.Append(url).Append(exchange).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
// This next call does not make any outgoing calls, it just attempts to parse out the morningstar security identifier.
|
|
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
|
|
if(null==securityId)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
|
|
return null;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));
|
|
|
|
// PROFITABILITY AND EFFICIENCY - SOURCES ROA AND ROIC
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyMetrics/profitabilityAndEfficiency/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-eqsv-key-metrics-profitability-efficiency&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::OperatingPerformance] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
List<List<String>> profitabilityItems=LocateJSONItems(httpNetResponse.ResponseString);
|
|
Dictionary<String,MStarDataSet> dataSetsProfitabilityAndEfficiency=GetData("fiscalPeriodYear", profitabilityItems, httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// KETSTATS - FINANCIALHEALTH
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/keyStats/financialHealth/").Append(securityId).Append("?languageId=en&locale=en&clientId=MDC&component=sal-components-key-stats-financial-health&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::FinancialHeath] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
|
|
Dictionary<String,MStarDataSet> dataSetsFinancialHealth=GetData("fiscalPeriodYearMonth", items, httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// NEWFINANCIALS - ANNUAL/SUMMARY
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/annual/summary?reportType=A&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-summary&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::AnnualSummary] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsAnnuals=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// NEWFINANCIALS - INCOMESTATEMENT
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/incomeStatement/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::IncomeStatement] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsIncomeStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// NEWFINANCIALS - CASHFLOW
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::Cashflow] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
//NEWFINANCIALS - BALANCESHEET
|
|
sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/balanceSheet/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details&version=").Append(SAL_VERSION);
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
try{Thread.Sleep(TIMEOUT_BETWEEN_REQUESTS_MS);}catch{;}
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("[GetHistoricalValues::BalanceSheet] Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return null;
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsBalanceSheet=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
// ASSEMBLE DATA SETS
|
|
// RETURN ON ASSETS
|
|
strDefinition="roa";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROA;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// RETURN ON INVESTED CAPITAL
|
|
strDefinition="roic";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsProfitabilityAndEfficiency.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.ROIC;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsProfitabilityAndEfficiency[strColumnDefinition],dataSetsProfitabilityAndEfficiency[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// BOOK VALUE PER SHARE
|
|
strDefinition="bookValuePerShare";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsFinancialHealth.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.BVPS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsFinancialHealth[strColumnDefinition],dataSetsFinancialHealth[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// INVENTORIES
|
|
strDefinition="Inventories";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Inventory;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// ACCOUNTS RECEIVABLES
|
|
strDefinition="Trade/Accounts Receivable, Current";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsBalanceSheet.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsBalanceSheet["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.AccountsReceivable;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsBalanceSheet[strColumnDefinition],dataSetsBalanceSheet[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// COST OF GOOD AND SERVICES
|
|
strDefinition="Cost of Goods and Services";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.COGS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// TOTAL OPERATING PROFIT/LOSS
|
|
strDefinition="Total Operating Profit/Loss";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingIncome;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// INTEREST EXPENSE
|
|
strDefinition="Interest Expense Net of Capitalized Interest";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsIncomeStatement["multiplier"].GetItem<double>(0)*-1.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.InterestExpense;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// REPORTED EFFECTIVE TAX RATE
|
|
strDefinition="Reported Effective Tax Rate";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsIncomeStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=100.00;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.TaxRate;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsIncomeStatement[strColumnDefinition],dataSetsIncomeStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// TOTAL REVENUE
|
|
strDefinition="Total Revenue";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.Revenue;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// NET INCOME AVAILABLE TO COMMON STOCKHOLDERS
|
|
strDefinition="Net Income Available to Common Stockholders";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
multiplier/=1000000;
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.NetIncomeAvailableToCommonShareholders;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// DILUTED EPS
|
|
strDefinition="Diluted EPS";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.EPS;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// WORKING CAPITAL
|
|
strDefinition="Working Capital";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.WorkingCapital;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString());
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// FREE CASHFLOW
|
|
strDefinition="Free Cash Flow";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsAnnuals.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsAnnuals["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.FreeCashflow;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsAnnuals[strColumnDefinition],dataSetsAnnuals[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
// OPERATING CASHFLOW
|
|
strDefinition="Cash Generated from Operating Activities";
|
|
strColumnDefinition="columnDefs";
|
|
if(dataSetsCashflowStatement.ContainsKey(strDefinition))
|
|
{
|
|
double multiplier=dataSetsCashflowStatement["multiplier"].GetItem<double>(0);
|
|
TimeSeriesElement.ElementType elementType=TimeSeriesElement.ElementType.OperatingCashflow;
|
|
TimeSeriesCollection timeSeriesCollection=CreateTimeSeriesCollection(symbol, dataSetsCashflowStatement[strColumnDefinition],dataSetsCashflowStatement[strDefinition],elementType,multiplier);
|
|
if (null != timeSeriesCollection && 0 != timeSeriesCollection.Count)
|
|
{
|
|
values.Add(elementType, timeSeriesCollection);
|
|
String strMessage=String.Format("got {0} records for '{1}' for {2}. Data:{3}",timeSeriesCollection.Count,symbol,TimeSeriesElement.StringForType(elementType),timeSeriesCollection.ToString() );
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strMessage);
|
|
}
|
|
}
|
|
|
|
return values;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
internal class MStarDataSet
|
|
{
|
|
private String name;
|
|
private List<String> values;
|
|
public MStarDataSet()
|
|
{
|
|
values=new List<String>();
|
|
}
|
|
public MStarDataSet(String name,List<String> values)
|
|
{
|
|
this.name=name;
|
|
this.values=values;
|
|
}
|
|
public MStarDataSet(String name,String value)
|
|
{
|
|
values=new List<String>();
|
|
this.name=name;
|
|
this.values.Add(value);
|
|
}
|
|
public String Name
|
|
{
|
|
get{return name;}
|
|
set{name=value;}
|
|
}
|
|
public List<String> Values
|
|
{
|
|
get{return values;}
|
|
private set{;}
|
|
}
|
|
public T GetItem<T>(int index=0)
|
|
{
|
|
T result=default(T);
|
|
try {result = (T)Convert.ChangeType(values[index], typeof(T));}
|
|
catch {result = default(T);}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
|
|
//\"orderOfMagnitude\":\"Million\",
|
|
private static double GetMultiplierForDataSet(String responseString)
|
|
{
|
|
double multiplier=1.00;
|
|
try
|
|
{
|
|
string token="\"indexName\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)
|
|
{
|
|
token="\"orderOfMagnitude\"";
|
|
index=responseString.IndexOf(token);
|
|
if(-1==index)return multiplier;
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(1!=parts.Length)return multiplier;
|
|
if(parts[0].Equals("Million"))multiplier=1000000;
|
|
else if(parts[0].Equals("Billion"))multiplier=1000000000;
|
|
else if(parts[0].Equals("Trillion"))multiplier=1000000000000;
|
|
else multiplier=1.00;
|
|
return multiplier;
|
|
}
|
|
else
|
|
{
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(2>parts.Length)return multiplier;
|
|
if(parts[parts.Length-2].Equals("TR"))multiplier=1000000000;
|
|
else
|
|
{
|
|
multiplier=1;
|
|
}
|
|
return multiplier;
|
|
}
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMultiplierForDataSet encountered an exception:{0}",exception.ToString()));
|
|
return multiplier;
|
|
}
|
|
}
|
|
|
|
// \"indexName\":\"Morningstar US Market TR USD\",\"ebitdaCurrency\":\"USD\"}}"
|
|
//\"orderOfMagnitude\":\"Million\",
|
|
// {"currency":"USD","currencySymbol":"$","orderOfMagnitude":"Billion","fiscalYearEndDate":"12-31"},"userType":"Free"}
|
|
private static String GetCurrencyForDataSet(String responseString)
|
|
{
|
|
String currency=null;
|
|
try
|
|
{
|
|
string token="\"indexName\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)
|
|
{
|
|
token="\"currency\"";
|
|
index=responseString.IndexOf(token);
|
|
if(-1==index)return currency;
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(1!=parts.Length)return currency;
|
|
currency=parts[0];
|
|
return currency;
|
|
}
|
|
else
|
|
{
|
|
string strSubString=responseString.Substring(index+token.Length);
|
|
strSubString=Utility.BetweenString(strSubString,"\"","\"");
|
|
string[] parts=strSubString.Split(' ');
|
|
if(2>parts.Length)return currency;
|
|
currency=parts[parts.Length-1];
|
|
return currency;
|
|
}
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCurrencyForDataSet encountered an exception:{0}",exception.ToString()));
|
|
return currency;
|
|
}
|
|
}
|
|
|
|
public static String GetMStarSecurityId(String symbol,String responseString)
|
|
{
|
|
String securityIdentifier=null;
|
|
if(MStarSecurityDA.HasSecurityId(symbol))
|
|
{
|
|
securityIdentifier=MStarSecurityDA.GetSecurityId(symbol);
|
|
return securityIdentifier;
|
|
}
|
|
securityIdentifier=GetMStarSecurityId_1(responseString);
|
|
if(null==securityIdentifier)
|
|
{
|
|
securityIdentifier=GetMStarSecurityId_2(symbol,responseString);
|
|
if(null==securityIdentifier)
|
|
{
|
|
securityIdentifier=GetMStarSecurityId_3(responseString);
|
|
}
|
|
}
|
|
if(null==securityIdentifier )return null;
|
|
securityIdentifier=securityIdentifier.Trim();
|
|
if(securityIdentifier.Equals(","))return null;
|
|
if (securityIdentifier.Length > 25) return null; // As a finakl check ... 25 is the maximum length of the security identifer in the database.
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"Adding Morningtstar securityIdentifier. Symbol:{symbol} SecurityIdentifier:{securityIdentifier}");
|
|
MStarSecurityDA.PutSecurityId(symbol,securityIdentifier);
|
|
return securityIdentifier;
|
|
}
|
|
|
|
//byId:{\"0P000003MU\":$}
|
|
// AAPL:0P000000GY
|
|
|
|
// 5","0P000003MU","MIDD",60
|
|
private static String GetMStarSecurityId_2(String symbol,String responseString)
|
|
{
|
|
try
|
|
{
|
|
String searchString="\""+symbol+"\"";
|
|
int index=responseString.LastIndexOf(searchString);
|
|
if(-1==index)return null;
|
|
int commaCount=0;
|
|
while(index>=0 && commaCount!=2)
|
|
{
|
|
char ch=responseString[index--];
|
|
if(ch.Equals(','))commaCount++;
|
|
}
|
|
if(index<0)return null;
|
|
responseString=responseString.Substring(index+1);
|
|
String securityIdentifier=Utility.BetweenString(responseString,"\"","\"");
|
|
if(securityIdentifier.Equals("en-us")||securityIdentifier.StartsWith("blt"))return null;
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_2 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// us","AAON",401,"subscription","blt8b91befd99376df6","bltc8ff21a187c6e079",2,"image\u002Fsvg+xml","blt187f65f0e94c77b5","0P00000012",60,80,"AAON In
|
|
private static String GetMStarSecurityId_3(String responseString)
|
|
{
|
|
try
|
|
{
|
|
string token="0P";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)return null;
|
|
string securityIdentifier=responseString.Substring(index-token.Length);
|
|
securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_3 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
//","performanceID":"0P000000GY","securityID":"0P000000GY","ticker":"AAPL","exc
|
|
private static String GetMStarSecurityId_1(String responseString)
|
|
{
|
|
try
|
|
{
|
|
string token="\"securityID\"";
|
|
int index=responseString.IndexOf(token);
|
|
if(-1==index)return null;
|
|
string securityIdentifier=responseString.Substring(index+token.Length);
|
|
securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\"");
|
|
return securityIdentifier;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId_1 encountered an exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static Dictionary<String,MStarDataSet> GetData(String responseString)
|
|
{
|
|
try
|
|
{
|
|
Dictionary<String,String> uniqueDataLabels=new Dictionary<String,String>();
|
|
String label="\"label\"";
|
|
String datum="\"datum\"";
|
|
String columnDefs="\"columnDefs\"";
|
|
String strData=null;
|
|
Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
|
|
if(null==responseString)return data;
|
|
int index=responseString.IndexOf(columnDefs);
|
|
if(-1==index)return null;
|
|
strData=responseString.Substring(index+columnDefs.Length);
|
|
strData=Utility.BetweenString(strData,"[","]");
|
|
List<String> items = Utility.ToList(strData,',');
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
String strItem=items[itemIndex];
|
|
strItem=Utility.RemoveQuotes(strItem);
|
|
items[itemIndex]=strItem;
|
|
}
|
|
String labelName=Utility.RemoveQuotes(columnDefs);
|
|
data.Add(labelName,new MStarDataSet(labelName,items));
|
|
|
|
int dataIndex=0;
|
|
for(index=responseString.IndexOf(label);index!=-1;index=responseString.IndexOf(label,dataIndex))
|
|
{
|
|
|
|
dataIndex=responseString.IndexOf(datum,index);
|
|
String strLabel=responseString.Substring(index+label.Length);
|
|
strLabel=Utility.BetweenString(strLabel,"\"","\"");
|
|
if(uniqueDataLabels.ContainsKey(strLabel))break;
|
|
uniqueDataLabels.Add(strLabel,strLabel);
|
|
strData=responseString.Substring(dataIndex+datum.Length);
|
|
strData=Utility.BetweenString(strData,"[","]");
|
|
items=Utility.ToList(strData,',');
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
String strItem=items[itemIndex];
|
|
strItem=Utility.RemoveQuotes(strItem);
|
|
items[itemIndex]=strItem;
|
|
}
|
|
data.Add(strLabel,new MStarDataSet(strLabel,items));
|
|
}
|
|
double multiplier=GetMultiplierForDataSet(responseString);
|
|
String currency=GetCurrencyForDataSet(responseString);
|
|
data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString()));
|
|
data.Add("currency",new MStarDataSet("currency",currency));
|
|
return data;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String responseString) encountered an exception:{0}",exception.ToString()));
|
|
return new Dictionary<String,MStarDataSet>();
|
|
}
|
|
}
|
|
|
|
private static Dictionary<String,MStarDataSet> GetData(String mainHeading, List<List<String>> items,String responseString)
|
|
{
|
|
try
|
|
{
|
|
Dictionary<String,MStarDataSet> data=new Dictionary<String,MStarDataSet>();
|
|
Dictionary<String,Dictionary<String,String>> dataDictionary=new Dictionary<String,Dictionary<String,String>>();
|
|
Dictionary<String,String> uniqueElementNamesDictionary=new Dictionary<String,String>();
|
|
|
|
if(null==mainHeading || null==items || null==responseString)
|
|
{
|
|
return data;
|
|
}
|
|
for(int itemIndex=0;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
List<String> subItems=items[itemIndex];
|
|
if(subItems[0].Equals(mainHeading))
|
|
{
|
|
String strDate=subItems[1];
|
|
dataDictionary.Add(strDate,new Dictionary<String,String>());
|
|
for(++itemIndex;itemIndex<items.Count;itemIndex++)
|
|
{
|
|
List<String> subsubItems=items[itemIndex];
|
|
if(subsubItems[0].Equals(mainHeading))
|
|
{
|
|
itemIndex--;
|
|
break;
|
|
}
|
|
Dictionary<String,String> datedItems=dataDictionary[strDate];
|
|
String subItemElementName=subsubItems[0];
|
|
if(!datedItems.ContainsKey(subItemElementName))
|
|
{
|
|
if(!uniqueElementNamesDictionary.ContainsKey(subItemElementName))uniqueElementNamesDictionary.Add(subItemElementName,subItemElementName);
|
|
datedItems.Add(subItemElementName,subsubItems[1]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
List<String> strDates=new List<String>(dataDictionary.Keys);
|
|
List<String> uniqueElementNames=new List<String>(uniqueElementNamesDictionary.Keys);
|
|
data.Add("columnDefs",new MStarDataSet("columnDefs",strDates));
|
|
|
|
foreach(String elementName in uniqueElementNames)
|
|
{
|
|
List<String> elementValues=new List<String>();
|
|
foreach(String strDate in strDates)
|
|
{
|
|
Dictionary<String,String> inner=dataDictionary[strDate];
|
|
if(!inner.ContainsKey(elementName))
|
|
{
|
|
continue;
|
|
}
|
|
elementValues.Add(inner[elementName]);
|
|
}
|
|
data.Add(elementName,new MStarDataSet(elementName,elementValues));
|
|
}
|
|
double multiplier=GetMultiplierForDataSet(responseString);
|
|
if(double.IsNaN(multiplier))multiplier=1.00; // This GetData method is only used to retrieve ratios so far so the data sets do not have a multiplier section.
|
|
data.Add("multiplier",new MStarDataSet("multiplier",multiplier.ToString())); // If this method is expanded to retrieve data points other than non-ratios then you will have to come up with method for retrieving the multiplier.
|
|
return data; // so that we know if the data is in trillions, billions, millions and what not.
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetData(String mainHeading, List<List<String>> items) encountered an exception {0}",exception.ToString()));
|
|
return new Dictionary<String,MStarDataSet>();
|
|
}
|
|
}
|
|
|
|
private static TimeSeriesCollection CreateTimeSeriesCollection(String symbol, MStarDataSet colDefData, MStarDataSet dataElements, TimeSeriesElement.ElementType elementType,double multiplier=1.00)
|
|
{
|
|
TimeSeriesCollection timeSeriesCollection = new TimeSeriesCollection();
|
|
Dictionary<DateTime,DateTime> dates = new Dictionary<DateTime,DateTime>();
|
|
System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US");
|
|
string[]formatyyyy =new string[]{"yyyy"};
|
|
string[]formatyyyyHMM =new string[]{"yyyy"};
|
|
|
|
|
|
DateTime today=DateTime.Now;
|
|
try
|
|
{
|
|
if(null==colDefData || null==dataElements || null==colDefData.Values || null==dataElements.Values || dataElements.Values.Count()!=colDefData.Values.Count())
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot create time series for symbol {0} for element type {1}",symbol,TimeSeriesElement.StringForType(elementType)));
|
|
return timeSeriesCollection;
|
|
}
|
|
for (int index = 0; index < colDefData.Values.Count;index++)
|
|
{
|
|
DateTime reportDate=DateTime.Now;
|
|
String strDateElement=colDefData.Values[index];
|
|
String strDataElement=dataElements.Values[index];
|
|
|
|
if("_PO_".Equals(strDataElement))
|
|
{
|
|
continue;
|
|
}
|
|
if("null".Equals(strDataElement))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if(strDateElement.Length.Equals(4))
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, formatyyyy, cultureInfo, DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
if(reportDate.Year.Equals(today.Year))
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,today.Month,1);
|
|
}
|
|
else
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,12,31);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, formatyyyyHMM, cultureInfo, DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
}
|
|
TimeSeriesElement timeSeriesElement = new TimeSeriesElement();
|
|
timeSeriesElement.Type = elementType;
|
|
timeSeriesElement.Symbol = symbol;
|
|
timeSeriesElement.AsOf =reportDate;
|
|
timeSeriesElement.Value = FeedParser.ParseValue(dataElements.Values[index])*multiplier;
|
|
if (double.IsNaN(timeSeriesElement.Value)) continue;
|
|
if (!dates.ContainsKey(timeSeriesElement.AsOf))
|
|
{
|
|
dates.Add(timeSeriesElement.AsOf, timeSeriesElement.AsOf);
|
|
timeSeriesCollection.Add(timeSeriesElement);
|
|
}
|
|
}
|
|
return timeSeriesCollection;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateTimeSeriesCollection encountered an exception processing symbol '{0}' {1}",symbol,exception.ToString()));
|
|
return new TimeSeriesCollection();
|
|
}
|
|
}
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 E N D ***********************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 **********************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
public static List<CashflowStatement> GetCashflowStatement(String symbol,CashflowStatement.PeriodType periodType)
|
|
{
|
|
List<CashflowStatement> cashflowStatements=null;
|
|
HttpNetResponse httpNetResponse=null;
|
|
String nasdaq = "xnas";
|
|
String nyse = "xnyse";
|
|
String nys="xnys";
|
|
|
|
try
|
|
{
|
|
if(!CashflowStatement.PeriodType.Annual.Equals(periodType))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Unsupported CashflowStatement.PeriodType Quarterly. Only Annual supported from M*"));
|
|
return null;
|
|
}
|
|
StringBuilder sb = new StringBuilder();
|
|
StringBuilder strReferer = new StringBuilder();
|
|
String strRequest;
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nasdaq).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement");
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nys).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("https://www.morningstar.com/stocks/").Append(nyse).Append("/").Append(symbol).Append("/performance");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest,webProxy);
|
|
if(!httpNetResponse.Success||null==httpNetResponse.ResponseString || "".Equals(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
String securityId=GetMStarSecurityId(symbol,httpNetResponse.ResponseString);
|
|
if(null==securityId)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Cannot determine MorningStar Identifier for {0}",symbol));
|
|
return null;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Morningstar Mapping:'{0}'=>'{1}'",symbol,securityId));
|
|
try{Thread.Sleep(250);}finally{;}
|
|
|
|
// Fetch the equity financial details. Success often requires several attempts.
|
|
httpNetResponse=GetMStarEquityFinancialDetails(securityId);
|
|
if(!httpNetResponse.Success || String.IsNullOrEmpty(httpNetResponse.ResponseString))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Failed to retrieve MorningStar EquityFinancialDetails."));
|
|
}
|
|
Dictionary<String,MStarDataSet> dataSetsCashflowStatement=GetData(httpNetResponse.ResponseString);
|
|
httpNetResponse.Dispose();
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} cashflow statement items for {1}",dataSetsCashflowStatement.Count,symbol));
|
|
|
|
cashflowStatements=CreateCashflowStatements(symbol, dataSetsCashflowStatement);
|
|
return cashflowStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCashflowStatement encountered an exception {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static HttpNetResponse GetMStarEquityFinancialDetails(String securityId)
|
|
{
|
|
HttpNetResponse httpNetResponse = new HttpNetResponse();
|
|
int maxRetries=10;
|
|
int retry=0;
|
|
int waitBetweenAttemptsMS=500;
|
|
|
|
StringBuilder sb=new StringBuilder();
|
|
sb.Append("https://api-global.morningstar.com/sal-service/v1/stock/newfinancials/").Append(securityId).Append("/cashFlow/detail?dataType=A&reportType=A&locale=en&languageId=en&locale=en&clientId=MDC&component=sal-components-equity-financials-details");
|
|
sb.Append("&version=").Append(SAL_VERSION);
|
|
|
|
String strRequest=sb.ToString();
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,strRequest);
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetCashflowStatement");
|
|
while(retry<maxRetries && false==httpNetResponse.Success)
|
|
{
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingMStar(strRequest,webProxy);
|
|
if(httpNetResponse.Success && !String.IsNullOrEmpty(httpNetResponse.ResponseString))return httpNetResponse;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarEquityFinancialDetails Attempt {0} of {1}",retry+1,maxRetries));
|
|
try{Thread.Sleep(waitBetweenAttemptsMS);}catch(Exception){;}
|
|
retry++;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request failed : {0}",strRequest));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,$"**** CHECK MORNINGSTAR SALVERSION = {SAL_VERSION} *****");
|
|
return httpNetResponse;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception running request: {0}, Exception:{1}",strRequest,exception.ToString()));
|
|
return httpNetResponse;
|
|
}
|
|
}
|
|
|
|
private static List<CashflowStatement> CreateCashflowStatements(String symbol,Dictionary<String,MStarDataSet> cashflowDataElements)
|
|
{
|
|
DateTime today=DateTime.Now;
|
|
List<CashflowStatement> cashflowStatements=new List<CashflowStatement>();
|
|
String currencyCashflow=cashflowDataElements["currency"].Values[0];
|
|
double cashflowStatementMultiplier=double.Parse(cashflowDataElements["multiplier"].Values[0]);
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
String baseCurrency="USD";
|
|
|
|
try
|
|
{
|
|
Dictionary<DateTime,Dictionary<String,String>> cashflowDictionaryByDate=CreateDictionaryByDate(cashflowDataElements);
|
|
List<DateTime> cashflowDateKeys=new List<DateTime>(cashflowDictionaryByDate.Keys);
|
|
if(0==cashflowDateKeys.Count)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: No dates in cashflowDataElement."));
|
|
return null;
|
|
}
|
|
|
|
foreach(DateTime cashflowDate in cashflowDateKeys)
|
|
{
|
|
Dictionary<String,String> cashflowDatedElements=cashflowDictionaryByDate[cashflowDate];
|
|
|
|
CashflowStatement cashflowStatement=new CashflowStatement();
|
|
cashflowStatement.Symbol=symbol;
|
|
cashflowStatement.AsOf=cashflowDate;
|
|
cashflowStatement.Modified=DateTime.Now;
|
|
cashflowStatement.Period=CashflowStatement.PeriodType.Annual;
|
|
|
|
if(!baseCurrency.Equals(currencyCashflow))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currencyCashflow,baseCurrency,cashflowStatement.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currencyCashflow,cashflowStatement.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currencyCashflow,cashflowStatement.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString()));
|
|
return null;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currencyCashflow,baseCurrency,cashflowStatement.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for CashflowStatement : {0} to {1} for date {2} multiplier {3}",currencyCashflow,baseCurrency,cashflowStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(cashflowStatement.AsOf,currencyConversionElement);
|
|
}
|
|
|
|
double currencyMultiplier=1.00;
|
|
currencyMultiplier=conversionRates.ContainsKey(cashflowStatement.AsOf)?conversionRates[cashflowStatement.AsOf].UnitsPerSource:currencyMultiplier;
|
|
|
|
if(cashflowDatedElements.ContainsKey("Depreciation and Amortization, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Depreciation and Amortization, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.DepreciationAndAmortization=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.DepreciationAndAmortization=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Deferred Taxes, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Deferred Taxes, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.DeferredIncomeTaxes=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.DeferredIncomeTaxes=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Stock-Based Compensation, Non-Cash Adjustment"))
|
|
{
|
|
String value = cashflowDatedElements["Stock-Based Compensation, Non-Cash Adjustment"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.StockBasedCompensation=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.StockBasedCompensation=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Change in Trade/Accounts Receivable"))
|
|
{
|
|
String value = cashflowDatedElements["Change in Trade/Accounts Receivable"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.AccountsReceivable=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.AccountsReceivable=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Change in Accrued Expenses"))
|
|
{
|
|
String value = cashflowDatedElements["Change in Accrued Expenses"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.AccruedLiabilities=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.AccruedLiabilities=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Cash Generated from Operating Activities"))
|
|
{
|
|
String value = cashflowDatedElements["Cash Generated from Operating Activities"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value))
|
|
{
|
|
cashflowStatement.OperatingCashflow=double.Parse(value)*cashflowStatementMultiplier*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.OperatingCashflow=double.NaN;
|
|
}
|
|
|
|
if(cashflowDatedElements.ContainsKey("Purchase/Sale and Disposal of Property, Plant and Equipment, Net"))
|
|
{
|
|
String value = cashflowDatedElements["Purchase/Sale and Disposal of Property, Plant and Equipment, Net"];
|
|
if(!"_PO_".Equals(value)&&!"null".Equals(value)&& !double.IsNaN(cashflowStatement.OperatingCashflow))
|
|
{
|
|
cashflowStatement.FreeCashflow=(cashflowStatement.OperatingCashflow+(double.Parse(value)*cashflowStatementMultiplier))*currencyMultiplier;
|
|
}
|
|
else cashflowStatement.FreeCashflow=double.NaN;
|
|
}
|
|
|
|
if(cashflowStatement.IsValid())
|
|
{
|
|
cashflowStatements.Add(cashflowStatement);
|
|
}
|
|
}
|
|
|
|
return cashflowStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateCashflowStatements: Exception encountered {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static Dictionary<DateTime,Dictionary<String,String>> CreateDictionaryByDate(Dictionary<String,MStarDataSet> dataSets)
|
|
{
|
|
System.Globalization.CultureInfo cultureInfo = new System.Globalization.CultureInfo("en-US");
|
|
String[] formatyyyy = new string[]{"yyyy"};
|
|
String[] formatyyyyHMM = new string[]{"yyyy-MM"};
|
|
|
|
try
|
|
{
|
|
DateTime today=DateTime.Now;
|
|
Dictionary<DateTime,Dictionary<String,String>> dictionaryByDate=new Dictionary<DateTime,Dictionary<String,String>>();
|
|
MStarDataSet columnDefs=dataSets["columnDefs"];
|
|
List<String> itemKeys=new List<String>(dataSets.Keys);
|
|
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("columnDefs"));
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("multiplier"));
|
|
itemKeys.RemoveAt(itemKeys.IndexOf("currency"));
|
|
for(int dateIndex=0;dateIndex<columnDefs.Values.Count;dateIndex++)
|
|
{
|
|
String strDateElement=columnDefs.Values[dateIndex];
|
|
DateTime reportDate=Utility.Epoch;
|
|
if(strDateElement.Length.Equals(4))
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, formatyyyy, cultureInfo, DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
if(reportDate.Year.Equals(today.Year))
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,today.Month,1);
|
|
}
|
|
else
|
|
{
|
|
reportDate=new DateTime(reportDate.Year,12,31);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
bool result=DateTime.TryParseExact(strDateElement, formatyyyyHMM, cultureInfo, DateTimeStyles.AssumeLocal,out reportDate);
|
|
if(!result)continue;
|
|
}
|
|
|
|
foreach(String itemKey in itemKeys)
|
|
{
|
|
MStarDataSet dataSet=dataSets[itemKey];
|
|
String value=null;
|
|
value=dataSet.Values[dateIndex];
|
|
if(!dictionaryByDate.ContainsKey(reportDate))
|
|
{
|
|
dictionaryByDate.Add(reportDate,new Dictionary<String,String>());
|
|
}
|
|
Dictionary<String,String> datedCollection=dictionaryByDate[reportDate];
|
|
|
|
if(!datedCollection.ContainsKey(itemKey))
|
|
{
|
|
datedCollection.Add(itemKey,value);
|
|
}
|
|
else datedCollection[itemKey]=value;
|
|
}
|
|
}
|
|
return dictionaryByDate;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("CreateDictionaryByDate: Exception encountered {0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// ************************************************************ C A S H F L O W S T A T E M E N T - M O R N I N G S T A R V 2 E N D ************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
|
|
// ****************************************************************************************************************************************************************************
|
|
// ************************************************************ I N C O M E S T A T E M E N T - N A S D A Q / F I N V I Z **************************************************
|
|
// ****************************************************************************************************************************************************************************
|
|
public static List<IncomeStatement> GetIncomeStatement(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
List<IncomeStatement> incomeStatements=GetIncomeStatementNASDAQ(symbol,periodType);
|
|
if(null!=incomeStatements && incomeStatements.Count>0)return incomeStatements;
|
|
incomeStatements=GetIncomeStatementFinViz(symbol,periodType);
|
|
return incomeStatements;
|
|
}
|
|
|
|
|
|
//{"currency":"USD","data":{
|
|
//"Period End Date":["TTM","1/2/2021","12/28/2019","12/29/2018","12/30/2017","12/31/2016","1/2/2016","1/3/2015"],
|
|
//"Total Revenue":["2,930.65","2,513.26","2,959.45","2,722.93","2,335.54","2,267.85","1,826.60","1,636.54"],
|
|
//"Cost of Revenue":["1,872.32","1,631.21","1,855.95","1,718.79","1,422.80","1,366.67","1,120.09","995.95"],
|
|
//"Selling, General and Administrative":["596.80","531.90","583.81","538.84","468.22","471.64","344.55","320.37"],
|
|
//"Gross Profit":["1,058.33","882.05","1,103.50","1,004.14","912.74","901.18","706.50","640.59"],
|
|
//"Total Operating Expense":["2,492.86","2,188.83","2,445.40","2,276.97","1,956.93","1,848.83","1,523.99","1,336.11"],
|
|
|
|
//"Research and Development":["","35.30","41.20","35.30","29.10","26.30","22.40","22.60"],
|
|
//"Net Income":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
//"Income Avail. to Common Incl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
|
|
//"Interest Income Net":["-71.44","-81.52","-83.51","-56.21","-23.64","-21.98","-10.17","-11.99"],
|
|
//"Period Length":["12 Months","53 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","52 Weeks","53 Weeks"],
|
|
//"Unusual Expense/Income":["23.74","25.72","5.64","19.33","65.91","10.52","59.35","19.78"],
|
|
//"Operating Income":["437.79","324.43","514.04","445.97","378.61","419.02","302.60","300.43"],
|
|
//"Other Income Net":["31.16","25.14","32.09","33.76","28.55","24.27","-11.27","-7.65"],
|
|
//"Net Income Before Taxes":["397.51","268.06","462.62","423.51","383.53","421.31","281.17","280.79"],
|
|
//"Provision for Income Taxes":["75.31","60.76","110.38","106.36","85.40","137.09","89.56","87.48"],
|
|
//"Income Avail. to Common Excl. Extraord.":["322.20","207.29","352.24","317.15","298.13","284.22","191.61","193.31"],
|
|
//"Diluted Average Shares":["55.71","55.14","55.66","55.60","56.72","57.09","56.97","56.78"],
|
|
//"Diluted EPS Excl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"],
|
|
//"Diluted EPS Incl. Extraord.":["5.76196","3.75969","6.32888","5.70376","5.25623","4.97882","3.36317","3.40434"]}}
|
|
public static List<IncomeStatement> GetIncomeStatementFinViz(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
String baseCurrency="USD";
|
|
try
|
|
{
|
|
List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
|
|
if(IncomeStatement.PeriodType.Annual.Equals(periodType))sb.Append("&s=IA");
|
|
else sb.Append("&s=IQ");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementsFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
String currency="USD";
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
|
|
if(null!=(jsonToken=json.SelectToken("currency")))
|
|
{
|
|
currency=jsonToken.ToObject<String>();
|
|
}
|
|
|
|
if(null==(jsonToken=json.SelectToken("data")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
|
|
return null;
|
|
}
|
|
|
|
List<String> strDateItems=null;
|
|
List<String> strTotalRevenueItems=null;
|
|
List<String> strCostOfRevenueItems=null;
|
|
List<String> strSGAItems=null;
|
|
List<String> strGrossProfitItems=null;
|
|
List<String> strTotalOperatingExpense=null;
|
|
List<String> strResearchAndDevelopmentItems=null;
|
|
List<String> strIncomeAvailableToCommonShareHolders=null;
|
|
|
|
JToken token=jsonToken["Period End Date"];
|
|
if(null==token)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No IncomeStatemnt dates returned for {0}",symbol));
|
|
return null;
|
|
}
|
|
strDateItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Revenue"];
|
|
if(null!=token)strTotalRevenueItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Cost of Revenue"];
|
|
if(null!=token)strCostOfRevenueItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Selling, General and Administrative"];
|
|
if(null!=token)strSGAItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Gross Profit"];
|
|
if(null!=token)strGrossProfitItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Operating Expense"];
|
|
if(null!=token)strTotalOperatingExpense=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Research and Development"];
|
|
if(null!=token)strResearchAndDevelopmentItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Income Avail. to Common Incl. Extraord."];
|
|
if(null!=token)strIncomeAvailableToCommonShareHolders=token.ToObject<List<String>>();
|
|
|
|
for(int index=0;index<strDateItems.Count;index++)
|
|
{
|
|
String strDateItem=strDateItems[index];
|
|
if(String.IsNullOrEmpty(strDateItem))continue;
|
|
if("TTM".Equals(strDateItem,StringComparison.CurrentCultureIgnoreCase))continue;
|
|
IncomeStatement incomeStatement=new IncomeStatement();
|
|
incomeStatement.Symbol=symbol;
|
|
incomeStatement.Period=periodType;
|
|
incomeStatement.AsOf=Utility.ParseDate(strDateItem);
|
|
incomeStatement.Modified=DateTime.Now;
|
|
double multiplier=1.00;
|
|
|
|
// perform currency conversion
|
|
if(!baseCurrency.Equals(currency))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,incomeStatement.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,incomeStatement.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,incomeStatement.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString()));
|
|
continue;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,incomeStatement.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for IncomeStatement : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,incomeStatement.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(incomeStatement.AsOf,currencyConversionElement);
|
|
}
|
|
multiplier=conversionRates.ContainsKey(incomeStatement.AsOf)?conversionRates[incomeStatement.AsOf].UnitsPerSource:multiplier;
|
|
|
|
if(null!=strTotalRevenueItems && strTotalRevenueItems.Count>index)incomeStatement.TotalRevenue=FeedParser.ParseValue(strTotalRevenueItems[index])*1000000.00*multiplier;
|
|
if(null!=strCostOfRevenueItems && strCostOfRevenueItems.Count>index)incomeStatement.CostOfRevenue=FeedParser.ParseValue(strCostOfRevenueItems[index])*1000000.00*multiplier;
|
|
if(null!=strSGAItems && strSGAItems.Count>index)incomeStatement.SGA=FeedParser.ParseValue(strSGAItems[index])*1000000.00*multiplier;
|
|
if(null!=strGrossProfitItems && strGrossProfitItems.Count>index)incomeStatement.GrossProfit=FeedParser.ParseValue(strGrossProfitItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalOperatingExpense && strTotalOperatingExpense.Count>index)incomeStatement.OperatingExpenses=FeedParser.ParseValue(strTotalOperatingExpense[index])*1000000.00*multiplier;
|
|
if(null!=strResearchAndDevelopmentItems && strResearchAndDevelopmentItems.Count>index)incomeStatement.ResearchAndDevelopment=FeedParser.ParseValue(strResearchAndDevelopmentItems[index])*1000000.00*multiplier;
|
|
if(null!=strIncomeAvailableToCommonShareHolders && strIncomeAvailableToCommonShareHolders.Count>index)incomeStatement.NetIncomeApplicableToCommonShares=FeedParser.ParseValue(strIncomeAvailableToCommonShareHolders[index])*1000000.00*multiplier;
|
|
incomeStatements.Add(incomeStatement);
|
|
}
|
|
return incomeStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetIncomeStatementFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
public static List<IncomeStatement> GetIncomeStatementNASDAQ(String symbol,IncomeStatement.PeriodType periodType)
|
|
{
|
|
List<IncomeStatement> incomeStatements=new List<IncomeStatement>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
double multiplier=1000.00;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
|
|
if(periodType.Equals(IncomeStatement.PeriodType.Annual))sb.Append("");
|
|
else sb.Append("?frequency=2");
|
|
|
|
strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success) return incomeStatements;
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
if(null==(jsonToken=json.SelectToken("data.incomeStatementTable.headers")))
|
|
{
|
|
String reason="unknown";
|
|
jsonToken=json.SelectToken("message");
|
|
if(null!=jsonToken)reason=jsonToken.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
|
|
return incomeStatements;
|
|
}
|
|
JObject headerObject=(JObject)jsonToken;
|
|
int elements=headerObject.Count-1;
|
|
String headerElement=headerObject["value1"].ToString();
|
|
if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
|
|
return incomeStatements;
|
|
}
|
|
|
|
for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
|
|
{
|
|
IncomeStatement incomeStatement=new IncomeStatement();
|
|
String itemIndex="value"+headerIndex;
|
|
jsonToken=json.SelectToken("data.incomeStatementTable.headers."+itemIndex);
|
|
if(null==jsonToken)continue;
|
|
String strDate=jsonToken.ToString();
|
|
if(string.IsNullOrEmpty(strDate))continue;
|
|
incomeStatement.AsOf=Utility.ParseDate(strDate);
|
|
incomeStatement.Symbol=symbol.ToUpper();
|
|
incomeStatement.Period=periodType;
|
|
incomeStatements.Add(incomeStatement);
|
|
}
|
|
|
|
jsonToken=json.SelectToken("data.incomeStatementTable.rows");
|
|
if(null==jsonToken)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
|
|
return incomeStatements;
|
|
}
|
|
|
|
JArray collection=(JArray)jsonToken;
|
|
for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
|
|
{
|
|
JObject rowItem=(JObject)collection[rowIndex];
|
|
String strHeader=rowItem.SelectToken("value1").ToString();
|
|
|
|
if(null==strHeader)continue;
|
|
if("Total Revenue".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].TotalRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Cost of Revenue".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].CostOfRevenue=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Gross Profit".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].GrossProfit=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Operating Expenses".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Research and Development".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].ResearchAndDevelopment=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Sales, General and Admin.".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].SGA=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Non-Recurring Items".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Operating Items".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].OperatingExpenses=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Operating Income".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Add'l income/expense items".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Earnings Before Interest and Tax".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].EBIT=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Interest Expense".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].InterestExpense=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Earnings Before Tax".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Income Tax".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].IncomeTaxExpense=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Minority Interest".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Equity Earnings/Loss Unconsolidated Subsidiary".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Income-Cont. Operations".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Income".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].NetIncome=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Net Income Applicable to Common Shareholders".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<incomeStatements.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
incomeStatements[index].NetIncomeApplicableToCommonShares=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
}
|
|
return incomeStatements;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
|
|
}
|
|
}
|
|
|
|
// *******************************************************************************************************************************************************************
|
|
// ************************************************************B A L A N C E S H E E T - N A S D A Q / F I N V I Z **************************************************
|
|
// *******************************************************************************************************************************************************************
|
|
public static List<BalanceSheet> GetBalanceSheet(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
List<BalanceSheet> balanceSheet=null;
|
|
balanceSheet=GetBalanceSheetNASDAQ(symbol,periodType);
|
|
if(null!=balanceSheet && balanceSheet.Count>0)return balanceSheet;
|
|
balanceSheet=GetBalanceSheetFinViz(symbol,periodType);
|
|
return balanceSheet;
|
|
}
|
|
//https://finviz.com/api/statement.ashx?t=aapl&s=BA
|
|
//https://finviz.com/api/statement.ashx?t=aapl&s=BQ
|
|
//{"currency":"USD",
|
|
//"data":{
|
|
//"Period End Date":["9/26/2020","9/28/2019","9/29/2018","9/30/2017","9/24/2016","9/26/2015","9/27/2014","9/28/2013"],
|
|
//"Cash and Equivalents":["20,243.00","36,640.00","14,338.00","12,307.00","11,883.00","9,731.00","3,612.00","5,554.00"],
|
|
//"Short Term Investments":["52,927.00","51,713.00","40,388.00","53,892.00","46,671.00","20,481.00","11,233.00","26,287.00"],
|
|
//"Cash and Short Term Investments":["90,943.00","100,557.00","66,301.00","74,181.00","67,155.00","41,601.00","25,077.00","40,546.00"],
|
|
//"Accounts Receivable - Trade, Net":["16,120.00","22,926.00","23,186.00","17,874.00","15,754.00","16,849.00","17,460.00","13,102.00"],
|
|
//"Total Receivables, Net":["37,445.00","45,804.00","48,995.00","35,673.00","29,299.00","30,343.00","27,219.00","20,641.00"],
|
|
//"Total Inventory":["4,061.00","4,106.00","3,956.00","4,855.00","2,132.00","2,349.00","2,111.00","1,764.00"],
|
|
//"Total Current Assets":["143,713.00","162,819.00","131,339.00","128,645.00","106,869.00","89,378.00","68,531.00","73,286.00"],
|
|
//"Property, Plant And Equipment - Gross":["112,096.00","95,957.00","90,403.00","75,076.00","61,245.00","49,257.00","39,015.00","28,519.00"],
|
|
//"Property, Plant And Equipment - Net":["45,336.00","37,378.00","41,304.00","33,783.00","27,010.00","22,471.00","20,624.00","16,597.00"],
|
|
//"Goodwill, Net":["","","","","5,414.00","5,116.00","4,616.00","1,577.00"],
|
|
//"Intangibles, Net":["","","","","3,206.00","3,893.00","4,142.00","4,179.00"],
|
|
//"Long Term Investments":["100,887.00","105,341.00","170,799.00","194,714.00","170,430.00","164,065.00","130,162.00","106,215.00"],
|
|
//"Other Long Term Assets":["33,952.00","32,978.00","22,283.00","18,177.00","8,757.00","5,422.00","3,764.00","5,146.00"],
|
|
//"Total Assets":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
|
|
//"Accounts Payable":["42,296.00","46,236.00","55,888.00","44,242.00","37,294.00","35,490.00","30,196.00","22,367.00"],
|
|
//"Accrued Expenses":["1,436.00","","","","20,951.00","24,169.00","7,689.00","4,782.00"],
|
|
//"Notes Payable/Short Term Debt":["4,996.00","5,980.00","11,964.00","11,977.00","8,105.00","8,499.00","6,308.00","0.00"],
|
|
//"Current Port. of LT Debt/Capital Leases":["8,797.00","10,260.00","8,784.00","6,496.00","3,500.00","2,500.00","",""],
|
|
//"Other Current liabilities":["47,867.00","43,242.00","39,293.00","38,099.00","9,156.00","9,952.00","19,255.00","16,509.00"],
|
|
//"Total Current Liabilities":["105,392.00","105,718.00","115,929.00","100,814.00","79,006.00","80,610.00","63,448.00","43,658.00"],
|
|
//"Long Term Debt":["98,667.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
|
|
//"Capital Lease Obligations":["637.00","","","","","","",""],
|
|
//"Total Long Term Debt":["99,304.00","91,807.00","93,735.00","97,207.00","75,427.00","53,329.00","28,987.00","16,960.00"],
|
|
//"Total Debt":["113,097.00","108,047.00","114,483.00","115,680.00","87,032.00","64,328.00","35,295.00","16,960.00"],
|
|
//"Other Liabilities":["53,853.00","50,503.00","48,914.00","11,747.00","12,985.00","12,989.00","7,598.00","6,344.00"],
|
|
//"Total Liabilities":["258,549.00","248,028.00","258,578.00","241,272.00","193,437.00","170,990.00","120,292.00","83,451.00"],
|
|
//"Common Stock":["50,779.00","45,174.00","40,201.00","35,867.00","31,251.00","27,416.00","23,313.00","19,764.00"],
|
|
//"Additional Paid-In Capital":["","","","","","","",""],
|
|
//"Retained Earnings":["14,966.00","45,898.00","70,400.00","98,330.00","96,364.00","92,284.00","87,152.00","104,256.00"],
|
|
//"Unrealized Gain (Loss)":["1,846.00","707.00","-3,209.00","328.00","1,174.00","-653.00","-242.00","-105.00"],
|
|
//"Other Equity":["-2,252.00","-1,291.00","-245.00","-478.00","-540.00","308.00","1,324.00","-366.00"],
|
|
//"Total Equity":["65,339.00","90,488.00","107,147.00","134,047.00","128,249.00","119,355.00","111,547.00","123,549.00"],
|
|
//"Total Liabilities and Equity":["323,888.00","338,516.00","365,725.00","375,319.00","321,686.00","290,345.00","231,839.00","207,000.00"],
|
|
//"Total Common Shares Outstanding":["16,976.76","17,772.94","19,019.94","20,504.80","21,344.66","22,315.01","23,464.64","25,177.46"],
|
|
//"Full-Time Employees":["147,000","137,000","132,000","123,000","116,000","110,000","92,600","80,300"],
|
|
//"Number of Common Shareholders":["22,797","23,233","23,712","25,333","25,641","25,924","26,112","24,710"]}}
|
|
public static List<BalanceSheet> GetBalanceSheetFinViz(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
String baseCurrency="USD";
|
|
try
|
|
{
|
|
List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
|
|
Dictionary<DateTime,CurrencyConversionElement> conversionRates=new Dictionary<DateTime,CurrencyConversionElement>();
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/api/statement.ashx?t=").Append(symbol.ToLower());
|
|
if(BalanceSheet.PeriodType.Annual.Equals(periodType))sb.Append("&s=BA");
|
|
else sb.Append("&s=BQ");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz {0}",strRequest));
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
String currency="USD";
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
|
|
if(null!=(jsonToken=json.SelectToken("currency")))
|
|
{
|
|
currency=jsonToken.ToObject<String>();
|
|
}
|
|
|
|
if(null==(jsonToken=json.SelectToken("data")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format(""));
|
|
return null;
|
|
}
|
|
|
|
List<String> strDateItems=null;
|
|
List<String> strLongTermDebtItems=null;
|
|
List<String> strTotalCurrentAssetItems=null;
|
|
List<String> strTotalLiabilityItems=null;
|
|
List<String> strTotalAssetItems=null;
|
|
List<String> strGoodwillItems=null;
|
|
List<String> strPPEItems=null;
|
|
List<String> strCashAndEquivalentsItems=null;
|
|
List<String> strTotalInventoryItems=null;
|
|
List<String> strIntangiblesItems=null;
|
|
List<String> strTotalCurrentLiabilityItems=null;
|
|
List<String> strOtherLiabilityItems=null;
|
|
List<String> strTotalEquityItems=null;
|
|
|
|
JToken token=jsonToken["Period End Date"];
|
|
if(null==token)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No BalanceSheet dates returned for {0}",symbol));
|
|
return null;
|
|
}
|
|
strDateItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Long Term Debt"];
|
|
if(null!=token)strLongTermDebtItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Current Assets"];
|
|
if(null!=token)strTotalCurrentAssetItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Liabilities"];
|
|
if(null!=token)strTotalLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Assets"];
|
|
if(null!=token)strTotalAssetItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Goodwill, Net"];
|
|
if(null!=token)strGoodwillItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Property, Plant And Equipment - Net"];
|
|
if(null!=token)strPPEItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Cash and Equivalents"];
|
|
if(null!=token)strCashAndEquivalentsItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Inventory"];
|
|
if(null!=token)strTotalInventoryItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Intangibles, Net"];
|
|
if(null!=token)strIntangiblesItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Current Liabilities"];
|
|
if(null!=token)strTotalCurrentLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Other Liabilities"];
|
|
if(null!=token)strOtherLiabilityItems=token.ToObject<List<String>>();
|
|
|
|
token=jsonToken["Total Equity"];
|
|
if(null!=token)strTotalEquityItems=token.ToObject<List<String>>();
|
|
|
|
for(int index=0;index<strDateItems.Count;index++)
|
|
{
|
|
BalanceSheet balanceSheet=new BalanceSheet();
|
|
|
|
balanceSheet.Symbol=symbol;
|
|
balanceSheet.Period=periodType;
|
|
balanceSheet.AsOf=Utility.ParseDate(strDateItems[index]);
|
|
balanceSheet.Modified=DateTime.Now;
|
|
double multiplier=1.00;
|
|
|
|
// perform currency conversion
|
|
if(!baseCurrency.Equals(currency))
|
|
{
|
|
if(null==CurrencyConversionDA.GetMaxDateForCurrency(currency,baseCurrency,balanceSheet.AsOf))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** Retrieving currency rates for {0} {1}",currency,balanceSheet.AsOf.ToShortDateString()));
|
|
CurrencyConversionCollection currencyConversionCollection=MarketDataHelper.GetCurrencyConversion(currency,balanceSheet.AsOf);
|
|
if(!CurrencyConversionDA.InsertCurrencyConversionRates(currencyConversionCollection))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*********** No currency conversion from {0} to {1} for date {2}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString()));
|
|
continue;
|
|
}
|
|
}
|
|
CurrencyConversionElement currencyConversionElement=CurrencyConversionDA.GetCurrencyConversionMaxDate(currency,baseCurrency,balanceSheet.AsOf);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************ Currency conversion for BalanceSheet : {0} to {1} for date {2} multiplier {3}",currency,baseCurrency,balanceSheet.AsOf.ToShortDateString(),Utility.FormatNumber(currencyConversionElement.UnitsPerSource,6)));
|
|
conversionRates.Add(balanceSheet.AsOf,currencyConversionElement);
|
|
}
|
|
multiplier=conversionRates.ContainsKey(balanceSheet.AsOf)?conversionRates[balanceSheet.AsOf].UnitsPerSource:multiplier;
|
|
|
|
if(null!=strLongTermDebtItems && strLongTermDebtItems.Count>index)balanceSheet.LongTermDebt=FeedParser.ParseValue(strLongTermDebtItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalCurrentAssetItems && strTotalCurrentAssetItems.Count>index)balanceSheet.TotalCurrentAssets=FeedParser.ParseValue(strTotalCurrentAssetItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalLiabilityItems && strTotalLiabilityItems.Count>index)balanceSheet.TotalLiabilities=FeedParser.ParseValue(strTotalLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalAssetItems && strTotalAssetItems.Count>index)balanceSheet.TotalAssets=FeedParser.ParseValue(strTotalAssetItems[index])*1000000.00*multiplier;
|
|
if(null!=strGoodwillItems && strGoodwillItems.Count>index)balanceSheet.Goodwill=FeedParser.ParseValue(strGoodwillItems[index])*1000000.00*multiplier;
|
|
if(null!=strPPEItems && strPPEItems.Count>index)balanceSheet.PropertyPlantAndEquipment=FeedParser.ParseValue(strPPEItems[index])*1000000.00*multiplier;
|
|
if(null!=strCashAndEquivalentsItems && strCashAndEquivalentsItems.Count>index)balanceSheet.CashAndCashEquivalents=FeedParser.ParseValue(strCashAndEquivalentsItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalInventoryItems && strTotalInventoryItems.Count>index)balanceSheet.Inventory=FeedParser.ParseValue(strTotalInventoryItems[index])*1000000.00*multiplier;
|
|
if(null!=strIntangiblesItems && strIntangiblesItems.Count>index)balanceSheet.IntangibleAssets=FeedParser.ParseValue(strIntangiblesItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalCurrentLiabilityItems && strTotalCurrentLiabilityItems.Count>index)balanceSheet.TotalCurrentLiabilities=FeedParser.ParseValue(strTotalCurrentLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strOtherLiabilityItems && strOtherLiabilityItems.Count>index)balanceSheet.OtherLiabilities=FeedParser.ParseValue(strOtherLiabilityItems[index])*1000000.00*multiplier;
|
|
if(null!=strTotalEquityItems && strTotalEquityItems.Count>index)balanceSheet.TotalStockHolderEquity=FeedParser.ParseValue(strTotalEquityItems[index])*1000000.00*multiplier;
|
|
balanceSheets.Add(balanceSheet);
|
|
}
|
|
return balanceSheets;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetFinViz Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<BalanceSheet> GetBalanceSheetNASDAQ(String symbol,BalanceSheet.PeriodType periodType)
|
|
{
|
|
List<BalanceSheet> balanceSheets=new List<BalanceSheet>();
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
double multiplier=1000.00;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://api.nasdaq.com/api/company/").Append(symbol).Append("/financials");
|
|
if(periodType.Equals(BalanceSheet.PeriodType.Annual)) sb.Append("?frequency=1");
|
|
else sb.Append("?frequency=2");
|
|
|
|
strRequest=sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success) return balanceSheets;
|
|
byte[] streamBytes=Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
JToken jsonToken=null;
|
|
if(null==(jsonToken=json.SelectToken("data.balanceSheetTable.headers")))
|
|
{
|
|
String reason="unknown";
|
|
jsonToken=json.SelectToken("message");
|
|
if(null!=jsonToken) reason=jsonToken.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains no headers for symbol:{0} for request:{1}, Reason:{2}",symbol,strRequest,reason));
|
|
return balanceSheets;
|
|
}
|
|
JObject headerObject=(JObject)jsonToken;
|
|
int elements=headerObject.Count-1;
|
|
String headerElement=headerObject["value1"].ToString();
|
|
if((periodType.Equals(IncomeStatement.PeriodType.Annual)&&!headerElement.Equals("Period Ending:"))||(periodType.Equals(IncomeStatement.PeriodType.Quarterly)&&!headerElement.Equals("Quarterly Ending:")))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Response contains unexpected header for symbol:{0} for request:{1}",symbol,strRequest));
|
|
return balanceSheets;
|
|
}
|
|
|
|
for(int headerIndex=2;headerIndex<elements+2;headerIndex++)
|
|
{
|
|
BalanceSheet balanceSheet =new BalanceSheet();
|
|
String itemIndex="value"+headerIndex;
|
|
jsonToken=json.SelectToken("data.balanceSheetTable.headers."+itemIndex);
|
|
if(null==jsonToken) continue;
|
|
String strDate=jsonToken.ToString();
|
|
if(string.IsNullOrEmpty(strDate))continue;
|
|
balanceSheet.AsOf=Utility.ParseDate(strDate);
|
|
balanceSheet.Symbol=symbol.ToUpper();
|
|
balanceSheet.Period=periodType;
|
|
balanceSheets.Add(balanceSheet);
|
|
}
|
|
|
|
jsonToken=json.SelectToken("data.balanceSheetTable.rows");
|
|
if(null==jsonToken)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Data comtains no rows symbol:{0} for request:{1}",symbol,strRequest));
|
|
return balanceSheets;
|
|
}
|
|
|
|
JArray collection=(JArray)jsonToken;
|
|
for(int rowIndex=0;rowIndex<collection.Count;rowIndex++)
|
|
{
|
|
JObject rowItem=(JObject)collection[rowIndex];
|
|
String strHeader=rowItem.SelectToken("value1").ToString();
|
|
|
|
if(null==strHeader) continue;
|
|
if("Current Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Cash and Cash Equivalents".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].CashAndCashEquivalents=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Short-Term Investments".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Net Receivables".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Inventory".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].Inventory=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Current Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Current Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalCurrentAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Long-Term Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Long-Term Investments".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Fixed Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].PropertyPlantAndEquipment=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Goodwill".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].Goodwill=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Intangible Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].IntangibleAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Assets".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Deferred Asset Charges".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Assets".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalAssets=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Current Liabilities".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Accounts Payable".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Short-Term Debt / Current Portion of Long-Term Debt".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Current Liabilities".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Current Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalCurrentLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Long-Term Debt".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].LongTermDebt=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Other Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].OtherLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Deferred Liability Charges".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].DeferredLongTermLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Misc. Stocks".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Minority Interest".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].MinorityInterest=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Total Liabilities".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalLiabilities=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Stock Holders Equity".Equals(strHeader))
|
|
{
|
|
for(int index=0;index<balanceSheets.Count;index++)
|
|
{
|
|
JToken element=rowItem.SelectToken("value"+(index+2));
|
|
balanceSheets[index].TotalStockHolderEquity=Utility.ParseCurrency(element.ToString())*multiplier;
|
|
}
|
|
}
|
|
else if("Total Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Common Stocks".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Capital Surplus".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Retained Earnings".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Treasury Stock".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Other Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
else if("Total Liabilities & Equity".Equals(strHeader)||"Total Liabilities & Equity".Equals(strHeader))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
return balanceSheets;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetBalanceSheetNASDAQ Symbol:{0} Exception:{1}",symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
if(null!=memoryStream) { memoryStream.Close(); memoryStream.Dispose(); }
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************************************************************************************************************
|
|
// **************************************************************************** C U R R E N C Y C O N V E R S I O N ****************************************************
|
|
// *****************************************************************************************************************************************************************************
|
|
public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
CurrencyConversionCollection currencyConversionElements=new CurrencyConversionCollection();
|
|
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sourceCurrency=sourceCurrency.ToUpper();
|
|
sb.Append("https://www.xe.com/currencytables/?from=").Append(sourceCurrency).Append("&date=").Append(Utility.DateTimeToStringYYYYHMMHDD(conversionDate)).Append("#table-section");
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV4(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
int itemIndex=0;
|
|
List<String> sections=Sections.GetSections(httpNetResponse.ResponseString);
|
|
if(!Sections.FindInSections(sections,"Currency Table:",0,ref itemIndex,false))return null;
|
|
if(!Sections.FindInSections(sections,"Currency",itemIndex,ref itemIndex))return null;
|
|
itemIndex+=20;
|
|
|
|
while(true)
|
|
{
|
|
CurrencyConversionElement currencyConversionElement=new CurrencyConversionElement();
|
|
currencyConversionElement.AsOf=conversionDate;
|
|
currencyConversionElement.SourceCurrency=sourceCurrency;
|
|
currencyConversionElement.DestinationCurrency=sections[itemIndex];
|
|
itemIndex+=3;
|
|
currencyConversionElement.DestinationCurrencyName=sections[itemIndex];
|
|
itemIndex+=2;
|
|
currencyConversionElement.UnitsPerSource=FeedParser.ParseValue(sections[itemIndex]);
|
|
itemIndex+=2;
|
|
currencyConversionElement.SourcePerUnit=FeedParser.ParseValue(sections[itemIndex]);
|
|
currencyConversionElements.Add(currencyConversionElement);
|
|
itemIndex+=5;
|
|
if(itemIndex>=sections.Count || null==sections[itemIndex] || "".Equals(sections[itemIndex]) || sections[itemIndex].Length<3)break;
|
|
}
|
|
return currencyConversionElements;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** F U N D A M E N T A L - F I N V I S *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Fundamental GetFundamentalFinViz(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
MemoryStream memoryStream=null;
|
|
try
|
|
{
|
|
Fundamental fundamental = new Fundamental();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finviz.com/quote.ashx?t=").Append(symbol.ToLower());
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalFinViz");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode));
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Get Fundamental for {0} failed with {1}",strRequest,httpNetResponse.ErrorMessage));
|
|
return null;
|
|
}
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
HtmlNodeCollection nodeCollection = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"js-snapshot-table snapshot-table2 screener_snapshot-table-body\"]");
|
|
if(null==nodeCollection || nodeCollection.Count<1)return null;
|
|
fundamental.AsOf=DateTime.Now;
|
|
fundamental.Symbol=symbol;
|
|
fundamental.Source = "FINVIZ";
|
|
for(int index=0;index<nodeCollection[0].ChildNodes.Count;index++)
|
|
{
|
|
HtmlNodeCollection dataCollection = nodeCollection[0].ChildNodes[index].SelectNodes(".//td");
|
|
if(null==dataCollection)continue;
|
|
for(int dataIndex=0;dataIndex<dataCollection.Count;dataIndex++)
|
|
{
|
|
String header=dataCollection[dataIndex].InnerText;
|
|
if(header.Equals("Market Cap"))
|
|
{
|
|
fundamental.MarketCap=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("P/E"))
|
|
{
|
|
fundamental.PE=fundamental.TrailingPE=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("EPS (ttm)"))
|
|
{
|
|
fundamental.EPS=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Shs Outstand"))
|
|
{
|
|
fundamental.SharesOutstanding=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Income"))
|
|
{
|
|
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("PEG"))
|
|
{
|
|
fundamental.PEG=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Book/sh"))
|
|
{
|
|
fundamental.BookValuePerShare=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("ROA"))
|
|
{
|
|
fundamental.ReturnOnAssets=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("ROE"))
|
|
{
|
|
fundamental.ReturnOnEquity=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("52W Range"))
|
|
{
|
|
String strLowHigh=dataCollection[dataIndex+1].InnerText;
|
|
if(null==strLowHigh)continue;
|
|
String[] items=strLowHigh.Split('-');
|
|
if(null==items||2!=items.Length)continue;
|
|
items[0]=items[0].Trim();
|
|
items[1]=items[1].Trim();
|
|
fundamental.Low52=FeedParser.ParseValue(items[0]);
|
|
fundamental.High52=FeedParser.ParseValue(items[1]);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Beta"))
|
|
{
|
|
fundamental.Beta=FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Earnings"))
|
|
{
|
|
String earnings=dataCollection[dataIndex+1].InnerText;
|
|
if(null==earnings)continue;
|
|
String[] items=earnings.Split(' ');
|
|
if(null==items||3!=items.Length)continue;
|
|
items[0]=items[0].Trim();
|
|
items[1]=items[1].Trim();
|
|
fundamental.NextEarningsDate=Utility.ParseDate(items[0]+" "+items[1]+" "+DateTime.Now.Year.ToString());
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
else if(header.Equals("Avg Volume"))
|
|
{
|
|
fundamental.Volume=(long)FeedParser.ParseValue(dataCollection[dataIndex+1].InnerText);
|
|
dataIndex++;
|
|
continue;
|
|
}
|
|
}
|
|
} // for all child nodes
|
|
fundamental.Equity=fundamental.BookValuePerShare*fundamental.SharesOutstanding;
|
|
|
|
IncomeStatement incomeStatement=null;
|
|
List<IncomeStatement> incomeStatements=GetIncomeStatement(symbol,IncomeStatement.PeriodType.Annual);
|
|
if(null!=incomeStatements&&0!=incomeStatements.Count) incomeStatement=incomeStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
|
|
CashflowStatement cashflowStatement=null;
|
|
List<CashflowStatement> cashflowStatements=GetCashflowStatement(symbol,CashflowStatement.PeriodType.Annual);
|
|
if(null != cashflowStatements && 0!=cashflowStatements.Count)
|
|
{
|
|
cashflowStatement = cashflowStatements.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
}
|
|
|
|
BalanceSheet balanceSheet=null;
|
|
List<BalanceSheet> balanceSheets=GetBalanceSheet(symbol,BalanceSheet.PeriodType.Annual);
|
|
if(null != balanceSheets && 0!=balanceSheets.Count)
|
|
{
|
|
balanceSheet = balanceSheets.OrderByDescending(x => x.AsOf).Take(1).FirstOrDefault();
|
|
}
|
|
|
|
if(null!=cashflowStatement)
|
|
{
|
|
fundamental.OperatingCashflow=cashflowStatement.OperatingCashflow;
|
|
fundamental.LeveragedFreeCashflow=cashflowStatement.FreeCashflow;
|
|
fundamental.EBITDA=fundamental.EBIT+cashflowStatement.DepreciationAndAmortization;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO CASHFLOW STATEMENT *************",symbol));
|
|
fundamental.OperatingCashflow=double.NaN;
|
|
fundamental.LeveragedFreeCashflow=double.NaN;
|
|
fundamental.EBITDA=double.NaN;
|
|
}
|
|
|
|
if(null!=incomeStatement)
|
|
{
|
|
fundamental.GrossProfit=incomeStatement.GrossProfit;
|
|
fundamental.Revenue=incomeStatement.TotalRevenue;
|
|
fundamental.EBIT=incomeStatement.EBIT;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO INCOME STATEMENT *************",symbol));
|
|
fundamental.GrossProfit=double.NaN;
|
|
fundamental.Revenue=double.NaN;
|
|
fundamental.EBIT=double.NaN;
|
|
}
|
|
|
|
if(null!=balanceSheet)
|
|
{
|
|
fundamental.TotalCash=balanceSheet.CashAndCashEquivalents;
|
|
fundamental.TotalDebt=balanceSheet.TotalLiabilities;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("************* GETFUNDAMENTALFINVIZ FOR {0} NO BALANCESHEET *************",symbol));
|
|
fundamental.TotalCash=double.NaN;
|
|
fundamental.TotalDebt=double.NaN;
|
|
}
|
|
|
|
if(!double.IsNaN(fundamental.MarketCap)&&!double.IsNaN(fundamental.TotalDebt)&&!double.IsNaN(fundamental.TotalCash)) fundamental.EnterpriseValue=fundamental.MarketCap+fundamental.TotalDebt-fundamental.TotalCash;
|
|
else fundamental.EnterpriseValue=double.NaN;
|
|
if(!Double.IsNaN(fundamental.SharesOutstanding)&&0!=fundamental.SharesOutstanding)fundamental.RevenuePerShare=fundamental.Revenue/fundamental.SharesOutstanding;
|
|
else fundamental.RevenuePerShare=double.NaN;
|
|
if(null!=balanceSheet&&!double.IsNaN(balanceSheet.TotalStockHolderEquity)&&!double.IsNaN(fundamental.TotalDebt)&&0!=fundamental.TotalDebt)fundamental.DebtToEquity=fundamental.TotalDebt/balanceSheet.TotalStockHolderEquity;
|
|
else fundamental.DebtToEquity=double.NaN;
|
|
if(double.IsNaN(fundamental.DebtToEquity)&&!double.IsNaN(fundamental.Equity)&&0.00!=fundamental.Equity)fundamental.DebtToEquity=fundamental.TotalDebt/fundamental.Equity;
|
|
return fundamental;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}.",exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();}
|
|
}
|
|
}
|
|
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ************************************************************************** F U N D A M E N T A L - Y A H O O F I N A N C E *********************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
public static Fundamental GetFundamental(String symbol)
|
|
{
|
|
Fundamental fundamental=GetFundamentalEx(symbol);
|
|
if(null==fundamental)return fundamental;
|
|
if(fundamental.GetLoad()<80)
|
|
{
|
|
Fundamental fundamental2 = GetFundamentalEx(symbol);
|
|
if(null!=fundamental2)fundamental.MergeFrom(fundamental2);
|
|
}
|
|
return fundamental;
|
|
}
|
|
private static Fundamental GetFundamentalEx(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
int TIMEOUT_MS=1000*30;
|
|
try
|
|
{
|
|
Fundamental fundamental = new Fundamental();
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("?p=").Append(symbol);
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,TIMEOUT_MS, webProxy); // user agents refreshed in this version
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
sb=new StringBuilder();
|
|
sb.Append("http://finance.yahoo.com/q?s=").Append(symbol).Append("&ql=1");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FAILED!, TRYING {1}",strRequest,sb.ToString()));
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} FINAL FAILURE.",strRequest));
|
|
return null;
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} SUCCESS.",strRequest));
|
|
fundamental.Symbol=symbol;
|
|
fundamental.AsOf=DateTime.Now;
|
|
fundamental.Source = "YAHOO";
|
|
fundamental.Beta=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Beta<"));
|
|
fundamental.NextEarningsDate=FeedParser.ParseValueDateTimeMonthFormat(Sections.LocateItemAcceptAnyText(httpNetResponse.ResponseString,">Earnings Date<"));
|
|
String strText=Sections.LocateFirstItem(httpNetResponse.ResponseString,">52 Week Range<");
|
|
if(null!=strText)
|
|
{
|
|
String[] strArray = strText.Split('-');
|
|
fundamental.Low52 = FeedParser.ParseValue(strArray[0].Trim());
|
|
fundamental.High52 = FeedParser.ParseValue(strArray[1].Trim());
|
|
}
|
|
fundamental.Volume=FeedParser.ParseValueLong(Sections.LocateItem(httpNetResponse.ResponseString,">Volume<"));
|
|
fundamental.MarketCap=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Market Cap<"));
|
|
fundamental.PE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PE Ratio (TTM)<"));
|
|
fundamental.EPS=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EPS (TTM)<"));
|
|
try { Thread.Sleep(250);PopulateKeyStatistics(fundamental,webProxy); }
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate key statistics.");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
|
|
}
|
|
try { Thread.Sleep(250);PopulateFinancials(fundamental); }
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"*********Unable to populate financials.");
|
|
MDTrace.WriteLine(LogLevel.DEBUG,"Exception was:"+exception.ToString());
|
|
}
|
|
if(fundamental.IsZero())
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("***** Fundamental for {0} is Invalid *****",symbol));
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}",fundamental.ToString()));
|
|
return null;
|
|
}
|
|
return fundamental;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// *************************************************************************************************************************************************************************************
|
|
// ***************************************************************** K E Y S T A T I S T I C S - Y A H O O F I N A N C E **********************************************************
|
|
// *************************************************************************************************************************************************************************************
|
|
private static bool PopulateKeyStatistics(Fundamental fundamental,WebProxy webProxy=null)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("http://finance.yahoo.com/q/ks?s=").Append(fundamental.Symbol);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
try{Thread.Sleep(250);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return false;
|
|
}
|
|
}
|
|
fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));
|
|
|
|
fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
|
|
if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<"));
|
|
|
|
fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
|
|
if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets (ttm)<"));
|
|
|
|
fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
|
|
if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity (ttm)<"));
|
|
|
|
fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
|
|
if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share (ttm)<"));
|
|
|
|
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
|
|
if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash (mrq)<"));
|
|
|
|
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
|
|
if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt (mrq)<"));
|
|
|
|
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
|
|
if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding <sup>6</sup> "));
|
|
|
|
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
|
|
if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue (ttm)<"));
|
|
|
|
fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
|
|
if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth (yoy)<"));
|
|
|
|
fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
|
|
if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit (ttm)<"));
|
|
|
|
fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
|
|
if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA <"));
|
|
|
|
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
|
|
if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common (ttm)<"));
|
|
|
|
fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
|
|
if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share (mrq)<"));
|
|
|
|
fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
|
|
if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow (ttm)<"));
|
|
|
|
fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
|
|
if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow (ttm)<"));
|
|
|
|
fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));
|
|
|
|
return true;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
// *************************************************************************************************************************************************************************************
|
|
// ***************************************************************** F I N A N C I A L S - Y A H O O F I N A N C E *****************************************************************
|
|
// *************************************************************************************************************************************************************************************
|
|
private static bool PopulateFinancials(Fundamental fundamental)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(fundamental.Symbol).Append("/financials?p=").Append(fundamental.Symbol);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
try{Thread.Sleep(250);}catch(Exception){;}
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return false;
|
|
}
|
|
}
|
|
fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
|
|
if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00;
|
|
|
|
return true;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return false;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
//**************************************************************************************************************************************************************************************
|
|
//*************************************************************************** I N T R A D A Y P R I C I N G **************************************************************************
|
|
// ************************************************************** M A I N : Y A H O O **************************************************************************************************
|
|
// ************************************************************** B A C K U P : B I G C H A R T S **************************************************************************************
|
|
//**************************************************************************************************************************************************************************************
|
|
private delegate Price LatestPriceDelegate(String symbol);
|
|
|
|
public static Price GetLatestPrice(String symbol)
|
|
{
|
|
if(null==symbol)return null;
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if(null!=companyProfile && companyProfile.FreezePricing)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
|
|
return null;
|
|
}
|
|
DateTime currentMarketDate=PremarketDA.GetLatestMarketDate();
|
|
LatestPriceDelegate[] latestPriceDelegates=new LatestPriceDelegate[]{GetLatestPriceRobinhood,GetLatestPriceBarChart,GetLatestPriceYahoo,GetLatestPriceGoogle};
|
|
Price latestPrice=null;
|
|
foreach(LatestPriceDelegate latestPriceDelegate in latestPriceDelegates)
|
|
{
|
|
latestPrice=latestPriceDelegate.Invoke(symbol);
|
|
if(null!=latestPrice && latestPrice.IsValid)
|
|
{
|
|
if(latestPrice.Date.Date.Equals(currentMarketDate.Date.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Price request from {0} succeeded for {1}",latestPriceDelegate.GetMethodInfo().Name,symbol));
|
|
return latestPrice;
|
|
}
|
|
else
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The date retrieved for {0} via {1} does not match the current market date. Market Date: {2} Price Date: {3}",symbol,latestPriceDelegate.Method.Name,currentMarketDate.ToShortDateString(),latestPrice.Date.ToShortDateString()));
|
|
}
|
|
}
|
|
}
|
|
if(null!=companyProfile && companyProfile.CanRollPrevious)
|
|
{
|
|
latestPrice=PricingMarketDataHelper.RollPriceForward(symbol);
|
|
if(null!=latestPrice)return latestPrice;
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol));
|
|
return null;
|
|
}
|
|
// ***************************************************************************************************************
|
|
// ******************** L A T E S T P R I C E R E T R I E V A L F O R I N T R A - D A Y F E E D ********
|
|
// ***************************************************************************************************************
|
|
public static Price GetLatestPriceYahoo(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
StringBuilder sb=new StringBuilder();
|
|
String strRequest;
|
|
if(null==symbol)return null;
|
|
sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol);
|
|
strRequest=sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
Price price=new Price();
|
|
price.Symbol=symbol.ToUpper();
|
|
price.Date=DateTime.Now;
|
|
price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN;
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
return BuildPriceFromResponse(httpNetResponse.ResponseString,price);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
private static Price BuildPriceFromResponse(String httpNetResponse,Price price)
|
|
{
|
|
if(httpNetResponse.Contains("HTML_TAG_END"))
|
|
{
|
|
price = BuildPriceFromSections(httpNetResponse, price);
|
|
}
|
|
else
|
|
{
|
|
price = BuildPriceFromTable(httpNetResponse, price);
|
|
}
|
|
return price;
|
|
}
|
|
|
|
private static Price BuildPriceFromSections(String httpNetResponse,Price price)
|
|
{
|
|
try
|
|
{
|
|
String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END");
|
|
List<String> sections = null;
|
|
|
|
sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
|
|
|
|
String marker ="data-symbol=\""+price.Symbol+"\"";
|
|
List<String> dataSections = sections.Where(x => x.Contains(marker)).ToList();
|
|
dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
|
|
if(dataSections.Count>0)
|
|
{
|
|
dataSections = Sections.GetSections(dataSections[0]);
|
|
if(null!=dataSections && dataSections.Count==1)
|
|
{
|
|
price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[0]);
|
|
}
|
|
else if(null!=dataSections && dataSections.Count>0)
|
|
{
|
|
price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]);
|
|
}
|
|
}
|
|
|
|
marker = "<fin-streamer data-symbol=\""+price.Symbol+"\"";
|
|
sections = sections.Where(x => x.StartsWith(marker)).ToList();
|
|
for(int index=0;index<sections.Count;index++)
|
|
{
|
|
String sectionText = sections[index];
|
|
String itemValue = null;
|
|
|
|
if(sectionText.Contains("regularMarketPreviousClose"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
price.PrevClose=FeedParser.ParseValue(itemValue);
|
|
}
|
|
else if(sectionText.Contains("regularMarketOpen"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
price.Open=FeedParser.ParseValue(itemValue);
|
|
}
|
|
else if(sectionText.Contains("regularMarketDayRange"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
String[] subItems = itemValue.Split('-');
|
|
price.Low=FeedParser.ParseValue(subItems[0]);
|
|
price.High=FeedParser.ParseValue(subItems[1]);
|
|
if(double.IsNaN(price.Close))
|
|
{
|
|
price.Close=price.AdjClose=(price.Low+price.High)/2.00;
|
|
}
|
|
}
|
|
else if(sectionText.Contains("fiftyTwoWeekRange"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
}
|
|
else if(sectionText.Contains("regularMarketVolume"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
price.Volume=FeedParser.ParseValueLong(itemValue);
|
|
}
|
|
else if(sectionText.Contains("averageVolume"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
}
|
|
else if(sectionText.Contains("averageVolume"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
}
|
|
else if(sectionText.Contains("marketCap"))
|
|
{
|
|
itemValue = Sections.GetSections(sections[index])[0];
|
|
}
|
|
}
|
|
CheckPrice(price);
|
|
return price;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: for {0} -> {1}",price.Symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static Price BuildPriceFromTable(String httpNetResponse,Price price)
|
|
{
|
|
MemoryStream memoryStream = null;
|
|
|
|
try
|
|
{
|
|
List<String> sections = null;
|
|
double bid = double.NaN;
|
|
double bidContracts = double.NaN;
|
|
double ask = double.NaN;
|
|
double askContracts = double.NaN;
|
|
|
|
String symbol=price.Symbol.ToUpper();
|
|
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse);
|
|
memoryStream = new MemoryStream(streamBytes);
|
|
HtmlDocument htmlDocument = new HtmlDocument();
|
|
htmlDocument.Load(memoryStream);
|
|
|
|
HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]");
|
|
if(null == table || 0 == table.Count)return null;
|
|
HtmlNodeCollection rows = table[0].SelectNodes(".//tr");
|
|
if (null == rows || 0 == rows.Count) return null;
|
|
for(int index=0;index< rows.Count; index++)
|
|
{
|
|
int itemIndex=0;
|
|
String itemValue=null;
|
|
String innerHtml = rows[index].InnerHtml;
|
|
sections=Sections.GetSections(innerHtml);
|
|
|
|
if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
}
|
|
else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
price.Open=FeedParser.ParseValue(itemValue);
|
|
}
|
|
else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
if(String.IsNullOrEmpty(itemValue))continue;
|
|
String[] bidContractSplit = itemValue.Split('x');
|
|
bid=FeedParser.ParseValue(bidContractSplit[0]);
|
|
bidContracts=FeedParser.ParseValue(bidContractSplit[1]);
|
|
}
|
|
else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
if(String.IsNullOrEmpty(itemValue))continue;
|
|
String[] askContractSplit = itemValue.Split('x');
|
|
ask = FeedParser.ParseValue(askContractSplit[0]);
|
|
askContracts = FeedParser.ParseValue(askContractSplit[1]);
|
|
}
|
|
else if(Sections.FindInSections(sections, "Day's Range",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
String[] subItems=itemValue.Split('-');
|
|
price.Low=FeedParser.ParseValue(subItems[0]);
|
|
price.High=FeedParser.ParseValue(subItems[1]);
|
|
}
|
|
else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
}
|
|
else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+4];
|
|
price.Volume=FeedParser.ParseValueLong(itemValue);
|
|
}
|
|
else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true))
|
|
{
|
|
itemValue = sections[itemIndex+3];
|
|
}
|
|
}
|
|
|
|
if (double.IsNaN(price.Close))
|
|
{
|
|
sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer");
|
|
String marker = "data-symbol=\"" + price.Symbol + "\"";
|
|
sections = sections.Where(x => x.Contains(marker)).ToList();
|
|
sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList();
|
|
if (sections.Count > 0)
|
|
{
|
|
sections = Sections.GetSections(sections[0]);
|
|
if (null != sections && sections.Count > 0)
|
|
{
|
|
price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]);
|
|
}
|
|
}
|
|
}
|
|
|
|
if(double.IsNaN(price.Close))
|
|
{
|
|
double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts);
|
|
if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice;
|
|
if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00;
|
|
}
|
|
|
|
CheckPrice(price);
|
|
return price;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: for {0} -> {1}",price.Symbol,exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=memoryStream)
|
|
{
|
|
memoryStream.Close();
|
|
memoryStream.Dispose();
|
|
memoryStream=null;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts)
|
|
{
|
|
if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN;
|
|
double totalContracts = bidContracts + askContracts;
|
|
double bidWeight = bidContracts/totalContracts;
|
|
double askWeight = askContracts/totalContracts;
|
|
double weightedBid = bid*bidWeight;
|
|
double weightedAsk = ask*askWeight;
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2)));
|
|
return weightedBid + weightedAsk;
|
|
}
|
|
|
|
public static void CheckPrice(Price price)
|
|
{
|
|
if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low))
|
|
{
|
|
price.Close=price.AdjClose=(price.High+price.Low)/2.00;
|
|
}
|
|
if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low)
|
|
{ // if the close price is garbage set it to the midpoint of the high and the low
|
|
if(price.Close<price.Low||price.Close>price.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00;
|
|
}
|
|
if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close;
|
|
if(double.IsNaN(price.Open))price.Open=0.00;
|
|
if(double.IsNaN(price.Low))price.Low=0.00;
|
|
if(double.IsNaN(price.High))price.High=0.00;
|
|
if(0==price.High && !double.IsNaN(price.Close) && 0!=price.Close)
|
|
{
|
|
price.High=price.Close;
|
|
}
|
|
if(0==price.Low && !double.IsNaN(price.Close) && 0!=price.Close)
|
|
{
|
|
price.Low=price.Close;
|
|
}
|
|
if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low)
|
|
{
|
|
price.Open=price.High=price.Low=price.Close;
|
|
}
|
|
if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low))
|
|
{
|
|
price.Open=(price.High+price.Low)/2.00;
|
|
}
|
|
}
|
|
|
|
public static Price GetLatestPriceGoogle(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
DateTime pricingDate = dateGenerator.GetPrevBusinessDay(DateTime.Now);
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
if(null==symbol) return null;
|
|
String exchangeNASDAQ="NASDAQ";
|
|
String exchangeNYSE="NYSE";
|
|
|
|
sb = new StringBuilder();
|
|
sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNASDAQ);
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceGoogle");
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
List<String> divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div");
|
|
if(null==divSections || 0==divSections.Count)return null;
|
|
List<String> searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList();
|
|
if(null!=searchPattern && searchPattern.Count>0)
|
|
{
|
|
httpNetResponse.Dispose();
|
|
sb = new StringBuilder();
|
|
sb.Append("https://www.google.com/finance/quote/").Append(symbol).Append(":").Append(exchangeNYSE);
|
|
strRequest = sb.ToString();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
|
|
if(!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
divSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"div");
|
|
searchPattern = divSections.Where(x => x.Contains("We couldn't find any match for your search.")).ToList();
|
|
if(null!=searchPattern && searchPattern.Count>0)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
List<String> sections = Sections.GetSections(httpNetResponse.ResponseString);
|
|
sections = Sections.FlattenSection(sections);
|
|
|
|
int index=0;
|
|
|
|
Price price = new Price();
|
|
price.Symbol = symbol.ToUpper();
|
|
price.Source=Price.PriceSource.Google;
|
|
price.Close=price.Open=price.High=price.Low=price.AdjClose=double.NaN;
|
|
price.Open=0.00;
|
|
price.Date = pricingDate;
|
|
|
|
if(Sections.FindInSections(sections, "After Hours:", 0,ref index, true))
|
|
{
|
|
price.Close=price.AdjClose=FeedParser.ParseValue(sections[index-1]);
|
|
}
|
|
else if(Sections.FindInSections(sections, "Share", 0,ref index, true))
|
|
{
|
|
price.Close=price.AdjClose=FeedParser.ParseValue(sections[index+1]);
|
|
}
|
|
|
|
if(Sections.FindInSections(sections, "Previous close", 0,ref index, true))
|
|
{
|
|
price.Open=FeedParser.ParseValue(sections[index+2]);
|
|
}
|
|
|
|
if(Sections.FindInSections(sections, "Day range", 0,ref index, true))
|
|
{
|
|
String itemValue = sections[index+2];
|
|
String[] highLow = itemValue.Split('-');
|
|
price.Low=FeedParser.ParseValue(highLow[0]);
|
|
price.High=FeedParser.ParseValue(highLow[1]);
|
|
}
|
|
if(!price.IsValid)return null;
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// GetLatestPriceBarChart - This is a new feed I am implementing because I am having difficulty retrieving prices from MarketWatch(BigCharts)
|
|
/// I am planning to use this field initially as a sweep to run after the BigCharts feed and prior to the Yahoo Sweep
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <returns></returns>
|
|
public static Price GetLatestPriceBarChart(String symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
try
|
|
{
|
|
DateGenerator dateGenerator = new DateGenerator();
|
|
DateTime currentPricingDate = dateGenerator.GetPrevBusinessDay(DateTime.Now);
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
if (null == symbol) return null;
|
|
|
|
sb = new StringBuilder();
|
|
sb.Append("https://www.barchart.com/stocks/quotes/").Append(symbol).Append("/overview");
|
|
strRequest = sb.ToString();
|
|
WebProxy webProxy = HttpNetRequest.GetProxy("GetLatestPriceBarChart");
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3C(strRequest, webProxy);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetLatestPriceBarChart: Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
String beginMarker = "data-ng-init='init(\"" + symbol + "\"";
|
|
String endMarker = ">";
|
|
String item = Utility.BetweenString(httpNetResponse.ResponseString, beginMarker, endMarker);
|
|
if (null == item) return null;
|
|
item = item.Replace(""", "\"");
|
|
item = item.Replace("{", null);
|
|
item = item.Replace("}", null);
|
|
String[] pairs = item.Split(",");
|
|
Dictionary<String, double> values = CreateBarChartValues(pairs);
|
|
Price price = new Price();
|
|
price.Symbol = symbol;
|
|
price.Date = currentPricingDate;
|
|
price.Source = Price.PriceSource.BarChart;
|
|
if (values.ContainsKey("lowPrice")) price.Low = values["lowPrice"];
|
|
if (values.ContainsKey("highPrice")) price.High = values["highPrice"];
|
|
if (values.ContainsKey("openPrice")) price.Open = values["openPrice"];
|
|
if (values.ContainsKey("lastPrice")) price.Close = values["lastPrice"];
|
|
if (values.ContainsKey("volume")) price.Volume = (long)values["volume"];
|
|
if (values.ContainsKey("prevClose")) price.PrevClose = values["prevClose"];
|
|
else price.PrevClose = price.Close;
|
|
price.AdjClose = price.Close;
|
|
MarketDataHelper.CheckPrice(price);
|
|
if (double.IsNaN(price.Open) && double.IsNaN(price.High) && double.IsNaN(price.Low) && double.IsNaN(price.Close))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("*** No closing price for {0}", price.Symbol));
|
|
return null;
|
|
}
|
|
if (double.IsNaN(price.Open)) price.Open = 0.00;
|
|
if (double.IsNaN(price.Low)) price.Low = 0.00;
|
|
if (double.IsNaN(price.High)) price.High = 0.00;
|
|
if (0 != price.Close && 0 == price.Open && 0 == price.High && 0 == price.Low)
|
|
{
|
|
price.Open = price.High = price.Low = price.Close;
|
|
}
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, exception.ToString());
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse)
|
|
{
|
|
httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Dictionary<String, double> CreateBarChartValues(String[] pairs)
|
|
{
|
|
Dictionary<String, double> barChartValues = new Dictionary<String, double>();
|
|
if (null == pairs || 0 == pairs.Length) return barChartValues;
|
|
foreach (String item in pairs)
|
|
{
|
|
String[] nameValue = item.Split(":");
|
|
if (null == nameValue || 2 != nameValue.Length) continue;
|
|
String name = nameValue[0].Replace("\"", null);
|
|
String value = nameValue[1].Replace("\"",null);
|
|
double result = 0.00;
|
|
if (double.TryParse(value, out result))
|
|
{
|
|
if (barChartValues.ContainsKey(name))
|
|
{
|
|
barChartValues[name] = result;
|
|
}
|
|
else
|
|
{
|
|
barChartValues.Add(name, result);
|
|
}
|
|
}
|
|
}
|
|
return barChartValues;
|
|
}
|
|
|
|
// *******************************************************************************************************************************************************************************
|
|
// ************************************************************** L A T E S T P R I C E - R O B I N H O O D ******************************************************************
|
|
// *******************************************************************************************************************************************************************************
|
|
/// <summary>
|
|
/// This version fetches the latest price from robinhood and ensures that the pricing date is what we expect
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <param name="startDate"></param>
|
|
/// <returns></returns>
|
|
public static Price GetPriceRobinHood(String symbol, DateTime startDate)
|
|
{
|
|
try
|
|
{
|
|
Price price = GetLatestPriceRobinhood(symbol);
|
|
if (null == price) return null;
|
|
if (!price.Date.Date.Equals(startDate.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Error, the price retrieved from Robinhood for : {symbol} is dated {price.Date.Date.ToShortDateString()}, expected {startDate.Date.ToShortDateString()}.");
|
|
return null;
|
|
}
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// This version fetches the latest price from Robinhood
|
|
/// </summary>
|
|
/// <param name="symbol"></param>
|
|
/// <returns></returns>
|
|
public static Price GetLatestPriceRobinhood(string symbol)
|
|
{
|
|
HttpNetResponse httpNetResponse = null;
|
|
try
|
|
{
|
|
if (null == symbol) return null;
|
|
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if (null != companyProfile && companyProfile.FreezePricing)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
|
|
return null;
|
|
}
|
|
|
|
StringBuilder sb = new StringBuilder();
|
|
String requestSymbol = symbol;
|
|
String strRequest = default;
|
|
if (requestSymbol.StartsWith("^")) requestSymbol = requestSymbol.Substring(1);
|
|
sb = new StringBuilder();
|
|
sb.Append("https://robinhood.com/us/en/stocks/").Append(symbol).Append("/");
|
|
strRequest = sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"{strRequest}");
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
String strInstrumentId = Utility.BetweenString(httpNetResponse.ResponseString, "<meta content=\"robinhood://instrument?id=", "\"");
|
|
if (null == strInstrumentId) return null;
|
|
|
|
List<String> spanSections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString, "span");
|
|
|
|
String strOpen = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">Open price</span>", 1);
|
|
if (null == strOpen) return null;
|
|
strOpen = Utility.BetweenString(strOpen, ">", "<");
|
|
if (null == strOpen) return null;
|
|
if (strOpen.Equals("???")) strOpen = null;
|
|
|
|
String strHigh = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">High today</span>", 1);
|
|
if (null == strHigh) return null;
|
|
strHigh = Utility.BetweenString(strHigh, ">", "<");
|
|
if (null == strHigh) return null;
|
|
if (strHigh.Equals("???")) strHigh = null;
|
|
|
|
String strLow = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-eimp3v\">Low today</span>", 1);
|
|
if (null == strLow) return null;
|
|
strLow = Utility.BetweenString(strLow, ">", "<");
|
|
if (null == strLow) return null;
|
|
if (strLow.Equals("???")) strLow = null;
|
|
|
|
String strVolume = Sections.ScanSectionsFindStartWithReturnIndexAfter(spanSections, "<span class=\"css-v72tci\">Volume</span>", 1);
|
|
if (null == strVolume) return null;
|
|
strVolume = Utility.BetweenString(strVolume, ">", "<");
|
|
if (null == strVolume) return null;
|
|
if (strVolume.Equals("???")) strVolume = null;
|
|
|
|
if (null == strOpen && null == strHigh && null == strLow)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// Fetch the current price
|
|
httpNetResponse.Dispose();
|
|
sb = new StringBuilder();
|
|
sb.Append("https://bonfire.robinhood.com/instruments/").Append(strInstrumentId).Append("/detail-page-live-updating-data/?display_span=day&hide_extended_hours=false");
|
|
strRequest = sb.ToString();
|
|
httpNetResponse = HttpNetRequest.GetRequestNoEncoding(strRequest);
|
|
if (!httpNetResponse.Success)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Request:{0} failed with status {1}", httpNetResponse.Request, httpNetResponse.StatusCode));
|
|
return null;
|
|
}
|
|
|
|
JsonDocument jsonDocument = JsonDocument.Parse(httpNetResponse.ResponseString);
|
|
JsonElement root = jsonDocument.RootElement;
|
|
JsonElement chartSection = root.GetProperty("chart_section");
|
|
JsonElement quoteSection = chartSection.GetProperty("quote");
|
|
String strLastTradePrice = quoteSection.GetProperty("last_trade_price").GetString();
|
|
String strUpdatedAt = quoteSection.GetProperty("updated_at").GetString();
|
|
String strSymbol = quoteSection.GetProperty("symbol").GetString();
|
|
|
|
DateTime lastUpdatedAt = DateTime.Parse(strUpdatedAt, null, System.Globalization.DateTimeStyles.RoundtripKind);
|
|
Price price = new Price();
|
|
price.Symbol = strSymbol;
|
|
price.Date = lastUpdatedAt;
|
|
price.Source = Price.PriceSource.Robinhood;
|
|
price.Close = price.AdjClose = FeedParser.ParseValueDouble(strLastTradePrice);
|
|
price.Open = Utility.ParseCurrency(strOpen);
|
|
price.High = Utility.ParseCurrency(strHigh);
|
|
price.Low = Utility.ParseCurrency(strLow);
|
|
price.Volume = (long)FeedParser.ParseValueDouble(strVolume);
|
|
CheckPrice(price);
|
|
if (!price.IsValid) return null;
|
|
if (!price.Symbol.Equals(symbol)) return null;
|
|
MDTrace.WriteLine(LogLevel.DEBUG, $"Got price from Robinhood for {symbol} : {price.ToString()}");
|
|
return price;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Exception : {0}", exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if (null != httpNetResponse) httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
|
|
//********************************************************************************************************************************************************************************************************
|
|
// ******************************************************************************** H I S T O R I C A L P R I C I N G Y A H O O *********************************************************************
|
|
//********************************************************************************************************************************************************************************************************
|
|
|
|
public static Price GetDailyPrice(String symbol, DateTime pricingDate)
|
|
{
|
|
if (null == symbol) return null;
|
|
CompanyProfile companyProfile = CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if (null != companyProfile && companyProfile.FreezePricing)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("Pricing for {0} is frozen.", symbol));
|
|
return null;
|
|
}
|
|
Prices prices = GetDailyPrices(symbol, pricingDate, pricingDate);
|
|
if (null == prices || 0 == prices.Count) return null;
|
|
Price price = prices.FirstOrDefault();
|
|
if (!price.Date.Date.Equals(pricingDate.Date))
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetDailyPrice: The pricing date returned for '{0}' was different than the date requested. Date requested '{1}', date returned '{2}'", symbol, pricingDate.ToShortDateString(), price.Date.Date.ToShortDateString()));
|
|
return null;
|
|
}
|
|
return price;
|
|
}
|
|
|
|
public static Prices GetDailyPrices(String symbol, DateTime startDate, DateTime endDate)
|
|
{
|
|
HttpNetResponse httpNetResponse=null;
|
|
try
|
|
{
|
|
if(symbol==null)return null;
|
|
CompanyProfile companyProfile=CompanyProfileDA.GetCompanyProfile(symbol);
|
|
if(null!=companyProfile && companyProfile.FreezePricing)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Pricing for {0} is frozen.",symbol));
|
|
return null;
|
|
}
|
|
startDate=startDate.Date;
|
|
endDate=endDate.Date;
|
|
if(startDate>endDate)
|
|
{
|
|
DateTime tempDate=startDate;
|
|
startDate=endDate;
|
|
endDate=tempDate;
|
|
}
|
|
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[MarketDataHelper:GetDailyPrices]{0} start:{1} end:{2}",symbol,Utility.DateTimeToStringMMHDDHYYYY(startDate),Utility.DateTimeToStringMMHDDHYYYY(endDate)));
|
|
StringBuilder sb = new StringBuilder();
|
|
String strRequest;
|
|
Prices prices=null;
|
|
sb.Append("https://query1.finance.yahoo.com/v7/finance/chart/").Append(symbol).Append("?period1=").Append(Utility.DateToUnixDate(startDate)).Append("&period2=").Append(Utility.DateToUnixDate(endDate)).Append("&interval=1d&events=history");
|
|
strRequest=sb.ToString();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Query:{0}",strRequest));
|
|
WebProxy webProxy=HttpNetRequest.GetProxy("GetDailyPrices");
|
|
CookieCollection cookieCollection=new CookieCollection();
|
|
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3B(strRequest,cookieCollection,webProxy);
|
|
if(!httpNetResponse.Success) return null;
|
|
JObject json=JObject.Parse(httpNetResponse.ResponseString);
|
|
prices=GetPricesFromJSONString(json,symbol);
|
|
return prices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
if(null!=httpNetResponse)httpNetResponse.Dispose();
|
|
}
|
|
}
|
|
private static Prices GetPricesFromJSONString(JObject json,String symbol)
|
|
{
|
|
Prices prices=new Prices();
|
|
|
|
try
|
|
{
|
|
JToken token=null;
|
|
int items=0;
|
|
|
|
try{token=json["chart"]["result"][0]["timestamp"];}catch(Exception){;}
|
|
if(null==token)
|
|
{
|
|
string instrumentType=null;
|
|
try{instrumentType=json["chart"]["result"][0]["meta"]["instrumentType"].ToString();}catch(Exception){;}
|
|
if(null==instrumentType||!instrumentType.Equals("MUTUALFUND"))return prices;
|
|
String strPrice=null;
|
|
String strDate=null;
|
|
try{strPrice=json["chart"]["result"][0]["meta"]["regularMarketPrice"].ToString();}catch(Exception){;}
|
|
try{strDate=json["chart"]["result"][0]["meta"]["currentTradingPeriod"]["regular"]["start"].ToString();}catch(Exception){;}
|
|
if(null==strPrice||null==strDate)return prices;
|
|
Price price=new Price();
|
|
price.Symbol=symbol;
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
price.High=price.Low=price.Close=price.Open=price.AdjClose=FeedParser.ParseValue(strPrice);
|
|
DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strDate));
|
|
price.Date=dateTimeOffset.DateTime;
|
|
price.Volume=0L;
|
|
prices.Add(price);
|
|
return prices;
|
|
}
|
|
try{items=token.Children().Count();}catch(Exception){;}
|
|
if(0==items)return prices;
|
|
for(int index=0;index<items;index++)
|
|
{
|
|
String strTradeDate=(json["chart"]["result"][0]["timestamp"][index]).ToString();
|
|
String strHigh=null;
|
|
String strOpen=null;
|
|
String strLow=null;
|
|
String strClose=null;
|
|
String strVolume=null;
|
|
String strAdjClose=null;
|
|
|
|
try{strHigh=(json["chart"]["result"][0]["indicators"]["quote"][0]["high"][index]).ToString();}catch(Exception){;}
|
|
try{strOpen=(json["chart"]["result"][0]["indicators"]["quote"][0]["open"][index]).ToString();}catch(Exception){;}
|
|
try{strLow=(json["chart"]["result"][0]["indicators"]["quote"][0]["low"][index]).ToString();}catch(Exception){;}
|
|
try{strClose=(json["chart"]["result"][0]["indicators"]["quote"][0]["close"][index]).ToString();}catch(Exception){;}
|
|
try{strVolume=(json["chart"]["result"][0]["indicators"]["quote"][0]["volume"][index]).ToString();}catch(Exception){;}
|
|
try{strAdjClose=(json["chart"]["result"][0]["indicators"]["adjclose"][0]["adjclose"][index]).ToString();}catch(Exception){;}
|
|
|
|
DateTimeOffset dateTimeOffset=DateTimeOffset.FromUnixTimeSeconds(long.Parse(strTradeDate));
|
|
Price price=new Price();
|
|
price.Source=Price.PriceSource.Yahoo;
|
|
price.Symbol=symbol;
|
|
price.Date=dateTimeOffset.DateTime;
|
|
price.High=FeedParser.ParseValue(strHigh);
|
|
price.Open=FeedParser.ParseValue(strOpen);
|
|
price.Close=FeedParser.ParseValue(strClose);
|
|
price.Low=FeedParser.ParseValue(strLow);
|
|
price.Volume=(long)FeedParser.ParseValue(strVolume);
|
|
price.AdjClose=FeedParser.ParseValue(strAdjClose);
|
|
|
|
if(price.Open<=0.05&&price.High<=0.05&&price.Low<=0.05&&price.Close<=0.05&&price.AdjClose<=0.05)continue;
|
|
if(double.IsNaN(price.Close)||price.Close==0.00)continue;
|
|
if(price.Volume<0)continue;
|
|
if(price.Open==0.00||double.IsNaN(price.Open))price.Open=price.Close;
|
|
if(price.High==0.00||double.IsNaN(price.High))price.High=price.Close;
|
|
if(price.Low==0.00||double.IsNaN(price.Low))price.Low=price.Close;
|
|
if(price.Close<price.Low||price.Close>price.High)price.Close=(price.High+price.Low)/2.00;
|
|
|
|
prices.Add(price);
|
|
|
|
}
|
|
return prices;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception:{0}",exception.ToString()));
|
|
return prices;
|
|
}
|
|
}
|
|
|
|
// ******************************************************************************************************************************************************************************************************************
|
|
// *************************************************************************** E N D Y A H O O H I S T O R I C A L P R I C I N G *******************************************************************************
|
|
// ******************************************************************************************************************************************************************************************************************
|
|
|
|
|
|
// ****************************************************************************************************************************************************************************************
|
|
// **************************************************************************** E N D D A T A F E E D S *********************************************************************************
|
|
// ****************************************************************************************************************************************************************************************
|
|
|
|
|
|
|
|
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
// ***************************************************************************************************************************************************************************************
|
|
|
|
// 2017 Yahoo Finance Modified Format of all feeds. Most of the HTMLDocument parser code is broken because the response strings are no longer in tabular format.
|
|
// It's difficult to make sense of the new response string outside of noting that it remains in aa tag format. The following code is an attempt to make sense of it all.
|
|
public static List<String> LocateJSONItems(String strInput,String item,bool isHeading=false)
|
|
{
|
|
try
|
|
{
|
|
if(null==item||null==strInput)return null;
|
|
int startPos=strInput.IndexOf(item);
|
|
if(-1==startPos)return null;
|
|
strInput=strInput.Substring(startPos-1);
|
|
if(isHeading)
|
|
{
|
|
strInput=strInput.Substring(item.Length);
|
|
startPos=strInput.IndexOf(item);
|
|
if(-1==startPos)return null;
|
|
strInput=strInput.Substring(startPos-1);
|
|
}
|
|
strInput=Utility.BetweenString(strInput,"[","]");
|
|
if(null==strInput)return null;
|
|
String[] itemsArray=strInput.Split(',');
|
|
return itemsArray.ToList();
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<KeyValue> LocateJSONKeyValuePairs(String strInput, String item1,String item2)
|
|
{
|
|
try
|
|
{
|
|
List<KeyValue> items=new List<KeyValue>();
|
|
if (null == item1 || null == strInput) return null;
|
|
int startPos = strInput.IndexOf(item1);
|
|
while(true)
|
|
{
|
|
KeyValue keyValue=new KeyValue();
|
|
if (-1 == startPos||null==strInput) break;
|
|
String strItem = strInput.Substring(startPos);
|
|
strItem = Utility.BetweenString(strItem, "\"", "\"");
|
|
String strValue = strInput.Substring((startPos+strItem.Length+4)-1);
|
|
strValue = Utility.BetweenString(strValue, "\"", "\"");
|
|
keyValue.Key=strValue;
|
|
try{strInput=strInput.Substring((startPos+strItem.Length+4+strValue.Length+4)-1);}
|
|
catch(Exception){break;}
|
|
if(null==strInput)break;
|
|
startPos = strInput.IndexOf(item2);
|
|
if(-1==startPos)break;
|
|
strItem = strInput.Substring(startPos);
|
|
strItem = Utility.BetweenString(strItem, "\"", "\"");
|
|
strValue = strInput.Substring((startPos + strItem.Length + 4) - 1);
|
|
strValue = Utility.BetweenString(strValue, "\"", "\"");
|
|
keyValue.Value=strValue;
|
|
try { strInput = strInput.Substring((startPos + strValue.Length + 4) - 1); }
|
|
catch (Exception) { break; }
|
|
if(null==strInput)break;
|
|
startPos = strInput.IndexOf(item1);
|
|
items.Add(keyValue);
|
|
}
|
|
return items;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<List<String>> LocateJSONItems(String strInput)
|
|
{
|
|
List<List<String>> items=new List<List<String>>();
|
|
try
|
|
{
|
|
if(null==strInput)return null;
|
|
strInput=Utility.BetweenString(strInput,"[","]");
|
|
if(null==strInput)return null;
|
|
String[] itemsArray=strInput.Split(',');
|
|
if(null==itemsArray||0==itemsArray.Length)return null;
|
|
for(int index=0;index<itemsArray.Length;index++)
|
|
{
|
|
String item=itemsArray[index];
|
|
item=item.Replace("{",null);
|
|
item=item.Replace("}",null);
|
|
item=item.Replace("\"",null);
|
|
String[] subItemArray=item.Split(':');
|
|
if(null==subItemArray||2!=subItemArray.Length)continue;
|
|
if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
|
|
if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
|
|
items.Add(subItemArray.ToList());
|
|
}
|
|
return items;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<List<String>> LocateJSONItemsTags(String strInput)
|
|
{
|
|
List<List<String>> items=new List<List<String>>();
|
|
try
|
|
{
|
|
if(null==strInput)return null;
|
|
String[] itemsArray=strInput.Split(',');
|
|
if(null==itemsArray||0==itemsArray.Length)return null;
|
|
for(int index=0;index<itemsArray.Length;index++)
|
|
{
|
|
String item=itemsArray[index];
|
|
item=item.Replace("{",null);
|
|
item=item.Replace("}",null);
|
|
item=item.Replace("\"",null);
|
|
String[] subItemArray=item.Split(':');
|
|
if(null==subItemArray||2!=subItemArray.Length)continue;
|
|
if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
|
|
if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
|
|
items.Add(subItemArray.ToList());
|
|
}
|
|
return items;
|
|
}
|
|
catch(Exception)
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static List<String> FindItemsInTag(String item, List<List<String>> itemTags)
|
|
{
|
|
List<String> foundItems = new List<String>();
|
|
|
|
if (null == itemTags || 0 == itemTags.Count) return null;
|
|
for (int index = 0; index < itemTags.Count; index++)
|
|
{
|
|
List<String> itemTag = itemTags[index];
|
|
if (itemTag[0].Equals(item)) foundItems.Add(itemTag[1]);
|
|
}
|
|
return foundItems;
|
|
}
|
|
|
|
|
|
private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
|
|
{
|
|
DateTime referenceDate = DateTime.Now;
|
|
string[] items = relativeDate.Split(' ');
|
|
if (items.Length < 2) return Utility.Epoch;
|
|
if ("days".Equals(items[1]) || "day".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0, 0, 0);
|
|
referenceDate -= timeSpan;
|
|
}
|
|
else if ("hours".Equals(items[1]) || "hour".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan = new TimeSpan(int.Parse(items[0]), 0, 0);
|
|
referenceDate -= timeSpan;
|
|
}
|
|
else if ("mins".Equals(items[1]) || "minutes".Equals(items[1]))
|
|
{
|
|
TimeSpan timeSpan = new TimeSpan(0, int.Parse(items[0]), 0);
|
|
referenceDate -= timeSpan;
|
|
}
|
|
else if ((new string[] { "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", "Jan", "Feb", "Mar", "Apr", "May", "Jun" }).Any(x => x.Equals(items[0])))
|
|
{
|
|
DateTime now = DateTime.Now;
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year);
|
|
referenceDate = Utility.ParseDate(sb.ToString());
|
|
if (referenceDate > now)
|
|
{
|
|
sb = new StringBuilder();
|
|
sb.Append(items[0]).Append(" ").Append(Utility.Pad(items[1], '0', 2)).Append(" ").Append(referenceDate.Year - 1);
|
|
referenceDate = Utility.ParseDate(sb.ToString());
|
|
}
|
|
}
|
|
else referenceDate = Utility.Epoch;
|
|
if (referenceDate > DateTime.Now) referenceDate = Utility.Epoch;
|
|
return referenceDate;
|
|
}
|
|
|
|
// ****************************************************************************************************************************************************************************************
|
|
// **************************************************************************** H E L P E R M E T H O D S *******************************************************************************
|
|
// ****************************************************************************************************************************************************************************************
|
|
private static void WriteNodeToDisk(HtmlNodeCollection rows,String strPathFileName)
|
|
{
|
|
StreamWriter streamWriter = File.CreateText(strPathFileName + ".txt");
|
|
for (int row = 0; row < rows.Count; row++)
|
|
{
|
|
streamWriter.WriteLine("row="+row.ToString()+", InnerHtml="+rows[row].InnerHtml);
|
|
}
|
|
streamWriter.Flush();
|
|
streamWriter.Close();
|
|
}
|
|
|
|
private static void WriteToDisk(String responseString, String strPathFileName)
|
|
{
|
|
StreamWriter streamWriter = File.CreateText(strPathFileName);
|
|
streamWriter.WriteLine(responseString);
|
|
streamWriter.Flush();
|
|
streamWriter.Close();
|
|
}
|
|
|
|
private static void WriteToDisk(List<String> sections, String strPathFileName)
|
|
{
|
|
StreamWriter streamWriter = File.CreateText(strPathFileName);
|
|
for(int index=0;index<sections.Count;index++)
|
|
{
|
|
streamWriter.WriteLine(String.Format("Section[{0}]",index));
|
|
streamWriter.WriteLine(sections[index]);
|
|
}
|
|
streamWriter.Flush();
|
|
streamWriter.Close();
|
|
}
|
|
}
|
|
}
|