Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Yahoo Fundamental retrieval.

Browse Source 
Fix SeekingAlpha news retrieval.
This commit is contained in:
Sean
2024-04-12 12:40:41 -04:00
parent 799356533a
commit c0c1d37bf0
11 changed files with 353 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
MarketDataLib/Helper/HeadlinesMarketDataHelper.cs
View File

@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MarketData.MarketDataModel;
using MarketData.DataAccess;
@@ -11,8 +10,8 @@ namespace MarketData.Helper
{
public class HeadlinesMarketDataHelper
{
private static int MaxThreads = 10; // (int)ThreadHelperEnum.MaxThreads;
private static int WAIT_BETWEEN_REQUESTS_MS = 1000; // wait 1000 ms between requests
private static int MaxThreads = 5; // (int)ThreadHelperEnum.MaxThreads;
private static int WAIT_BETWEEN_REQUESTS_MS = 2000; // wait ms between requests
private List<String> symbols;
private int currentIndex = 0;
@@ -134,23 +133,6 @@ namespace MarketData.Helper
marketDate=new DateTime(marketDate.Year,marketDate.Month,marketDate.Day,23,59,59);
Headlines headlines=null;
// SEEKING ALPHA
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
if(headlines.IsNullOrEmpty())
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
HeadlinesDA.InsertHeadlines(headlines);
foreach(Headline headline in headlines)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
// NASDAQ
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesNASDAQ {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesNASDAQ(symbol);
@@ -185,6 +167,23 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("MARKETWATCH: {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
// SEEKING ALPHA
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetCompanyHeadlinesSeekingAlpha {0}",symbol));
headlines=MarketDataHelper.GetCompanyHeadlinesSeekingAlpha(symbol);
if(headlines.IsNullOrEmpty())
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("No headlines for {0} from Seeking Alpha",symbol));
}
else
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Got {0} headlines for {1} from Seeking Alpha",headlines.Count,symbol));
headlines=new Headlines(headlines.Where(x=>x.Date<marketDate).ToList());
HeadlinesDA.InsertHeadlines(headlines);
foreach(Headline headline in headlines)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Seeking Alpha {0}, {1} -> {2}",headline.Symbol,headline.Date.ToShortDateString(),headline.Entry));
}
}
return;
}
}

126
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1655,6 +1655,7 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S - S E E K I N G A L P H A ***********************************************************
// ***************************************************************************************************************************************************************************
@@ -1665,7 +1666,7 @@ namespace MarketData.Helper
return headlines;
}
private static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -1675,14 +1676,25 @@ namespace MarketData.Helper
String strRequest;
symbol = symbol.ToUpper();
CookieCollection cookieCollection=new CookieCollection();
httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.seekingalpha.com",cookieCollection);
Thread.Sleep(250);
#region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,cookieCollection);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1707,7 +1719,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1716,7 +1728,7 @@ namespace MarketData.Helper
}
}
private static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -1730,16 +1742,29 @@ namespace MarketData.Helper
DateTime marketDate=PremarketDA.GetLatestMarketDate();
if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
#region Create Cookies
sb.Append("https://seekingalpha.com/symbol/").Append(symbol).Append("/news?from=");
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T04%3A00%3A00.000Z&to=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T14%3A20%3A34.999Z");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,null);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1764,7 +1789,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1861,7 +1886,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1913,12 +1938,11 @@ namespace MarketData.Helper
headline.Entry=Utility.RemoveHtml(headline.Entry);
headlines.Add(headline);
}
// headlines=new Headlines(headlines.GroupBy(x=>x.Entry).Select(y=>y.First()).ToList());
return headlines;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -4547,9 +4571,15 @@ namespace MarketData.Helper
public static Fundamental GetFundamental(String symbol)
{
Fundamental fundamental=GetFundamentalEx(symbol);
if(null==fundamental)return fundamental;
if(fundamental.GetLoad()<80)
{
Fundamental fundamental2 = GetFundamentalEx(symbol);
if(null!=fundamental2)fundamental.MergeFrom(fundamental2);
}
return fundamental;
}
public static Fundamental GetFundamentalEx(String symbol)
private static Fundamental GetFundamentalEx(String symbol)
{
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS=1000*30;
@@ -4641,26 +4671,63 @@ namespace MarketData.Helper
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
try{Thread.Sleep(250);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
}
}
fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));
fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<"));
fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets (ttm)<"));
fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity (ttm)<"));
fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share (ttm)<"));
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash (mrq)<"));
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt (mrq)<"));
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding <sup>6</sup> "));
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue (ttm)<"));
fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth (yoy)<"));
fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit (ttm)<"));
fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA <"));
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common (ttm)<"));
fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share (mrq)<"));
fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow (ttm)<"));
fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow (ttm)<"));
fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));
return true;
}
catch(Exception exception)
@@ -4688,10 +4755,17 @@ namespace MarketData.Helper
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
try{Thread.Sleep(250);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
}
}
fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00;
return true;
}
catch(Exception exception)
@@ -5032,6 +5106,10 @@ namespace MarketData.Helper
{
price.Open=price.High=price.Low=price.Close;
}
if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low))
{
price.Open=(price.High+price.Low)/2.00;
}
}
// This is used in the intra-day price feed.