Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Yahoo Fundamental retrieval.

Browse Source 
Fix SeekingAlpha news retrieval.
This commit is contained in:
Sean
2024-04-12 12:40:41 -04:00
parent 799356533a
commit c0c1d37bf0
11 changed files with 353 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1655,6 +1655,7 @@ namespace MarketData.Helper
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// ***************************************************************************************************************************************************************************
//***************************************************************** H E A D L I N E S - S E E K I N G A L P H A ***********************************************************
// ***************************************************************************************************************************************************************************
@@ -1665,7 +1666,7 @@ namespace MarketData.Helper
return headlines;
}
private static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -1675,14 +1676,25 @@ namespace MarketData.Helper
String strRequest;
symbol = symbol.ToUpper();
CookieCollection cookieCollection=new CookieCollection();
httpNetResponse= HttpNetRequest.GetRequestNoEncodingV4("https://www.seekingalpha.com",cookieCollection);
Thread.Sleep(250);
#region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,cookieCollection);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1707,7 +1719,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1716,7 +1728,7 @@ namespace MarketData.Helper
}
}
private static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -1730,16 +1742,29 @@ namespace MarketData.Helper
DateTime marketDate=PremarketDA.GetLatestMarketDate();
if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
#region Create Cookies
sb.Append("https://seekingalpha.com/symbol/").Append(symbol).Append("/news?from=");
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T04%3A00%3A00.000Z&to=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T14%3A20%3A34.999Z");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlpha");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5B(strRequest,30000,webProxy,true,null);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
@@ -1764,7 +1789,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1861,7 +1886,7 @@ namespace MarketData.Helper
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -1913,12 +1938,11 @@ namespace MarketData.Helper
headline.Entry=Utility.RemoveHtml(headline.Entry);
headlines.Add(headline);
}
// headlines=new Headlines(headlines.GroupBy(x=>x.Entry).Select(y=>y.First()).ToList());
return headlines;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception);
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
@@ -4547,9 +4571,15 @@ namespace MarketData.Helper
public static Fundamental GetFundamental(String symbol)
{
Fundamental fundamental=GetFundamentalEx(symbol);
if(null==fundamental)return fundamental;
if(fundamental.GetLoad()<80)
{
Fundamental fundamental2 = GetFundamentalEx(symbol);
if(null!=fundamental2)fundamental.MergeFrom(fundamental2);
}
return fundamental;
}
public static Fundamental GetFundamentalEx(String symbol)
private static Fundamental GetFundamentalEx(String symbol)
{
HttpNetResponse httpNetResponse=null;
int TIMEOUT_MS=1000*30;
@@ -4641,26 +4671,63 @@ namespace MarketData.Helper
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
try{Thread.Sleep(250);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
}
}
fundamental.TrailingPE=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Trailing P/E<"));
fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5 yr expected)<"));
if(double.IsNaN(fundamental.PEG))fundamental.PEG=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">PEG Ratio (5yr expected)<"));
fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets<"));
if(double.IsNaN(fundamental.ReturnOnAssets))fundamental.ReturnOnAssets=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Assets (ttm)<"));
fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity<"));
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
if(double.IsNaN(fundamental.ReturnOnEquity))fundamental.ReturnOnEquity=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Return on Equity (ttm)<"));
fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share<"));
if(double.IsNaN(fundamental.RevenuePerShare))fundamental.RevenuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue Per Share (ttm)<"));
fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash<"));
if(double.IsNaN(fundamental.TotalCash))fundamental.TotalCash=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Cash (mrq)<"));
fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt<"));
if(double.IsNaN(fundamental.TotalDebt))fundamental.TotalDebt=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Total Debt (mrq)<"));
fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Shares Outstanding<",7));
if(double.IsNaN(fundamental.SharesOutstanding))fundamental.SharesOutstanding=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,"Implied Shares Outstanding <sup>6</sup> "));
fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue<"));
if(double.IsNaN(fundamental.Revenue))fundamental.Revenue=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Revenue (ttm)<"));
fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth<"));
if(double.IsNaN(fundamental.QtrlyRevenueGrowth))fundamental.QtrlyRevenueGrowth=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Quarterly Revenue Growth (yoy)<"));
fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit<"));
if(double.IsNaN(fundamental.GrossProfit))fundamental.GrossProfit=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Gross Profit (ttm)<"));
fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA<"));
if(double.IsNaN(fundamental.EBITDA))fundamental.EBITDA=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBITDA <"));
fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common<"));
if(double.IsNaN(fundamental.NetIncomeAvailableToCommon))fundamental.NetIncomeAvailableToCommon=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Net Income Avi to Common (ttm)<"));
fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share<"));
if(double.IsNaN(fundamental.BookValuePerShare))fundamental.BookValuePerShare=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Book Value Per Share (mrq)<"));
fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow<"));
if(double.IsNaN(fundamental.OperatingCashflow))fundamental.OperatingCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Operating Cash Flow (ttm)<"));
fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow<"));
if(double.IsNaN(fundamental.LeveragedFreeCashflow))fundamental.LeveragedFreeCashflow=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Levered Free Cash Flow (ttm)<"));
fundamental.EnterpriseValue=FeedParser.ParseValue(Sections.LocateItemMinDepth(httpNetResponse.ResponseString,">Enterprise Value<",7));
return true;
}
catch(Exception exception)
@@ -4688,10 +4755,17 @@ namespace MarketData.Helper
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
try{Thread.Sleep(250);}catch(Exception){;}
httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return false;
}
}
fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">Earnings Before Interest and Taxes<"))*1000.00;
if(double.IsNaN(fundamental.EBIT))fundamental.EBIT=FeedParser.ParseValue(Sections.LocateItem(httpNetResponse.ResponseString,">EBIT<"))*1000.00;
return true;
}
catch(Exception exception)
@@ -5032,6 +5106,10 @@ namespace MarketData.Helper
{
price.Open=price.High=price.Low=price.Close;
}
if(Utility.IsZeroOrNaN(price.Open) && !Utility.IsZeroOrNaN(price.High) && !Utility.IsZeroOrNaN(price.Low))
{
price.Open=(price.High+price.Low)/2.00;
}
}
// This is used in the intra-day price feed.