diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index 9a665f6..11c7def 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1706,8 +1706,9 @@ namespace MarketData.Helper // *************************************************************************************************************************************************************************** public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol) { - Headlines headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol); - if(null==headlines || 0==headlines.Count)headlines= GetCompanyHeadlinesSeekingAlphaV2(symbol); + Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol); + if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol); + if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol); return headlines; } @@ -1846,6 +1847,97 @@ namespace MarketData.Helper } } + public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol) + { + HttpNetResponse httpNetResponse=null; + Headlines headlines=new Headlines(); + + try + { + String domain = "seekingalpha.com"; + StringBuilder sb = new StringBuilder(); + String strRequest; + symbol = symbol.ToUpper(); + +// Visit the home page + WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV3"); + sb.Append("https://").Append(domain); + + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(sb.ToString(),domain,30000,webProxy,false); + if(!httpNetResponse.Success)return null; + CookieCollection cookieCollection = httpNetResponse.CookieCollection; + + sb=new StringBuilder(); + DateTime marketDate=PremarketDA.GetLatestMarketDate(); + if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; + + #region Create Cookies + + StringBuilder lastVisitedPage = new StringBuilder(); + lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); + lastVisitedPage.Append(symbol); + lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); + + cookieCollection.Add(new Cookie("LAST_VISITED_PAGE", lastVisitedPage.ToString()) { Domain = domain }); + cookieCollection.Add(new Cookie("_ga", "GA1.1.1862228764.1739274352") { Domain = domain }); + cookieCollection.Add(new Cookie("_ga_KGRFF2R2C5","GS1.1.1739274352.1.1.1739274637.60.0.0"){ Domain = domain }); + cookieCollection.Add(new Cookie("_gcl_au","1.1.1611738349.1739274352"){ Domain = domain }); + cookieCollection.Add(new Cookie("_pxvid","bfabea2f-e86d-11ef-832d-24c9f227f821"){ Domain = domain }); + cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain }); + cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain }); + + #endregion + sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]="); +// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2)); +// sb.Append("T04%3A00%3A00.000Z"); + sb.Append("0"); + sb.Append("&filter[until]="); +// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2)); +// sb.Append("T14%3A20%3A34.999Z"); + sb.Append("0"); + sb.Append("&id=").Append(symbol); + sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1"); + strRequest = sb.ToString(); + MDTrace.WriteLine(LogLevel.DEBUG, strRequest); + + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection); + + if(!httpNetResponse.Success) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + return null; + } + byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); + if(null==keyValuePairs)return null; + + foreach (KeyValue keyValue in keyValuePairs) + { + Headline headline=null; + if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); + else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); + headline.Entry=headline.Entry.Replace("\\"," "); + headline.Entry=headline.Entry.Trim(); + headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); + headline.Entry=Uri.UnescapeDataString(headline.Entry); + headline.Source = "Seeking Alpha"; + headlines.Add(headline); + } + headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList()); + + return headlines; + } + catch (Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + return null; + } + finally + { + if(null!=httpNetResponse)httpNetResponse.Dispose(); + } + } + // *************************************************************************************************************************************************************************** //***************************************************************** H E A D L I N E S - M A R K E T W A T C H ************************************************************* // *************************************************************************************************************************************************************************** diff --git a/MarketDataLib/Integration/HttpNetRequest.cs b/MarketDataLib/Integration/HttpNetRequest.cs index 3c4d0c7..fdd04a6 100644 --- a/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketDataLib/Integration/HttpNetRequest.cs @@ -1116,6 +1116,63 @@ namespace MarketData.Integration } } + public static HttpNetResponse GetRequestNoEncodingV5D(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest)); + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.KeepAlive=true; + webRequest.Headers.Add("Priority: u=0, i"); + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd"); + webRequest.Host = host; + webRequest.Headers.Add("Sec-GPC: 1"); + webRequest.Headers.Add("Sec-Fetch-Dest: document"); + webRequest.Headers.Add("Sec-Fetch-Mode: navigate"); + webRequest.Headers.Add("Sec-Fetch-Site: crosssite"); + webRequest.Headers.Add("Sec-Fetch-User: ?1"); + webRequest.Headers.Add("DNT: 1"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.Headers.Add("TE: trailers"); + webRequest.Headers.Add("If-None-Match: "+"W/\"e76e3614af0e341af5df8daa8b855a4e\""); + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent(); + else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36"; + webRequest.CookieContainer = new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse); + httpNetResponse.CookieCollection = new CookieCollection(); + httpNetResponse.CookieCollection.Add(webResponse.Cookies); + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return httpNetResponse; + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]"); + } + } + private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse) { try