From 310cd83f075afb14c8b08e1afb69ebd0471b02e4 Mon Sep 17 00:00:00 2001 From: Sean Date: Fri, 14 Feb 2025 18:57:40 -0500 Subject: [PATCH] Remove defunct Seeking Alpha getters and retain V3 --- MarketDataLib/Helper/MarketDataHelper.cs | 313 +++++++++++-------- MarketDataLib/Helper/MarketDataHelperBase.cs | 6 + 2 files changed, 189 insertions(+), 130 deletions(-) diff --git a/MarketDataLib/Helper/MarketDataHelper.cs b/MarketDataLib/Helper/MarketDataHelper.cs index bbe6698..9f4dd84 100644 --- a/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketDataLib/Helper/MarketDataHelper.cs @@ -1707,147 +1707,154 @@ namespace MarketData.Helper public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol) { Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol); - if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol); - if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol); + //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol); + //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol); return headlines; } - public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol) - { - HttpNetResponse httpNetResponse=null; - Headlines headlines=new Headlines(); - try - { - StringBuilder sb = new StringBuilder(); - String strRequest; - symbol = symbol.ToUpper(); + //public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol) + //{ + // HttpNetResponse httpNetResponse=null; + // Headlines headlines=new Headlines(); + // try + // { + // StringBuilder sb = new StringBuilder(); + // String strRequest; + // symbol = symbol.ToUpper(); - #region Create Cookies + // #region Create Cookies - StringBuilder lastVisitedPage = new StringBuilder(); - lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); - lastVisitedPage.Append(symbol); - lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); + // StringBuilder lastVisitedPage = new StringBuilder(); + // lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); + // lastVisitedPage.Append(symbol); + // lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); - String domain = "seekingalpha.com"; - CookieCollection cookieCollection = new CookieCollection(); - cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); - cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); + // String domain = "seekingalpha.com"; + // CookieCollection cookieCollection = new CookieCollection(); + // cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); + // cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); - #endregion + // #endregion - sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11"); - strRequest = sb.ToString(); - MDTrace.WriteLine(LogLevel.DEBUG, strRequest); - WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); - if(!httpNetResponse.Success) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - return null; - } - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); - if(null==keyValuePairs)return null; + // sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11"); + // strRequest = sb.ToString(); + // MDTrace.WriteLine(LogLevel.DEBUG, strRequest); + // WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1"); + // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); + // if(!httpNetResponse.Success) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + // return null; + // } + // byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + // List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); + // if(null==keyValuePairs)return null; - foreach (KeyValue keyValue in keyValuePairs) - { - Headline headline=null; - if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); - else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); - headline.Entry=headline.Entry.Replace("\\"," "); - headline.Entry=headline.Entry.Trim(); - headline.Entry=Uri.UnescapeDataString(headline.Entry); - headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); - headline.Entry=Utility.RemoveHtml(headline.Entry); - headline.Source = "Seeking Alpha"; - headlines.Add(headline); - } - return headlines; - } - catch (Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); - return null; - } - finally - { - if(null!=httpNetResponse)httpNetResponse.Dispose(); - } - } + // foreach (KeyValue keyValue in keyValuePairs) + // { + // Headline headline=null; + // if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); + // else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); + // headline.Entry=headline.Entry.Replace("\\"," "); + // headline.Entry=headline.Entry.Trim(); + // headline.Entry=Uri.UnescapeDataString(headline.Entry); + // headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); + // headline.Entry=Utility.RemoveHtml(headline.Entry); + // headline.Source = "Seeking Alpha"; + // headlines.Add(headline); + // } + // return headlines; + // } + // catch (Exception exception) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + // return null; + // } + // finally + // { + // if(null!=httpNetResponse)httpNetResponse.Dispose(); + // } + //} - public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol) - { - HttpNetResponse httpNetResponse=null; - Headlines headlines=new Headlines(); + //public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol) + //{ + // HttpNetResponse httpNetResponse=null; + // Headlines headlines=new Headlines(); - try - { - StringBuilder sb = new StringBuilder(); - String strRequest; - symbol = symbol.ToUpper(); + // try + // { + // StringBuilder sb = new StringBuilder(); + // String strRequest; + // symbol = symbol.ToUpper(); - DateTime marketDate=PremarketDA.GetLatestMarketDate(); - if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; + // DateTime marketDate=PremarketDA.GetLatestMarketDate(); + // if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; - #region Create Cookies + // #region Create Cookies - StringBuilder lastVisitedPage = new StringBuilder(); - lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); - lastVisitedPage.Append(symbol); - lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); + // StringBuilder lastVisitedPage = new StringBuilder(); + // lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); + // lastVisitedPage.Append(symbol); + // lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); - String domain = "seekingalpha.com"; - CookieCollection cookieCollection = new CookieCollection(); - cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); - cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); + // String domain = "seekingalpha.com"; + // CookieCollection cookieCollection = new CookieCollection(); + // cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); + // cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); - #endregion + // #endregion - sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from="); - sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); - sb.Append("T04%3A00%3A00.000Z&to="); - sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); - sb.Append("T14%3A20%3A34.999Z"); - strRequest = sb.ToString(); - MDTrace.WriteLine(LogLevel.DEBUG, strRequest); - WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2"); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); - if(!httpNetResponse.Success) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - return null; - } - byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); - List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); - if(null==keyValuePairs)return null; + // sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from="); + // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); + // sb.Append("T04%3A00%3A00.000Z&to="); + // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); + // sb.Append("T14%3A20%3A34.999Z"); + // strRequest = sb.ToString(); + // MDTrace.WriteLine(LogLevel.DEBUG, strRequest); + // WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2"); + // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); + // if(!httpNetResponse.Success) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + // return null; + // } + // byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); + // List keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); + // if(null==keyValuePairs)return null; - foreach (KeyValue keyValue in keyValuePairs) - { - Headline headline=null; - if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); - else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); - headline.Entry=headline.Entry.Replace("\\"," "); - headline.Entry=headline.Entry.Trim(); - headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); - headline.Entry=Uri.UnescapeDataString(headline.Entry); - headline.Source = "Seeking Alpha"; - headlines.Add(headline); - } - return headlines; - } - catch (Exception exception) - { - MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); - return null; - } - finally - { - if(null!=httpNetResponse)httpNetResponse.Dispose(); - } - } + // foreach (KeyValue keyValue in keyValuePairs) + // { + // Headline headline=null; + // if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); + // else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); + // headline.Entry=headline.Entry.Replace("\\"," "); + // headline.Entry=headline.Entry.Trim(); + // headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); + // headline.Entry=Uri.UnescapeDataString(headline.Entry); + // headline.Source = "Seeking Alpha"; + // headlines.Add(headline); + // } + // return headlines; + // } + // catch (Exception exception) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); + // return null; + // } + // finally + // { + // if(null!=httpNetResponse)httpNetResponse.Dispose(); + // } + //} - public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol) + /// + /// Retrieves headlines from seeking alpha. If ignoreMarketDate is TRUE then the results will not be + /// filtered by the current market date . + /// + /// + /// + /// + public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false) { HttpNetResponse httpNetResponse=null; Headlines headlines=new Headlines(); @@ -1886,8 +1893,6 @@ namespace MarketData.Helper cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain }); cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain }); - - #endregion sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]="); // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2)); @@ -1901,7 +1906,6 @@ namespace MarketData.Helper sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG, strRequest); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection); @@ -1926,7 +1930,11 @@ namespace MarketData.Helper headline.Source = "Seeking Alpha"; headlines.Add(headline); } - headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList()); + + if(!ignoreMarketDate) + { + headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList()); + } return headlines; } @@ -2060,7 +2068,8 @@ namespace MarketData.Helper return null; } - List> items=LocateJSONItems(httpNetResponse.ResponseString); +// List> items=LocateJSONItems(httpNetResponse.ResponseString); + List> items=LocateJSONItemsTags(httpNetResponse.ResponseString); if(null==items || items.Count<7)return null; int index=0; for(;index> LocateJSONItemsTags(String strInput) + { + List> items=new List>(); + try + { + if(null==strInput)return null; +// strInput=Utility.BetweenString(strInput,"[","]"); +// if(null==strInput)return null; + String[] itemsArray=strInput.Split(','); + if(null==itemsArray||0==itemsArray.Length)return null; + for(int index=0;index= queue.Count) return default(T); return queue[currentIndex+1]; } + public T GetQueueItem() { if(null==queue)return default(T); if (currentIndex+1 >= queue.Count) return default(T); return queue[++currentIndex]; } + public int Index { get{return currentIndex;} set{currentIndex=value;} } + public ManualResetEvent[] ResizeEvents(ManualResetEvent[] events) { ManualResetEvent[] busyEvents=GetBusyEvents(events); @@ -66,6 +70,7 @@ namespace MarketData.Helper } return events; } + public ManualResetEvent[] GetBusyEvents(ManualResetEvent[] resetEvents) { List availableEvents = new List(); @@ -80,6 +85,7 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count)); return availableEvents.ToArray(); } + public ManualResetEvent[] GetAvailableEvents(ManualResetEvent[] resetEvents) { List availableEvents = new List();