Remove defunct Seeking Alpha getters and retain V3

This commit is contained in:
2025-02-14 18:57:40 -05:00
parent d4c5bcbafa
commit 310cd83f07
2 changed files with 189 additions and 130 deletions

View File

@@ -1707,147 +1707,154 @@ namespace MarketData.Helper
public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol) public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol)
{ {
Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol); Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol);
if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol); //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol);
if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol); //if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol);
return headlines; return headlines;
} }
public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol) //public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
{ //{
HttpNetResponse httpNetResponse=null; // HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines(); // Headlines headlines=new Headlines();
try // try
{ // {
StringBuilder sb = new StringBuilder(); // StringBuilder sb = new StringBuilder();
String strRequest; // String strRequest;
symbol = symbol.ToUpper(); // symbol = symbol.ToUpper();
#region Create Cookies // #region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder(); // StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); // lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol); // lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); // lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com"; // String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection(); // CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); // cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); // cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion // #endregion
sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11"); // sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
strRequest = sb.ToString(); // strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest); // MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1"); // WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success) // if(!httpNetResponse.Success)
{ // {
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null; // return null;
} // }
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); // byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); // List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
if(null==keyValuePairs)return null; // if(null==keyValuePairs)return null;
foreach (KeyValue keyValue in keyValuePairs) // foreach (KeyValue keyValue in keyValuePairs)
{ // {
Headline headline=null; // Headline headline=null;
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); // if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); // else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
headline.Entry=headline.Entry.Replace("\\"," "); // headline.Entry=headline.Entry.Replace("\\"," ");
headline.Entry=headline.Entry.Trim(); // headline.Entry=headline.Entry.Trim();
headline.Entry=Uri.UnescapeDataString(headline.Entry); // headline.Entry=Uri.UnescapeDataString(headline.Entry);
headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); // headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
headline.Entry=Utility.RemoveHtml(headline.Entry); // headline.Entry=Utility.RemoveHtml(headline.Entry);
headline.Source = "Seeking Alpha"; // headline.Source = "Seeking Alpha";
headlines.Add(headline); // headlines.Add(headline);
} // }
return headlines; // return headlines;
} // }
catch (Exception exception) // catch (Exception exception)
{ // {
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); // MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null; // return null;
} // }
finally // finally
{ // {
if(null!=httpNetResponse)httpNetResponse.Dispose(); // if(null!=httpNetResponse)httpNetResponse.Dispose();
} // }
} //}
public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol) //public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
{ //{
HttpNetResponse httpNetResponse=null; // HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines(); // Headlines headlines=new Headlines();
try // try
{ // {
StringBuilder sb = new StringBuilder(); // StringBuilder sb = new StringBuilder();
String strRequest; // String strRequest;
symbol = symbol.ToUpper(); // symbol = symbol.ToUpper();
DateTime marketDate=PremarketDA.GetLatestMarketDate(); // DateTime marketDate=PremarketDA.GetLatestMarketDate();
if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now; // if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
#region Create Cookies // #region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder(); // StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F"); // lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol); // lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D"); // lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com"; // String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection(); // CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain}); // cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain}); // cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion // #endregion
sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from="); // sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T04%3A00%3A00.000Z&to="); // sb.Append("T04%3A00%3A00.000Z&to=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2)); // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T14%3A20%3A34.999Z"); // sb.Append("T14%3A20%3A34.999Z");
strRequest = sb.ToString(); // strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest); // MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2"); // WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection); // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success) // if(!httpNetResponse.Success)
{ // {
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null; // return null;
} // }
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString); // byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\""); // List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
if(null==keyValuePairs)return null; // if(null==keyValuePairs)return null;
foreach (KeyValue keyValue in keyValuePairs) // foreach (KeyValue keyValue in keyValuePairs)
{ // {
Headline headline=null; // Headline headline=null;
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value)); // if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value); // else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
headline.Entry=headline.Entry.Replace("\\"," "); // headline.Entry=headline.Entry.Replace("\\"," ");
headline.Entry=headline.Entry.Trim(); // headline.Entry=headline.Entry.Trim();
headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry)); // headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
headline.Entry=Uri.UnescapeDataString(headline.Entry); // headline.Entry=Uri.UnescapeDataString(headline.Entry);
headline.Source = "Seeking Alpha"; // headline.Source = "Seeking Alpha";
headlines.Add(headline); // headlines.Add(headline);
} // }
return headlines; // return headlines;
} // }
catch (Exception exception) // catch (Exception exception)
{ // {
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString()); // MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null; // return null;
} // }
finally // finally
{ // {
if(null!=httpNetResponse)httpNetResponse.Dispose(); // if(null!=httpNetResponse)httpNetResponse.Dispose();
} // }
} //}
public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol) /// <summary>
/// Retrieves headlines from seeking alpha. If ignoreMarketDate is TRUE then the results will not be
/// filtered by the current market date .
/// </summary>
/// <param name="symbol"></param>
/// <param name="ignoreMarketDate"></param>
/// <returns></returns>
public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false)
{ {
HttpNetResponse httpNetResponse=null; HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines(); Headlines headlines=new Headlines();
@@ -1886,8 +1893,6 @@ namespace MarketData.Helper
cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain }); cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain });
cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain }); cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain });
#endregion #endregion
sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]="); sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]=");
// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2)); // sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2));
@@ -1901,7 +1906,6 @@ namespace MarketData.Helper
sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1"); sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1");
strRequest = sb.ToString(); strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest); MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection); httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection);
@@ -1926,7 +1930,11 @@ namespace MarketData.Helper
headline.Source = "Seeking Alpha"; headline.Source = "Seeking Alpha";
headlines.Add(headline); headlines.Add(headline);
} }
headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
if(!ignoreMarketDate)
{
headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
}
return headlines; return headlines;
} }
@@ -2060,7 +2068,8 @@ namespace MarketData.Helper
return null; return null;
} }
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString); // List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
List<List<String>> items=LocateJSONItemsTags(httpNetResponse.ResponseString);
if(null==items || items.Count<7)return null; if(null==items || items.Count<7)return null;
int index=0; int index=0;
for(;index<items.Count;index++) for(;index<items.Count;index++)
@@ -2070,8 +2079,20 @@ namespace MarketData.Helper
for(;index<items.Count;index+=7) for(;index<items.Count;index+=7)
{ {
if(!(items[index])[0].Equals("title"))continue; if(!(items[index])[0].Equals("title"))continue;
if(!(items[index+3])[0].Equals("ago"))continue;
DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]); DateTime referenceDate = Utility.Epoch;
if((items[index+3])[0].Equals("ago"))
{
referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
}
else if((items[index+2])[0].Equals("ago"))
{
referenceDate=ProcessNASDAQRelativeDate((items[index+2])[1]);
}
else continue;
// if(!(items[index+3])[0].Equals("ago"))continue;
// DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
if(Utility.IsEpoch(referenceDate))continue; if(Utility.IsEpoch(referenceDate))continue;
Headline headline=new Headline(); Headline headline=new Headline();
headline.Source="NASDAQ"; headline.Source="NASDAQ";
@@ -5878,6 +5899,38 @@ namespace MarketData.Helper
return null; return null;
} }
} }
public static List<List<String>> LocateJSONItemsTags(String strInput)
{
List<List<String>> items=new List<List<String>>();
try
{
if(null==strInput)return null;
// strInput=Utility.BetweenString(strInput,"[","]");
// if(null==strInput)return null;
String[] itemsArray=strInput.Split(',');
if(null==itemsArray||0==itemsArray.Length)return null;
for(int index=0;index<itemsArray.Length;index++)
{
String item=itemsArray[index];
item=item.Replace("{",null);
item=item.Replace("}",null);
item=item.Replace("\"",null);
String[] subItemArray=item.Split(':');
if(null==subItemArray||2!=subItemArray.Length)continue;
if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
items.Add(subItemArray.ToList());
}
return items;
}
catch(Exception)
{
return null;
}
}
private static DateTime ProcessNASDAQRelativeDate(String relativeDate) private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
{ {
DateTime referenceDate=DateTime.Now; DateTime referenceDate=DateTime.Now;
@@ -5893,7 +5946,7 @@ namespace MarketData.Helper
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0); TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
referenceDate-=timeSpan; referenceDate-=timeSpan;
} }
else if("mins".Equals(items[1])) else if("mins".Equals(items[1]) || "minutes".Equals(items[1]))
{ {
TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0); TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
referenceDate-=timeSpan; referenceDate-=timeSpan;

View File

@@ -37,23 +37,27 @@ namespace MarketData.Helper
queue[r2] = temp; queue[r2] = temp;
} }
} }
public T PeekQueueItem() public T PeekQueueItem()
{ {
if(null==queue)return default(T); if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T); if (currentIndex+1 >= queue.Count) return default(T);
return queue[currentIndex+1]; return queue[currentIndex+1];
} }
public T GetQueueItem() public T GetQueueItem()
{ {
if(null==queue)return default(T); if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T); if (currentIndex+1 >= queue.Count) return default(T);
return queue[++currentIndex]; return queue[++currentIndex];
} }
public int Index public int Index
{ {
get{return currentIndex;} get{return currentIndex;}
set{currentIndex=value;} set{currentIndex=value;}
} }
public ManualResetEvent[] ResizeEvents(ManualResetEvent[] events) public ManualResetEvent[] ResizeEvents(ManualResetEvent[] events)
{ {
ManualResetEvent[] busyEvents=GetBusyEvents(events); ManualResetEvent[] busyEvents=GetBusyEvents(events);
@@ -66,6 +70,7 @@ namespace MarketData.Helper
} }
return events; return events;
} }
public ManualResetEvent[] GetBusyEvents(ManualResetEvent[] resetEvents) public ManualResetEvent[] GetBusyEvents(ManualResetEvent[] resetEvents)
{ {
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>(); List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();
@@ -80,6 +85,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count)); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count));
return availableEvents.ToArray(); return availableEvents.ToArray();
} }
public ManualResetEvent[] GetAvailableEvents(ManualResetEvent[] resetEvents) public ManualResetEvent[] GetAvailableEvents(ManualResetEvent[] resetEvents)
{ {
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>(); List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();