Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove defunct Seeking Alpha getters and retain V3

Browse Source
This commit is contained in:
Sean
2025-02-14 18:57:40 -05:00
parent d4c5bcbafa
commit 310cd83f07
2 changed files with 189 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
MarketDataLib/Helper/MarketDataHelper.cs
View File

@@ -1707,147 +1707,154 @@ namespace MarketData.Helper
public static Headlines GetCompanyHeadlinesSeekingAlpha(String symbol)
{
Headlines headlines = GetCompanyHeadlinesSeekingAlphaV3(symbol);
if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol);
if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol);
//if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV1(symbol);
//if (null == headlines || 0 == headlines.Count) headlines = GetCompanyHeadlinesSeekingAlphaV2(symbol);
return headlines;
}
public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
try
{
StringBuilder sb = new StringBuilder();
String strRequest;
symbol = symbol.ToUpper();
//public static Headlines GetCompanyHeadlinesSeekingAlphaV1(String symbol)
//{
// HttpNetResponse httpNetResponse=null;
// Headlines headlines=new Headlines();
// try
// {
// StringBuilder sb = new StringBuilder();
// String strRequest;
// symbol = symbol.ToUpper();
#region Create Cookies
// #region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
// StringBuilder lastVisitedPage = new StringBuilder();
// lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
// lastVisitedPage.Append(symbol);
// lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
// String domain = "seekingalpha.com";
// CookieCollection cookieCollection = new CookieCollection();
// cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
// cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
// #endregion
sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
if(null==keyValuePairs)return null;
// sb.Append("https://seekingalpha.com/api/v3/symbols/").Append(symbol).Append("/news?filter[until]=0&id=").Append(symbol).Append("&include=author,primaryTickers,secondaryTickers,sentiments&page[number]=1&page[size]=11");
// strRequest = sb.ToString();
// MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
// WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV1");
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
// if(!httpNetResponse.Success)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
// return null;
// }
// byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
// List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
// if(null==keyValuePairs)return null;
foreach (KeyValue keyValue in keyValuePairs)
{
Headline headline=null;
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
headline.Entry=headline.Entry.Replace("\\"," ");
headline.Entry=headline.Entry.Trim();
headline.Entry=Uri.UnescapeDataString(headline.Entry);
headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
headline.Entry=Utility.RemoveHtml(headline.Entry);
headline.Source = "Seeking Alpha";
headlines.Add(headline);
}
return headlines;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
{
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// foreach (KeyValue keyValue in keyValuePairs)
// {
// Headline headline=null;
// if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
// else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
// headline.Entry=headline.Entry.Replace("\\"," ");
// headline.Entry=headline.Entry.Trim();
// headline.Entry=Uri.UnescapeDataString(headline.Entry);
// headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
// headline.Entry=Utility.RemoveHtml(headline.Entry);
// headline.Source = "Seeking Alpha";
// headlines.Add(headline);
// }
// return headlines;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
// return null;
// }
// finally
// {
// if(null!=httpNetResponse)httpNetResponse.Dispose();
// }
//}
public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
//public static Headlines GetCompanyHeadlinesSeekingAlphaV2(String symbol)
//{
// HttpNetResponse httpNetResponse=null;
// Headlines headlines=new Headlines();
try
{
StringBuilder sb = new StringBuilder();
String strRequest;
symbol = symbol.ToUpper();
// try
// {
// StringBuilder sb = new StringBuilder();
// String strRequest;
// symbol = symbol.ToUpper();
DateTime marketDate=PremarketDA.GetLatestMarketDate();
if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
// DateTime marketDate=PremarketDA.GetLatestMarketDate();
// if(Utility.IsEpoch(marketDate))marketDate=DateTime.Now;
#region Create Cookies
// #region Create Cookies
StringBuilder lastVisitedPage = new StringBuilder();
lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
lastVisitedPage.Append(symbol);
lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
// StringBuilder lastVisitedPage = new StringBuilder();
// lastVisitedPage.Append("%7B%22pathname%22%3A%22https%3A%2F%2Fseekingalpha.com%2Fsymbol%2F");
// lastVisitedPage.Append(symbol);
// lastVisitedPage.Append("%2Fnews%22%2C%22pageKey%22%3A%22947f55ae-51ad-480f-9f22-54ef1d5904d1%22%7D");
String domain = "seekingalpha.com";
CookieCollection cookieCollection = new CookieCollection();
cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
// String domain = "seekingalpha.com";
// CookieCollection cookieCollection = new CookieCollection();
// cookieCollection.Add(new Cookie("session_id",Guid.NewGuid().ToString()){Domain=domain});
// cookieCollection.Add(new Cookie("LAST_VISITED_PAGE",lastVisitedPage.ToString()){Domain=domain});
#endregion
// #endregion
sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T04%3A00%3A00.000Z&to=");
sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
sb.Append("T14%3A20%3A34.999Z");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
if(!httpNetResponse.Success)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
return null;
}
byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
if(null==keyValuePairs)return null;
// sb.Append("https://").Append(domain).Append("/symbol/").Append(symbol).Append("/news?from=");
// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
// sb.Append("T04%3A00%3A00.000Z&to=");
// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(),'0',2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(),'0',2));
// sb.Append("T14%3A20%3A34.999Z");
// strRequest = sb.ToString();
// MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
// WebProxy webProxy=HttpNetRequest.GetProxy("GetCompanyHeadlinesSeekingAlphaV2");
// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5C(strRequest,"seekingalpha.com",30000,webProxy,false,cookieCollection);
// if(!httpNetResponse.Success)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
// return null;
// }
// byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse.ResponseString);
// List<KeyValue> keyValuePairs = MarketDataHelper.LocateJSONKeyValuePairs(httpNetResponse.ResponseString, "\"publishOn\"", "\"title\"");
// if(null==keyValuePairs)return null;
foreach (KeyValue keyValue in keyValuePairs)
{
Headline headline=null;
if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
headline.Entry=headline.Entry.Replace("\\"," ");
headline.Entry=headline.Entry.Trim();
headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
headline.Entry=Uri.UnescapeDataString(headline.Entry);
headline.Source = "Seeking Alpha";
headlines.Add(headline);
}
return headlines;
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
return null;
}
finally
{
if(null!=httpNetResponse)httpNetResponse.Dispose();
}
}
// foreach (KeyValue keyValue in keyValuePairs)
// {
// Headline headline=null;
// if (keyValue.Value.StartsWith("Video")) headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))),Uri.UnescapeDataString(keyValue.Value));
// else headline = new Headline(symbol, Utility.ParseDate(keyValue.Key.Substring(0, keyValue.Key.IndexOf('T'))), keyValue.Value);
// headline.Entry=headline.Entry.Replace("\\"," ");
// headline.Entry=headline.Entry.Trim();
// headline.Entry=Encoding.UTF8.GetString(Encoding.Default.GetBytes(headline.Entry));
// headline.Entry=Uri.UnescapeDataString(headline.Entry);
// headline.Source = "Seeking Alpha";
// headlines.Add(headline);
// }
// return headlines;
// }
// catch (Exception exception)
// {
// MDTrace.WriteLine(LogLevel.DEBUG,exception.ToString());
// return null;
// }
// finally
// {
// if(null!=httpNetResponse)httpNetResponse.Dispose();
// }
//}
public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol)
/// <summary>
/// Retrieves headlines from seeking alpha. If ignoreMarketDate is TRUE then the results will not be
/// filtered by the current market date .
/// </summary>
/// <param name="symbol"></param>
/// <param name="ignoreMarketDate"></param>
/// <returns></returns>
public static Headlines GetCompanyHeadlinesSeekingAlphaV3(String symbol, bool ignoreMarketDate=false)
{
HttpNetResponse httpNetResponse=null;
Headlines headlines=new Headlines();
@@ -1886,8 +1893,6 @@ namespace MarketData.Helper
cookieCollection.Add(new Cookie("_sasource",""){ Domain = domain });
cookieCollection.Add(new Cookie("pxcts","bfabfefc-e86d-11ef-832f-7d1e5f897eb5"){ Domain = domain });
#endregion
sb.Append("https://").Append(domain).Append("/api/v3/symbols/").Append(symbol).Append("/news?filter[since]=");
// sb.Append(marketDate.Year).Append("-").Append(Utility.Pad(marketDate.Month.ToString(), '0', 2)).Append("-").Append(Utility.Pad(marketDate.Day.ToString(), '0', 2));
@@ -1901,7 +1906,6 @@ namespace MarketData.Helper
sb.Append("&include=author,primaryTickers,secondaryTickers,sentiments,otherTags&isMounting=true&page[size]=40&page[number]=1");
strRequest = sb.ToString();
MDTrace.WriteLine(LogLevel.DEBUG, strRequest);
httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5D(strRequest,domain,30000,webProxy,false,cookieCollection);
@@ -1926,7 +1930,11 @@ namespace MarketData.Helper
headline.Source = "Seeking Alpha";
headlines.Add(headline);
}
headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
if(!ignoreMarketDate)
{
headlines = new Headlines(headlines.Where(x => x.Date.Date.Equals(marketDate.Date)).ToList());
}
return headlines;
}
@@ -2060,7 +2068,8 @@ namespace MarketData.Helper
return null;
}
List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
// List<List<String>> items=LocateJSONItems(httpNetResponse.ResponseString);
List<List<String>> items=LocateJSONItemsTags(httpNetResponse.ResponseString);
if(null==items || items.Count<7)return null;
int index=0;
for(;index<items.Count;index++)
@@ -2070,8 +2079,20 @@ namespace MarketData.Helper
for(;index<items.Count;index+=7)
{
if(!(items[index])[0].Equals("title"))continue;
if(!(items[index+3])[0].Equals("ago"))continue;
DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
DateTime referenceDate = Utility.Epoch;
if((items[index+3])[0].Equals("ago"))
{
referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
}
else if((items[index+2])[0].Equals("ago"))
{
referenceDate=ProcessNASDAQRelativeDate((items[index+2])[1]);
}
else continue;
// if(!(items[index+3])[0].Equals("ago"))continue;
// DateTime referenceDate=ProcessNASDAQRelativeDate((items[index+3])[1]);
if(Utility.IsEpoch(referenceDate))continue;
Headline headline=new Headline();
headline.Source="NASDAQ";
@@ -5878,6 +5899,38 @@ namespace MarketData.Helper
return null;
}
}
public static List<List<String>> LocateJSONItemsTags(String strInput)
{
List<List<String>> items=new List<List<String>>();
try
{
if(null==strInput)return null;
// strInput=Utility.BetweenString(strInput,"[","]");
// if(null==strInput)return null;
String[] itemsArray=strInput.Split(',');
if(null==itemsArray||0==itemsArray.Length)return null;
for(int index=0;index<itemsArray.Length;index++)
{
String item=itemsArray[index];
item=item.Replace("{",null);
item=item.Replace("}",null);
item=item.Replace("\"",null);
String[] subItemArray=item.Split(':');
if(null==subItemArray||2!=subItemArray.Length)continue;
if(null!=subItemArray[0])subItemArray[0]=subItemArray[0].Trim();
if(null!=subItemArray[1])subItemArray[1]=subItemArray[1].Trim();
items.Add(subItemArray.ToList());
}
return items;
}
catch(Exception)
{
return null;
}
}
private static DateTime ProcessNASDAQRelativeDate(String relativeDate)
{
DateTime referenceDate=DateTime.Now;
@@ -5893,7 +5946,7 @@ namespace MarketData.Helper
TimeSpan timeSpan=new TimeSpan(int.Parse(items[0]),0,0);
referenceDate-=timeSpan;
}
else if("mins".Equals(items[1]))
else if("mins".Equals(items[1]) || "minutes".Equals(items[1]))
{
TimeSpan timeSpan=new TimeSpan(0,int.Parse(items[0]),0);
referenceDate-=timeSpan;

6
MarketDataLib/Helper/MarketDataHelperBase.cs
View File

@@ -37,23 +37,27 @@ namespace MarketData.Helper
queue[r2] = temp;
}
}
public T PeekQueueItem()
{
if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T);
return queue[currentIndex+1];
}
public T GetQueueItem()
{
if(null==queue)return default(T);
if (currentIndex+1 >= queue.Count) return default(T);
return queue[++currentIndex];
}
public int Index
{
get{return currentIndex;}
set{currentIndex=value;}
}
public ManualResetEvent[] ResizeEvents(ManualResetEvent[] events)
{
ManualResetEvent[] busyEvents=GetBusyEvents(events);
@@ -66,6 +70,7 @@ namespace MarketData.Helper
}
return events;
}
public ManualResetEvent[] GetBusyEvents(ManualResetEvent[] resetEvents)
{
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();
@@ -80,6 +85,7 @@ namespace MarketData.Helper
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("The event queue has {0} busy slots",availableEvents.Count));
return availableEvents.ToArray();
}
public ManualResetEvent[] GetAvailableEvents(ManualResetEvent[] resetEvents)
{
List<ManualResetEvent> availableEvents = new List<ManualResetEvent>();