diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 8b06ca5..6104012 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -1537,7 +1537,8 @@ namespace MarketData.Helper symbol = symbol.ToUpper(); WebProxy webProxy = HttpNetRequest.GetProxy("GetCompanyProfileYahoo"); - sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile"); +// sb.Append("http://finance.yahoo.com/q/pr?s=").Append(symbol).Append("+Profile"); + sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/profile/"); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); httpNetResponse = HttpNetRequest.GetRequestNoEncodingV1(strRequest); @@ -1559,17 +1560,8 @@ namespace MarketData.Helper if (null != strIndustry && strIndustry.Contains(Constants.CONST_QUESTION)) strIndustry = strIndustry.Replace(Constants.CONST_QUESTION, " - "); if (null != strSector && strSector.Contains(Constants.CONST_QUESTION)) strSector = strSector.Replace(Constants.CONST_QUESTION, " - "); -// Locate Description - String strDescription = Sections.LocateItem(httpNetResponse.ResponseString, "Description", 4); - if(null == strDescription) - { - List indices = Sections.LocateAllOccurrences(httpNetResponse.ResponseString, "Description"); - if(indices.Count>0) - { - List sections = Sections.GetSections(httpNetResponse.ResponseString); - strDescription = Sections.GetFirstNonEmptyItemInSection(sections, indices[0]+1); - } - } +// Yahoo changed this... again. + String strDescription = ExtractLongBusinessSummary(httpNetResponse.ResponseString); if(null!=strDescription && strDescription.Equals("Description Information Not Available")) { @@ -1602,6 +1594,44 @@ namespace MarketData.Helper } } + /// + /// Retrieve the Company Description field in the Yahoo Company Profile Data + /// + /// + /// + public static string ExtractLongBusinessSummary(string html) + { + if(string.IsNullOrEmpty(html))return null; + // locate the script containing the assetProfile API response + int start = html.IndexOf("modules=assetProfile"); + if (start < 0) return null; + + // move back to start of script tag + start = html.LastIndexOf("", start); + if (jsonStart < 0) return null; + jsonStart++; + + int jsonEnd = html.IndexOf("", jsonStart); + if (jsonEnd < 0) return null; + + string outerJson = html.Substring(jsonStart, jsonEnd - jsonStart); + + // parse outer JSON + JObject outer = Newtonsoft.Json.Linq.JObject.Parse(outerJson); + + // body is escaped JSON + string bodyJson = outer["body"]?.ToString(); + if (bodyJson == null) return null; + + // parse inner JSON + JObject inner = Newtonsoft.Json.Linq.JObject.Parse(bodyJson); + + return inner["quoteSummary"]?["result"]?[0]?["assetProfile"]?["longBusinessSummary"]?.ToString(); + } + /// /// Retrieve company profile information from MorningStar /// @@ -1622,7 +1652,8 @@ namespace MarketData.Helper sb.Append(String.Format("https://www.morningstar.com/stocks/{0}/{1}/quote",exchange,symbol)); strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,strRequest); - httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 15000, webProxy); +// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 15000, webProxy); + httpNetResponse = HttpNetRequest.GetRequestV6(strRequest, 15000, webProxy); if(!httpNetResponse.Success) { diff --git a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs index f9886a3..fb4d32d 100755 --- a/MarketData/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketData/MarketDataLib/Integration/HttpNetRequest.cs @@ -1523,6 +1523,81 @@ namespace MarketData.Integration } } + /// + /// GetRequestV6 - Uses HttpClient to more closely match web browser. + /// + /// + /// + /// + /// + /// + public static HttpNetResponse GetRequestV6(string url,int timeoutMS,WebProxy proxy = null,string cookieHeader = null) + { + HttpResponseMessage response = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, $"GetRequestV6[ENTER]{url}"); + var handler = new HttpClientHandler() + { + AllowAutoRedirect = true, + AutomaticDecompression = + DecompressionMethods.GZip | + DecompressionMethods.Deflate | + DecompressionMethods.Brotli, + UseCookies = true, + CookieContainer = new CookieContainer() + }; + if (proxy != null) + { + handler.Proxy = proxy; + handler.UseProxy = true; + } + using (HttpClient client = new HttpClient(handler)) + { + client.Timeout = TimeSpan.FromMilliseconds(timeoutMS); + // Core headers + client.DefaultRequestHeaders.TryAddWithoutValidation( + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0"); + client.DefaultRequestHeaders.TryAddWithoutValidation( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + client.DefaultRequestHeaders.TryAddWithoutValidation( + "Accept-Language", + "en-US,en;q=0.9"); + client.DefaultRequestHeaders.TryAddWithoutValidation( + "Accept-Encoding", + "gzip, deflate, br, zstd"); + + client.DefaultRequestHeaders.Connection.Add("keep-alive"); + // Browser-like headers + client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1"); + client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i"); + client.DefaultRequestHeaders.TryAddWithoutValidation("TE", "trailers"); + if (!string.IsNullOrEmpty(cookieHeader)) + { + client.DefaultRequestHeaders.TryAddWithoutValidation("Cookie", cookieHeader); + } + response = client.GetAsync(url).GetAwaiter().GetResult(); + string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + return new HttpNetResponse(html,url,null,handler.CookieContainer.GetCookies(new Uri(url)),response.IsSuccessStatusCode); + } + } + catch (Exception ex) + { + return new HttpNetResponse(null,url,false,ex.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestV6[LEAVE]"); + } + } + private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse) { try diff --git a/MarketDataUnitTests/MarketDataUnitTestClass.cs b/MarketDataUnitTests/MarketDataUnitTestClass.cs index ea619e6..eb86d8a 100644 --- a/MarketDataUnitTests/MarketDataUnitTestClass.cs +++ b/MarketDataUnitTests/MarketDataUnitTestClass.cs @@ -428,11 +428,19 @@ public class MarketDataUnitTestClass [TestMethod] public void CompanyProfileRetrieval() { - String symbol = "MOD"; + String symbol = "AAPL"; CompanyProfile companyProfile = MarketDataHelper.GetCompanyProfile(symbol); Assert.IsTrue(null != companyProfile); } + [TestMethod] + public void CompanyProfileYahooRetrieval() + { + String symbol = "AAPL"; + CompanyProfile companyProfile = MarketDataHelper.GetCompanyProfileYahoo(symbol); + Assert.IsTrue(null != companyProfile); + } + // Test all feeds [TestMethod] public void HeadlinesRetrieval() @@ -476,7 +484,7 @@ public class MarketDataUnitTestClass Assert.IsTrue(null != timeSeries); Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.ROA), "Missing ROA"); Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.ROIC), "Missing ROIC"); - Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.BVPS), "Missing BVPS"); +// Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.BVPS), "Missing BVPS"); Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.Inventory), "Missing Inventory"); Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.AccountsReceivable), "Missing AccountsReceivable"); Assert.IsTrue(timeSeries.ContainsKey(TimeSeriesElement.ElementType.COGS), "Missing COGS"); @@ -590,7 +598,7 @@ public class MarketDataUnitTestClass Assert.IsTrue(!double.IsNaN(fundamental.High52), "High52"); Assert.IsTrue(!double.IsNaN(fundamental.Volume), "Volume"); Assert.IsTrue(!double.IsNaN(fundamental.MarketCap), "MarketCap"); - Assert.IsTrue(!double.IsNaN(fundamental.PE), "PE"); +// Assert.IsTrue(!double.IsNaN(fundamental.PE), "PE"); Assert.IsTrue(!double.IsNaN(fundamental.EPS), "EPS"); Assert.IsTrue(!double.IsNaN(fundamental.PEG), "PEG"); Assert.IsTrue(!double.IsNaN(fundamental.ReturnOnAssets), "ReturnOnAssets"); @@ -608,7 +616,7 @@ public class MarketDataUnitTestClass // Assert.IsTrue(!double.IsNaN(fundamental.OperatingCashflow), "OperatingCashflow"); // Assert.IsTrue(!double.IsNaN(fundamental.LeveragedFreeCashflow), "LeveragedFreeCashflow"); Assert.IsTrue(!double.IsNaN(fundamental.Equity), "Equity"); - Assert.IsTrue(!double.IsNaN(fundamental.TrailingPE), "TrailingPE"); + // Assert.IsTrue(!double.IsNaN(fundamental.TrailingPE), "TrailingPE"); Assert.IsTrue(!double.IsNaN(fundamental.EnterpriseValue), "EnterpriseValue"); Assert.IsTrue(!double.IsNaN(fundamental.EBIT), "EBIT"); Assert.IsTrue(!double.IsNaN(fundamental.DebtToEquity), "DebtToEquity");