From ce90d2060e8ab1e5ad2d97b4d9c3c1f4a757d719 Mon Sep 17 00:00:00 2001 From: Sean Date: Sat, 12 Apr 2025 11:25:50 -0400 Subject: [PATCH] Fix update etf holdings --- MarketData/MarketData/Services/MainService.cs | 9 ++++++++- .../MarketDataLib/DataAccess/ETFHoldingsDA.cs | 15 ++++++++++++--- .../MarketDataLib/Helper/MarketDataHelper.cs | 15 ++++++++++----- MarketData/MarketDataLib/Utility/FeedParser.cs | 9 +++++++++ 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/MarketData/MarketData/Services/MainService.cs b/MarketData/MarketData/Services/MainService.cs index 6637012..556437b 100755 --- a/MarketData/MarketData/Services/MainService.cs +++ b/MarketData/MarketData/Services/MainService.cs @@ -31,6 +31,7 @@ namespace MarketData.Services MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST /WATCHLIST:"); MDTrace.WriteLine(LogLevel.DEBUG,"UPDATECOMPANYPROFILES"); MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEGDPPERCAPITA"); + MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEETFHOLDINGS"); MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)"); MDTrace.WriteLine(LogLevel.DEBUG,"MGSHSESSION /SESSIONFILE:"); MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNBACKTEST /USEHEDGING: /HEDGEINITIALCASH: /USESTOPLIMITS: /KEEPSLOTPOSITIONS: /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /SESSIONFILE: "); @@ -59,6 +60,7 @@ namespace MarketData.Services tasks.Add("UPDATESECFILINGSWATCHLIST",TaskUpdateSECFilingsWatchList); tasks.Add("UPDATECOMPANYPROFILES",TaskUpdateCompanyProfiles); tasks.Add("UPDATEGDPPERCAPITA",TaskUpdateGDPPerCapita); + tasks.Add("UPDATEETFHOLDINGS",TaskUpdateETFHoldings); tasks.Add("MGSHSESSION",TaskMGSHSession); tasks.Add("MGSHRUNBACKTEST",TaskMGSHRunBacktest); tasks.Add("MGSHRUNDAILY",TaskMGSHRunDaily); @@ -119,6 +121,11 @@ namespace MarketData.Services // ********************************************************************************************************************************************************** // ********************************************************* T A S K S ************************************************************************************* // ********************************************************************************************************************************************************** + public async Task TaskUpdateETFHoldings(CommandArgs commandArgs) + { + GetETFHoldings(); + await Task.FromResult(true); + } public async Task TaskUpdateGDPPerCapita(CommandArgs commandArgs) { @@ -678,7 +685,7 @@ namespace MarketData.Services } finally { - MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)"); + MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)"); } } diff --git a/MarketData/MarketDataLib/DataAccess/ETFHoldingsDA.cs b/MarketData/MarketDataLib/DataAccess/ETFHoldingsDA.cs index eff190c..feb9356 100755 --- a/MarketData/MarketDataLib/DataAccess/ETFHoldingsDA.cs +++ b/MarketData/MarketDataLib/DataAccess/ETFHoldingsDA.cs @@ -127,11 +127,20 @@ namespace MarketData.DataAccess ETFHolding etfHolding = etfHoldings[index]; StringBuilder sb = new StringBuilder(); sb.Append("insert into ETFHoldings(etf_symbol,holding_symbol,holding_symbol_sc,pcnt_of_assets,company,modified)values("); + sb.Append(SqlUtils.ToSqlString(etfHolding.ETFSymbol)).Append(","); + sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbol)).Append(","); - sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbolShareClass)).Append(","); - sb.Append(etfHolding.PercentOfAssets).Append(","); - sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingCompanyName)).Append(","); + + if(null==etfHolding.HoldingSymbolShareClass)sb.Append("null").Append(","); + else sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbolShareClass)).Append(","); + + if(double.IsNaN(etfHolding.PercentOfAssets))sb.Append("null").Append(","); + else sb.Append(etfHolding.PercentOfAssets).Append(","); + + if(null==etfHolding.HoldingCompanyName)sb.Append("null").Append(","); + else sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingCompanyName)).Append(","); + sb.Append(SqlUtils.ToSqlString(Utility.DateTimeToStringYYYYHMMHDD(modified))); sb.Append(")"); strQuery = sb.ToString(); diff --git a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs index 2b89288..b1ddb6e 100755 --- a/MarketData/MarketDataLib/Helper/MarketDataHelper.cs +++ b/MarketData/MarketDataLib/Helper/MarketDataHelper.cs @@ -1321,14 +1321,13 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } - List sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table"); if(null == sections || 1!=sections.Count) { etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString); if(null==etfHoldings) { - MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response."); + MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret the response string."); return null; } return etfHoldings; @@ -1375,11 +1374,13 @@ namespace MarketData.Helper } } +//
private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString) { ETFHoldings etfHoldings = new ETFHoldings(); try { + WriteToDisk(responseString,$"{etfSymbol}_etfholdings.txt"); DateTime modified = DateTime.Now; int groupBy=3; List sections = Sections.GetAllItemsInSections(responseString,"section"); @@ -1405,13 +1406,17 @@ namespace MarketData.Helper else { ETFHolding etfHolding = new ETFHolding(); + if(index+2>=spans.Count())continue; + List subSectionsSymbol = Sections.GetSections(spans[index]); + List subSectionsPercentOfAssets = Sections.GetSections(spans[index+2]); etfHolding.ETFSymbol = etfSymbol; etfHolding.HoldingSymbolShareClass = null; etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<"); - etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<"); + if(null!=subSectionsSymbol && subSectionsSymbol.Count>0)etfHolding.HoldingSymbol = subSectionsSymbol.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault(); if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue; - etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<")); + etfHolding.PercentOfAssets = FeedParser.ParseValue(subSectionsPercentOfAssets.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault()); etfHolding.Modified = modified; + if(FeedParser.IsNumeric(etfHolding.HoldingSymbol))continue; etfHoldings.Add(etfHolding); } } @@ -1419,7 +1424,7 @@ namespace MarketData.Helper } catch(Exception exception) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString())); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Symbol:{0} Exception:{1}",etfSymbol,exception.ToString())); return null; } } diff --git a/MarketData/MarketDataLib/Utility/FeedParser.cs b/MarketData/MarketDataLib/Utility/FeedParser.cs index a29951e..e22fe62 100755 --- a/MarketData/MarketDataLib/Utility/FeedParser.cs +++ b/MarketData/MarketDataLib/Utility/FeedParser.cs @@ -303,6 +303,15 @@ namespace MarketData.Utils return double.NaN; } } + + public static bool IsNumeric(String strText) + { + strText=strText.Replace("%", ""); + strText = strText.Replace("$", ""); + double value = 0.00; + return double.TryParse(strText, out value); + } + public static long ParseValueLong(String strText) { long value;