Fix update etf holdings

MarketData/MarketData/Services/MainService.cs

@@ -31,6 +31,7 @@ namespace MarketData.Services
       MDTrace.WriteLine(LogLevel.DEBUG,"UPDATESECFILINGSWATCHLIST /WATCHLIST:");   
       MDTrace.WriteLine(LogLevel.DEBUG,"UPDATECOMPANYPROFILES");
       MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEGDPPERCAPITA");
+      MDTrace.WriteLine(LogLevel.DEBUG,"UPDATEETFHOLDINGS");
       MDTrace.WriteLine(LogLevel.DEBUG,"ECHO {param1} {param2} {param(n)");
       MDTrace.WriteLine(LogLevel.DEBUG,"MGSHSESSION /SESSIONFILE:");
       MDTrace.WriteLine(LogLevel.DEBUG,"MGSHRUNBACKTEST /USEHEDGING: /HEDGEINITIALCASH: /USESTOPLIMITS: /KEEPSLOTPOSITIONS: /STARTDATE: /MAXPOSITIONS: /INITIALCASH: /HOLDINGPERIOD: /{ENDDATE}: /SESSIONFILE: ");
@@ -59,6 +60,7 @@ namespace MarketData.Services
       tasks.Add("UPDATESECFILINGSWATCHLIST",TaskUpdateSECFilingsWatchList);
       tasks.Add("UPDATECOMPANYPROFILES",TaskUpdateCompanyProfiles);
       tasks.Add("UPDATEGDPPERCAPITA",TaskUpdateGDPPerCapita);
+      tasks.Add("UPDATEETFHOLDINGS",TaskUpdateETFHoldings);
       tasks.Add("MGSHSESSION",TaskMGSHSession);
       tasks.Add("MGSHRUNBACKTEST",TaskMGSHRunBacktest);
       tasks.Add("MGSHRUNDAILY",TaskMGSHRunDaily);
@@ -119,6 +121,11 @@ namespace MarketData.Services
 // **********************************************************************************************************************************************************
 // ********************************************************* T A S K S  *************************************************************************************
 // **********************************************************************************************************************************************************
+    public async Task TaskUpdateETFHoldings(CommandArgs commandArgs)
+    {
+      GetETFHoldings();
+      await Task.FromResult(true);
+    }
 
     public async Task TaskUpdateGDPPerCapita(CommandArgs commandArgs)
     {
@@ -678,7 +685,7 @@ namespace MarketData.Services
       }
       finally
       {
-        MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End}(ms)");
+        MDTrace.WriteLine(LogLevel.DEBUG,$"Done, total took {profiler.End()}(ms)");
       }
     }
 

MarketData/MarketDataLib/DataAccess/ETFHoldingsDA.cs

@@ -127,11 +127,20 @@ namespace MarketData.DataAccess
           ETFHolding etfHolding = etfHoldings[index];
           StringBuilder sb = new StringBuilder();
           sb.Append("insert into ETFHoldings(etf_symbol,holding_symbol,holding_symbol_sc,pcnt_of_assets,company,modified)values(");
+
           sb.Append(SqlUtils.ToSqlString(etfHolding.ETFSymbol)).Append(",");
+
           sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbol)).Append(",");
-          sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbolShareClass)).Append(",");
-          sb.Append(etfHolding.PercentOfAssets).Append(",");
-          sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingCompanyName)).Append(",");
+
+          if(null==etfHolding.HoldingSymbolShareClass)sb.Append("null").Append(",");
+          else sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingSymbolShareClass)).Append(",");
+
+          if(double.IsNaN(etfHolding.PercentOfAssets))sb.Append("null").Append(",");
+          else sb.Append(etfHolding.PercentOfAssets).Append(",");
+
+          if(null==etfHolding.HoldingCompanyName)sb.Append("null").Append(",");
+          else sb.Append(SqlUtils.ToSqlString(etfHolding.HoldingCompanyName)).Append(",");
+          
           sb.Append(SqlUtils.ToSqlString(Utility.DateTimeToStringYYYYHMMHDD(modified)));
           sb.Append(")");
           strQuery = sb.ToString();

MarketData/MarketDataLib/Helper/MarketDataHelper.cs

@@ -1321,14 +1321,13 @@ namespace MarketData.Helper
           MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode));
           return null;
         }
-
         List<String> sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table");
         if(null == sections || 1!=sections.Count)
         {
           etfHoldings=TryParseYahooFinanceETFHoldings(etfSymbol,httpNetResponse.ResponseString);
           if(null==etfHoldings)
           {
-            MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret response.");
+            MDTrace.WriteLine(LogLevel.DEBUG,"GetETFHoldings: Unable to interpret the response string.");
             return null;
           }
           return etfHoldings;
@@ -1375,11 +1374,13 @@ namespace MarketData.Helper
       }
     }
 
+// <section data-testid="top-holdings" class="yf-1hj9jti">
     private static ETFHoldings TryParseYahooFinanceETFHoldings(String etfSymbol,String responseString)
     {
       ETFHoldings etfHoldings = new ETFHoldings();
       try
       {
+        WriteToDisk(responseString,$"{etfSymbol}_etfholdings.txt");
         DateTime modified = DateTime.Now;
         int groupBy=3;
         List<String> sections = Sections.GetAllItemsInSections(responseString,"section");
@@ -1405,13 +1406,17 @@ namespace MarketData.Helper
           else
           {
             ETFHolding etfHolding = new ETFHolding();
+            if(index+2>=spans.Count())continue;
+            List<String> subSectionsSymbol = Sections.GetSections(spans[index]);
+            List<String> subSectionsPercentOfAssets = Sections.GetSections(spans[index+2]);
             etfHolding.ETFSymbol = etfSymbol;
             etfHolding.HoldingSymbolShareClass = null;
             etfHolding.HoldingCompanyName = Utility.BetweenString(spans[index+1],">","<");
-            etfHolding.HoldingSymbol = etfHolding.HoldingSymbolShareClass = Utility.BetweenString(spans[index],">","<");
+            if(null!=subSectionsSymbol && subSectionsSymbol.Count>0)etfHolding.HoldingSymbol = subSectionsSymbol.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault();
             if (null == etfHolding.HoldingSymbol || "N/A".Equals(etfHolding.HoldingSymbol) || "".Equals(etfHolding.HoldingSymbol)) continue;
-            etfHolding.PercentOfAssets = FeedParser.ParseValue(Utility.BetweenString(spans[index+2],">","<"));
+            etfHolding.PercentOfAssets = FeedParser.ParseValue(subSectionsPercentOfAssets.Where(x => !String.IsNullOrEmpty(x)).FirstOrDefault());
             etfHolding.Modified = modified;
+            if(FeedParser.IsNumeric(etfHolding.HoldingSymbol))continue;
             etfHoldings.Add(etfHolding);
           }
         }
@@ -1419,7 +1424,7 @@ namespace MarketData.Helper
        }
        catch(Exception exception)
        {
-         MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Exception: {0}",exception.ToString()));
+         MDTrace.WriteLine(LogLevel.DEBUG,String.Format("[TryParseYahooFinanceETFHoldings] Symbol:{0} Exception:{1}",etfSymbol,exception.ToString()));
          return null;
        }
     }

MarketData/MarketDataLib/Utility/FeedParser.cs

@@ -303,6 +303,15 @@ namespace MarketData.Utils
         return double.NaN;
       }
     }
+
+    public static bool IsNumeric(String strText)
+    {
+      strText=strText.Replace("%", "");
+      strText = strText.Replace("$", "");    
+      double value = 0.00;
+      return double.TryParse(strText, out value);
+    }
+
     public static long ParseValueLong(String strText)
     {
       long value;