Add Translate project for exporting MySql into MariaDb

Translate/Translate/Executor.cs

@@ -0,0 +1,26 @@
+using Translate.Interface;
+using Microsoft.Extensions.Configuration;
+
+namespace Translate
+{
+  public class Executor
+  {
+      private readonly IConfiguration _configuration;
+      private readonly IMainService _mainService;
+      private readonly ICommandArgs _commandArgs;
+
+      public Executor(IMainService mainService,IConfigurationRoot configuration,ICommandArgs commandArgs)
+      {
+          _configuration = configuration ?? throw new ArgumentException(nameof(configuration));
+          _mainService = mainService ?? throw new ArgumentException(nameof(mainService));
+          _commandArgs = commandArgs ?? throw new ArgumentException(nameof(commandArgs));
+      }
+
+      /// <summary>
+      /// This is essentially the starting point where we bootstrap the legacy entry point 
+      /// </summary>
+      public void Execute()
+      {  _mainService.RunService(_commandArgs.GetArgs(), _configuration);
+      }
+  }
+}

Translate/Translate/Interface/ICommandArgs.cs

@@ -0,0 +1,7 @@
+namespace Translate.Interface
+{
+  public interface ICommandArgs
+  {
+    public String[] GetArgs();
+  }
+}

Translate/Translate/Interface/IMainService.cs

@@ -0,0 +1,9 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Translate.Interface
+{
+  public interface IMainService
+  {
+    public void RunService(String[] args,IConfiguration configuration);
+  }
+}

Translate/Translate/Program.cs

@@ -0,0 +1,25 @@
+using Translate.Interface;
+using Translate.Services;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Translate
+{
+  class Program
+  {
+      static void Main(string[] args)
+      {
+        IConfigurationBuilder builder = new ConfigurationBuilder()
+        .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
+        IConfigurationRoot configurationRoot = builder.Build();
+        CommandArgs commandArgs = new CommandArgs(args);
+        IServiceCollection services = new ServiceCollection();
+        services.AddSingleton<ICommandArgs>(commandArgs);
+        services.AddSingleton<IConfigurationRoot>(configurationRoot);
+        services.AddSingleton<IMainService,MainService>();
+        services.AddSingleton<Executor,Executor>();
+        services.BuildServiceProvider().GetService<Executor>().Execute();
+      }
+
+  }
+}

Translate/Translate/Services/CommandArgs.cs

@@ -0,0 +1,19 @@
+using Translate.Interface;
+
+namespace Translate.Services
+{
+  public class CommandArgs : ICommandArgs
+  {
+    private readonly String[] _args;
+
+    public CommandArgs(String[] args)
+    {
+      _args = args ?? throw new ArgumentNullException(nameof(args));
+    }
+    
+    public String[] GetArgs()
+    {
+      return _args;
+    }
+  }
+}

Translate/Translate/Services/MainService.cs

@@ -0,0 +1,136 @@
+using Translate.Interface;
+
+using Microsoft.Extensions.Configuration;
+using System.Text;
+
+namespace Translate.Services
+{
+  /// <summary>
+  /// This program performs translations from a MySql dump file to a file that can be imported by MariaDb.
+  /// The main purpose is to translate 'utf8mb4_0900_ai_ci' embeddings in the MySql dump file to 'utf8mb4_unicode_ci'
+  /// which is what MariaDb can recognize.  It also skips the MySql database and only imports the 
+  /// market_data, portfolio_data, and user_data databases.
+  /// It also has some logic for options I used for debugging like skipping the insert statements and limting the numnber of output lines
+  /// </summary>
+  public class MainService : IMainService
+  {
+    public void RunService(String[] args,IConfiguration configuration)
+    {
+      bool includeInserts = true;
+      bool useMaxLines = false;
+      long maxLines = 2000;
+      int writerBufferSize=65536;
+
+      String strTargetItem="utf8mb4_0900_ai_ci";    
+      String strReplacementItem="utf8mb4_general_ci";   // "utf8mb4_unicode_ci"
+
+      if(2!=args.Length)
+      {
+        Console.WriteLine("Incorrect number of arguments. ./translate input_file output_file");
+        return;
+      }
+
+      Console.WriteLine($"Replacing {strTargetItem} with {strReplacementItem}");
+
+
+      Console.WriteLine("Input: {0}, Output:{1}, correct Y/N?",args[0],args[1]);
+      String response = Console.ReadLine();
+      if(!"YES".Equals(response,StringComparison.InvariantCultureIgnoreCase))return;
+
+      String pathInput = args[0];
+      String pathOutput=args[1];
+
+
+      if(!File.Exists(pathInput))
+      {
+        Console.WriteLine("Input file not found.");
+        return;
+      }
+
+      if(File.Exists(pathOutput))
+      {
+        File.Delete(pathOutput);
+      }
+
+      String[] unexpectedLines =
+      {
+//        "-- Current Database: `mysql`",
+        "-- Current Database: `performance_schema`",
+        "-- Current Database: `information_schema`",
+        "-- Current Database: `sys`",
+      };
+
+      StreamReader streamReader = new StreamReader(new FileStream(pathInput,FileMode.Open));
+      StreamWriter streamWriter = new StreamWriter(new FileStream(pathOutput,FileMode.CreateNew),Encoding.UTF8,writerBufferSize);
+
+      streamWriter.WriteLine("SET SQL_LOG_BIN = 0;");  // disable logging
+
+      bool writing=true;
+      for(long count=0;true;count++)
+      {
+        String strLine = streamReader.ReadLine();
+        if(null==strLine)break;
+
+        if(!includeInserts && strLine.Contains("INSERT INTO"))
+        {
+          continue;
+        }
+
+        if(strLine.StartsWith("CHANGE MASTER TO MASTER_LOG_FILE='EUROPA-bin.000230', MASTER_LOG_POS=9484249;"))
+        {
+          continue;
+        }
+
+        if(strLine.StartsWith("-- Current Database:"))
+        {
+          Console.WriteLine(strLine);
+        }
+
+        if(unexpectedLines.Any(x => strLine.StartsWith(x)))
+        {
+          Console.WriteLine("Found unexpected start of line:{0}",strLine);
+          break;
+        }
+
+        if(strLine.StartsWith("-- Current Database: `mysql`"))
+        {
+          Console.WriteLine("skipping mysql");
+          writing=false;
+          continue;
+        }
+        else if(strLine.StartsWith("-- Current Database: `market_data`"))
+        {
+          writing=true;
+        }
+
+        if(strLine.Contains(strTargetItem))
+        {
+          strLine = strLine.Replace(strTargetItem, strReplacementItem);
+        }
+
+        if(writing)
+        {
+          streamWriter.WriteLine(strLine);
+        }
+
+        if(0==(count%500))
+        {
+          Console.WriteLine("Wrote {0}",count);
+        }
+
+        if(useMaxLines && count>maxLines)
+        {
+          break;
+        }
+      }
+
+      streamReader.Close();
+      streamReader.Dispose();
+      streamWriter.Flush();
+      streamWriter.Close();
+      streamWriter.Dispose();
+
+      Console.WriteLine("Done.");
+   }
+  }
+}

Translate/Translate/Translate.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>disable</Nullable>
+    <NoWarn>CA1416;CS8769;CS0108;CS8602;CS8601;CS8620;CS8618;CS8603;CS8767;CS8625;CS8604;CS8600;CS8604</NoWarn>    
+  </PropertyGroup>
+
+
+  <PropertyGroup>
+    <AssemblyName>translate</AssemblyName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Hosting" Version="2.3.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.3" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

Translate/Translate/appsettings.json

@@ -0,0 +1,21 @@
+{
+"market_data" : "Database=market_data;Datasource=euporie;Username=guest;Password=dbas",
+"portfolio_data" : "Database=portfolio_data;Datasource=euporie;Username=guest;Password=dbas",
+"user_data" : "Database=user_data;Datasource=euporie;Username=guest;Password=dbas",
+"sms_smtpaddress" : "smtp.gmail.com",
+"sms_smsusername" : "skessler1964@gmail.com",
+"sms_smspassword" : "xjfo isnf gmyi zovr",
+"sms_smsrecipients" : "skessler1964sms@gmail.com",
+"proxy_address" : "http://127.0.0.1:8182",
+"proxy_GetLatestPriceYahoo" : "false",
+"proxy_GetLatestPriceFidelity" : "true",
+"proxy_GetLatestPriceBigCharts" : "false",
+"proxy_GetETFHoldings" : "false",
+"proxy_GetAnalystPriceTargetYahoo" : "true",
+"proxy_GetDailyPrices" : "false",
+"proxy_GetFundamentalEx" : "false",
+"proxy_GetDividendHistory" : "false",
+"proxy_GetAnalystPriceTargetMarketBeat" : "false",
+"proxy_GetCompanyHeadlinesSeekingAlphaV1" : "true",
+"proxy_GetCompanyHeadlinesSeekingAlphaV2" : "true"
+}