ARM64/Translate/Translate/Services/MainService.cs

using Translate.Interface;
using Microsoft.Extensions.Configuration;
using System.Text;
using MarketData.Utils;

namespace Translate.Services
{
  /// <summary>
  /// This program performs translations from a MySql dump file to a file that can be imported by MariaDb.
  /// The main purpose is to translate 'utf8mb4_0900_ai_ci' embeddings in the MySql dump file to 'utf8mb4_unicode_ci'
  /// which is what MariaDb can recognize.   
  /// The import includes MySql, market_data, portfolio_data, and user_data databases.
  /// It also has some logic for options I used for debugging like skipping the insert statements and limting the numnber of output lines
  /// </summary>
  public class MainService : IMainService
  {
    public void RunService(String[] args,IConfiguration configuration)
    {
      Profiler profiler = new Profiler();

      try
      {
        Translate(args, configuration);
      }
      catch(Exception exception)
      {
        Console.WriteLine($"{exception.ToString()}");
      }
      finally
      {
        Console.WriteLine($"Translate done, total took {Utility.FormatNumber(profiler.End(),0,true)}(ms)");        
      }
    }

    private void Translate(String[] args,IConfiguration configuration)
    {
      bool includeInserts = true;
      bool useMaxLines = false;
      long maxLines = 2000;
      int writerBufferSize=65536;       // the write buffer size for the output stream

      String strTargetItem="utf8mb4_0900_ai_ci";    
      String strReplacementItem="utf8mb4_general_ci";   // "utf8mb4_unicode_ci"

      if(2!=args.Length)
      {
        Console.WriteLine("Incorrect number of arguments. ./translate input_file output_file");
        return;
      }

      Console.WriteLine($"Replacing {strTargetItem} with {strReplacementItem}");

      Console.WriteLine("Input: {0}, Output:{1}, correct Y/N?",args[0],args[1]);
      String response = Console.ReadLine();
      if(!"YES".Equals(response,StringComparison.InvariantCultureIgnoreCase))return;

      String pathInput = args[0];
      String pathOutput=args[1];


      if(!File.Exists(pathInput))
      {
        Console.WriteLine("Input file not found.");
        return;
      }

      if(File.Exists(pathOutput))
      {
        Console.WriteLine($"Deleting {pathOutput}");
        File.Delete(pathOutput);
      }

// if encounter these in the input then stop
      String[] unexpectedLines =
      {
        "-- Current Database: `performance_schema`",
        "-- Current Database: `information_schema`",
        "-- Current Database: `sys`",
      };

      StreamReader streamReader = new StreamReader(new FileStream(pathInput,FileMode.Open));
      StreamWriter streamWriter = new StreamWriter(new FileStream(pathOutput,FileMode.CreateNew),Encoding.UTF8,writerBufferSize);

      streamWriter.WriteLine("SET SQL_LOG_BIN = 0;");  // disable logging

      bool writing=true;
      for(long count=0;true;count++)
      {
        String strLine = streamReader.ReadLine();
        if(null==strLine)break;

        if(!includeInserts && strLine.Contains("INSERT INTO"))
        {
          continue;
        }

        if(strLine.StartsWith("CHANGE MASTER TO MASTER_LOG_FILE='EUROPA-bin.000230', MASTER_LOG_POS=9484249;"))
        {
          continue;
        }

        if(strLine.StartsWith("-- Current Database:"))
        {
          Console.WriteLine(strLine);
        }

        if(unexpectedLines.Any(x => strLine.StartsWith(x)))
        {
          Console.WriteLine("Found unexpected start of line:{0}",strLine);
          break;
        }

        if(strLine.StartsWith("-- Current Database: `mysql`"))
        {
          Console.WriteLine("skipping mysql");
          writing=false;
          continue;
//          writing=true;
        }
        else if(strLine.StartsWith("-- Current Database: `market_data`"))
        {
          writing=true;
        }

        if(strLine.Contains(strTargetItem))
        {
          strLine = strLine.Replace(strTargetItem, strReplacementItem);
        }

        if(writing)
        {
          streamWriter.WriteLine(strLine);
        }

        if(0==(count%500))
        {
          Console.WriteLine("Wrote {0}",count);
        }

        if(useMaxLines && count>maxLines)
        {
          break;
        }
      }

      streamReader.Close();
      streamReader.Dispose();
      streamWriter.Flush();
      streamWriter.Close();
      streamWriter.Dispose();
   }
  }
}