using System; using System.Collections.Generic; using System.Collections; using System.Text; using System.IO; using MySql.Data.MySqlClient; using System.Data; namespace CopyDb { public class Constants { public static String IBD_EXTENSION=".ibd"; public static int COMMAND_TIMEOUT=0; // no timeout public static string CHARACTER_SET="utf8mb4"; public static string COLLATION="utf8mb4_0900_ai_ci"; } public class Schemas : List { public void Display() { for(int index=0;index(); HasForeignKey=false; } public List SchemaLines{get;set;} public String TableName{get;set;} public bool HasForeignKey{get;set;} public String ForeignKeyTable{get;set;} public String ForeignKeyConstraint{get;set;} } // ****************************************************************** public class Config { public String Server{get;set;} public int Port{get;set;} public String Database{get;set;} public String User{get;set;} public String Password{get;set;} public String PathSchemaFileName{get;set;} public String PathIbd{get;set;} public String Mode{get;set;} public String DatabaseDirectory{get;set;} public String CharacterSet{get;set;} public String Collation{get;set;} public void Display() { Console.WriteLine(String.Format("SERVER:{0}",Server)); Console.WriteLine(String.Format("PORT:{0}",Port)); Console.WriteLine(String.Format("DATABASE:{0}",Database)); Console.WriteLine(String.Format("USER:{0}",User)); Console.WriteLine(String.Format("PASSWORD:{0}",Password)); Console.WriteLine(String.Format("PATHSCHEMAFILE:{0}",PathSchemaFileName)); Console.WriteLine(String.Format("PATHIBD:{0}",PathIbd)); Console.WriteLine(String.Format("MODE:{0}",Mode)); Console.WriteLine(String.Format("DATABASEDIRECTORY:{0}",DatabaseDirectory)); Console.WriteLine(String.Format("CHARACTERSET:{0}",CharacterSet)); Console.WriteLine(String.Format("COLLATION:{0}",Collation)); } } class Program { static void Main(string[] args) { if(1==args.Length) { Console.WriteLine(String.Format("USAGE: COPYDB /SERVER:{server} /PORT:{port} /ROWCOUNTSONLY:{true|false} /CONVERTCHARSETCOLLATION:{charset@collation} /DATABASE:{databasename} /USER:{username} /PASSWORD:{password} /PATHSCHEMAFILE:{pathschemafilename} /PATHIDB:{pathibdfiles} /MODE:{LIVE|TEST} /DATABASEDIRECTORY:{path}")); return; } PerformImport(args); } public static void PerformImport(string[] args) { String strResult=null; CommandArgs commandArgs=new CommandArgs(args); Console.WriteLine("*********** C O N F I G ***********"); Config config=GetConfig(commandArgs); config.Display(); if(!commandArgs.Has("SERVER,PORT,DATABASE,USER,PASSWORD,PATHSCHEMAFILE")) { Console.WriteLine(String.Format("One or more required parameters are missing. SERVER,PORT,DATABASE,USER,PASSWORD,PATHSCHEMAFILE")); return; } // ************************************************************* S C H E M A ******************************************************* Schemas schemas=Program.GetSchemas(config); if(null==schemas) { Console.WriteLine(String.Format("Unable to read schemas")); return; } Console.WriteLine("*********** S C H E M A ***********"); schemas.Display(); // *********************************************************** D A T A B A S E ******************************************************** Console.WriteLine("*********** V E R I F Y I N G D A T A B A S E ***********"); MySqlConnection sqlConnection=null; try{ sqlConnection=SqlUtils.CreateMySqlConnection(config.Server,config.Port.ToString(),config.Database,config.User,config.Password);} catch(Exception exception) { Console.WriteLine(String.Format("Encountered Exception: {0}",exception.ToString())); return; } ConnectionState connectionState=sqlConnection.State; if(!connectionState.Equals(ConnectionState.Open)) { Console.WriteLine(String.Format("Unable to connect to databse {0}",config.Database)); return; } Console.WriteLine("Database connection verified."); // ***************************************************** T A B L E R O W C O U N T S*********************************************************** GetTableRowCounts(schemas, config, sqlConnection); if(commandArgs.Has("ROWCOUNTSONLY") && commandArgs.Get("ROWCOUNTSONLY").Equals(true)) { Console.WriteLine("Exiting because ROWCOUNTSONLY is set to True."); return; } // ********************************************************************************************************************************************* // (i.e.) CONVERTCHARSETCOLLATION:utf8mb4@utf8mb4_0900_ai_ci if(commandArgs.Has("CONVERTCHARSETCOLLATION")) { Console.WriteLine("**WARNING** This is a destructive event. This process will convert tables in {0}",config.Database); Console.WriteLine("Please review settings. Proceed Y/N?"); strResult=Console.ReadLine(); if(null==strResult||(!strResult.ToUpper().Equals("Y") && !strResult.ToUpper().Equals("YES"))) { Console.WriteLine("Cancelled"); return; } ConvertSchema(schemas ,config, sqlConnection); GetTableRowCounts(schemas, config, sqlConnection); Console.WriteLine("Done."); Console.ReadLine(); return; } if(!commandArgs.Has("PATHIBD,MODE,DATABASEDIRECTORY")) { Console.WriteLine(String.Format("One or more required parameters are missing.")); return; } Console.WriteLine("*********** V E R I F Y I N G I D B F I L E S ***********"); if(!VerifyIdbFiles(schemas,config)) { Console.WriteLine("Unable to verify IDBFiles"); return; } if(config.Mode.ToUpper().Equals("TEST")) { Console.WriteLine("Mode is TEST."); sqlConnection.Close(); sqlConnection.Dispose(); return; } Console.WriteLine("**WARNING** This is a destructive event. This process will drop tables in {0}",config.Database); Console.WriteLine("Please review settings. Proceed Y/N?"); strResult=Console.ReadLine(); if(null==strResult||(!strResult.ToUpper().Equals("Y") && !strResult.ToUpper().Equals("YES"))) { Console.WriteLine("Cancelled"); return; } Console.WriteLine("*********** I M P O R T ***********"); foreach(Schema schema in schemas) { ImportTable(schema ,config, sqlConnection); } sqlConnection.Close(); sqlConnection.Dispose(); Console.WriteLine("Done."); return; } // ******************************************************************************************************************************************************************************** // *************************************************************************** C O N V E R T C H A R A C T E R S E T *********************************************************** // ******************************************************************************************************************************************************************************** private static void ConvertSchema(Schemas schemas, Config config, MySqlConnection sqlConnection) { MySqlCommand sqlCommand=null; String strSetKeyChecksTrue="SET foreign_key_checks =1;"; String strSetKeyChecksFalse="SET foreign_key_checks =0;"; try { Console.WriteLine(String.Format("{0}",strSetKeyChecksFalse)); sqlCommand = new MySqlCommand(strSetKeyChecksFalse, sqlConnection); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); foreach(Schema schema in schemas) { StringBuilder sb=new StringBuilder(); sb.Append("ALTER TABLE ").Append(schema.TableName).Append(" CONVERT TO CHARACTER SET ").Append(config.CharacterSet).Append(" COLLATE ").Append(config.Collation).Append(" ;"); String strQuery=sb.ToString(); Console.WriteLine(String.Format(strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); if(!VerifyCharSetCollation(schema.TableName, config, sqlConnection)) { Console.WriteLine(String.Format("FAILED to change character set / collation on table {0}",schema.TableName)); } else Console.WriteLine(String.Format("VERIFIED")); } Console.WriteLine(String.Format("{0}",strSetKeyChecksTrue)); Console.WriteLine(""); sqlCommand = new MySqlCommand(strSetKeyChecksTrue, sqlConnection); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); } catch(Exception exception) { if(null!=sqlCommand) { sqlCommand.Dispose(); sqlCommand=null; } Console.WriteLine("Exception converting character set :{0}",exception.ToString()); } } // ******************************************************************************************************************************************************************************** // *************************************************************************** V E R I F Y C H A R A C T E R S E T *********************************************************** // ******************************************************************************************************************************************************************************** private static bool VerifyCharSetCollation(String tableName, Config config, MySqlConnection sqlConnection) { MySqlCommand sqlCommand=null; MySqlDataReader sqlReader=null; bool verified=true; try { StringBuilder sb=new StringBuilder(); sb.Append("SHOW FULL COLUMNS FROM ").Append(tableName).Append(" ;"); sqlCommand=sqlConnection.CreateCommand(); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; String strQuery=sb.ToString(); Console.WriteLine(String.Format(strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection); sqlReader=sqlCommand.ExecuteReader(); while(sqlReader.Read()) { String field=sqlReader.IsDBNull(0)?null:sqlReader.GetString(0); String type=sqlReader.IsDBNull(1)?null:sqlReader.GetString(1).ToLower(); String collation=sqlReader.IsDBNull(2)?null:sqlReader.GetString(2).ToLower(); if(!type.StartsWith("varchar"))continue; if(!collation.Equals(config.Collation))verified=false; } sqlReader.Close(); sqlReader.Dispose(); sqlCommand.Dispose(); sqlCommand=null; sqlReader=null; return verified; } catch(Exception exception) { Console.WriteLine("Exception converting character set :{0}",exception.ToString()); if(null!=sqlReader) { sqlReader.Close(); sqlReader.Dispose(); } if(null!=sqlCommand) { sqlCommand.Dispose(); sqlCommand=null; } return false; } } // ******************************************************************************************************************************************************************************** // ******************************************************************************** T A B L E R O W C O U N T S *********************************************************************** // ******************************************************************************************************************************************************************************** private static void GetTableRowCounts(Schemas schemas, Config config, MySqlConnection sqlConnection) { MySqlCommand sqlCommand=null; MySqlDataReader sqlReader=null; try { foreach(Schema schema in schemas) { StringBuilder sb=new StringBuilder(); sb.Append("SELECT COUNT(*) FROM ").Append(schema.TableName).Append(" ;"); sqlCommand=sqlConnection.CreateCommand(); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; String strQuery=sb.ToString(); sqlCommand = new MySqlCommand(strQuery, sqlConnection); sqlReader=sqlCommand.ExecuteReader(); while(sqlReader.Read()) { int rows=sqlReader.GetInt32(0); Console.WriteLine(String.Format("{0},{1}",schema.TableName,Utility.FormatNumber(rows,0,false))); } sqlReader.Close(); sqlReader.Dispose(); sqlCommand.Dispose(); sqlCommand=null; sqlReader=null; } } catch(Exception exception) { Console.WriteLine("Exception:{0}",exception.ToString()); if(null!=sqlReader) { sqlReader.Close(); sqlReader.Dispose(); } if(null!=sqlCommand) { sqlCommand.Dispose(); sqlCommand=null; } } } // ***************************************************************************************************************************************************** // *************************************************************** I M P O R T T A B L E ************************************************************** // ***************************************************************************************************************************************************** private static void ImportTable(Schema schema, Config config,MySqlConnection sqlConnection) { MySqlTransaction sqlTransaction=null; String pathIdbFile=null; String pathIdbFileDestination=null; try { MySqlCommand sqlCommand=null; String strQuery=null; StringBuilder sb=null; pathIdbFile=config.PathIbd+"\\"+schema.TableName+".ibd"; pathIdbFileDestination=config.DatabaseDirectory+"\\"+schema.TableName+".ibd"; sqlTransaction=sqlConnection.BeginTransaction(IsolationLevel.RepeatableRead); sb=new StringBuilder(); sb.Append("DROP TABLE IF EXISTS ").Append(schema.TableName).Append(";"); sqlCommand=sqlConnection.CreateCommand(); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; strQuery=sb.ToString(); Console.WriteLine(String.Format(strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); sqlCommand=sqlConnection.CreateCommand(); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; strQuery=Utility.ListToString(schema.SchemaLines,' '); Console.WriteLine(String.Format(schema.SchemaLines[0])); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); int result=sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); sb=new StringBuilder(); sb.Append("ALTER TABLE ").Append(schema.TableName).Append(" DISCARD TABLESPACE;"); strQuery=sb.ToString(); Console.WriteLine(String.Format("{0}",strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; result=sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); Console.WriteLine(String.Format("Copy file {0} -> {1}",pathIdbFile,pathIdbFileDestination)); File.Copy(pathIdbFile,pathIdbFileDestination,true); sb=new StringBuilder(); sb.Append("ALTER TABLE ").Append(schema.TableName).Append(" IMPORT TABLESPACE;"); strQuery=sb.ToString(); Console.WriteLine(String.Format("{0}",strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); try { sb=new StringBuilder(); sb.Append("ALTER TABLE ").Append(schema.TableName).Append(" CONVERT TO CHARACTER SET ").Append(Constants.CHARACTER_SET).Append(" COLLATE ").Append(Constants.COLLATION).Append(";"); strQuery=sb.ToString(); Console.WriteLine(String.Format("{0}",strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); } catch(Exception exception) { Console.WriteLine("Exception converting character set :{0}",exception.ToString()); } if(schema.HasForeignKey) { sb=new StringBuilder(); sb.Append("ALTER TABLE ").Append(schema.TableName).Append(" ADD ").Append(schema.ForeignKeyConstraint).Append(";"); strQuery=sb.ToString(); Console.WriteLine(String.Format("{0}",strQuery)); sqlCommand = new MySqlCommand(strQuery, sqlConnection, sqlTransaction); sqlCommand.CommandTimeout=Constants.COMMAND_TIMEOUT; sqlCommand.ExecuteNonQuery(); sqlCommand.Dispose(); } sqlTransaction.Commit(); Console.WriteLine(""); } catch(Exception exception) { Console.WriteLine(String.Format("Exception:{0}",exception.ToString())); try{sqlTransaction.Rollback();} catch(Exception rollbackException) { Console.WriteLine(String.Format("Rollback exception:{0}",rollbackException.ToString())); } } } private static bool VerifyIdbFiles(Schemas schemas,Config config) { bool success=true; foreach(Schema schema in schemas) { String pathIdbFile=config.PathIbd+"\\"+schema.TableName+Constants.IBD_EXTENSION;; Console.Write(String.Format("{0}",pathIdbFile)); if(!File.Exists(pathIdbFile)) { Console.WriteLine("...Not found."); success=false; } Console.WriteLine("...Verified"); } return success; } private static Schemas GetSchemas(Config config) { Schemas schemas=new Schemas(); try { FileStream inStream = new FileStream(config.PathSchemaFileName,FileMode.Open); StreamReader streamReader=new StreamReader(inStream); String strLine=null; bool inCreate=false; Schema schema=null; while(null!=(strLine=streamReader.ReadLine())) { if(strLine.StartsWith("CREATE TABLE") && !inCreate) { schema=new Schema(); schema.TableName=Utility.BetweenString(strLine,"`","`"); schema.SchemaLines.Add(strLine); inCreate=true; } else if(inCreate) { // schema.SchemaLines.Add(strLine); if(strLine.Contains("FOREIGN KEY")) { // schema.SchemaLines.Add(strLine); schema.ForeignKeyConstraint=strLine; schema.HasForeignKey=true; String[] elements=strLine.Split(' '); schema.ForeignKeyTable=GetForeignKeyTable(elements); } else if(strLine.Trim().EndsWith(";")) { // schema.SchemaLines.Add(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;"); if(schema.SchemaLines[schema.SchemaLines.Count-1].EndsWith(",")) { String line=schema.SchemaLines[schema.SchemaLines.Count-1]; line=line.Substring(0,line.Length-1); // FUNNY schema.SchemaLines[schema.SchemaLines.Count-1]=line; } schema.SchemaLines.Add(strLine); inCreate=false; schemas.Add(schema); schema=null; } else { schema.SchemaLines.Add(strLine); } } } streamReader.Close(); streamReader.Dispose(); inStream.Close(); inStream.Dispose(); OrderSchemas(schemas); return schemas; } catch(Exception exception) { Console.WriteLine(String.Format("Exception:{0}",exception.ToString())); return null; } } private static Schemas OrderSchemas(Schemas schemas) { for(int index=0;indexforeignKeyIndex)continue; SwapSchemaPositions(schemas,index,foreignKeyIndex); index=-1; continue; } } return schemas; } private static void SwapSchemaPositions(Schemas schemas,int schemaIndex,int foreignKeySchemaIndex) { Schema tempSchema=schemas[schemaIndex]; schemas[schemaIndex]=schemas[foreignKeySchemaIndex]; schemas[foreignKeySchemaIndex]=tempSchema; } private static int FindSchemaLocation(Schemas schemas,String tableName) { for(int index=0;index("SERVER"); config.Port=commandArgs.Get("PORT"); config.Database=commandArgs.Get("DATABASE"); config.User=commandArgs.Get("USER"); config.Password=commandArgs.Get("PASSWORD"); config.PathSchemaFileName=commandArgs.Get("PATHSCHEMAFILE"); config.PathIbd=commandArgs.Get("PATHIBD"); config.Mode=commandArgs.Get("MODE"); config.DatabaseDirectory=commandArgs.Get("DATABASEDIRECTORY"); if(commandArgs.Has("CONVERTCHARSETCOLLATION")) { String[] charsetcollation=commandArgs.Get("CONVERTCHARSETCOLLATION").Split('@'); config.CharacterSet=charsetcollation[0]; config.Collation=charsetcollation[1]; } return config; } } }