Kopieren der Datenbank in SQLite datei
This commit is contained in:
@@ -140,33 +140,29 @@ func migrateCharacterClasses() error {
|
||||
return fmt.Errorf("KOD source not found: %w", err)
|
||||
}
|
||||
|
||||
characterClasses := map[string]struct {
|
||||
name string
|
||||
page int
|
||||
}{
|
||||
"As": {"Assassine", 0}, // Seitenzahlen müssen noch eingepflegt werden
|
||||
"Bb": {"Barbar", 0},
|
||||
"Gl": {"Glücksritter", 0},
|
||||
"Hä": {"Händler", 0},
|
||||
"Kr": {"Krieger", 0},
|
||||
"Sp": {"Spitzbube", 0},
|
||||
"Wa": {"Waldläufer", 0},
|
||||
"Ba": {"Barde", 0},
|
||||
"Or": {"Ordenskrieger", 0},
|
||||
"Dr": {"Druide", 0},
|
||||
"Hx": {"Hexer", 0},
|
||||
"Ma": {"Magier", 0},
|
||||
"PB": {"Priester Beschützer", 0},
|
||||
"PS": {"Priester Streiter", 0},
|
||||
"Sc": {"Schamane", 0},
|
||||
characterClasses := map[string]string{
|
||||
"As": "Assassine",
|
||||
"Bb": "Barbar",
|
||||
"Gl": "Glücksritter",
|
||||
"Hä": "Händler",
|
||||
"Kr": "Krieger",
|
||||
"Sp": "Spitzbube",
|
||||
"Wa": "Waldläufer",
|
||||
"Ba": "Barde",
|
||||
"Or": "Ordenskrieger",
|
||||
"Dr": "Druide",
|
||||
"Hx": "Hexer",
|
||||
"Ma": "Magier",
|
||||
"PB": "Priester Beschützer",
|
||||
"PS": "Priester Streiter",
|
||||
"Sc": "Schamane",
|
||||
}
|
||||
|
||||
for code, info := range characterClasses {
|
||||
for code, name := range characterClasses {
|
||||
class := models.CharacterClass{
|
||||
Code: code,
|
||||
Name: info.name,
|
||||
Name: name,
|
||||
SourceID: kodSource.ID,
|
||||
PageNumber: info.page,
|
||||
GameSystem: "midgard",
|
||||
}
|
||||
|
||||
@@ -177,7 +173,7 @@ func migrateCharacterClasses() error {
|
||||
if err := class.Create(); err != nil {
|
||||
return fmt.Errorf("failed to create character class %s: %w", code, err)
|
||||
}
|
||||
log.Printf("Created character class: %s - %s (Source: %s)", code, info.name, kodSource.Code)
|
||||
log.Printf("Created character class: %s - %s (Source: %s)", code, name, kodSource.Code)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+122
-276
@@ -43,21 +43,10 @@ func migrateAllStructures(db *gorm.DB) error {
|
||||
if err := user.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate user structures: %w", err)
|
||||
}
|
||||
/*
|
||||
if err := character.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate character structures: %w", err)
|
||||
}
|
||||
*/
|
||||
if err := models.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate gsmaster structures: %w", err)
|
||||
}
|
||||
/*if err := equipment.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate equipment structures: %w", err)
|
||||
}
|
||||
if err := skills.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate skills structures: %w", err)
|
||||
}
|
||||
*/
|
||||
|
||||
if err := importer.MigrateStructure(db); err != nil {
|
||||
return fmt.Errorf("failed to migrate importer structures: %w", err)
|
||||
}
|
||||
@@ -65,309 +54,166 @@ func migrateAllStructures(db *gorm.DB) error {
|
||||
}
|
||||
|
||||
func MakeTestdataFromLive(c *gin.Context) {
|
||||
db := database.ConnectDatabase()
|
||||
if db == nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to connect to DataBase")
|
||||
liveDB := database.ConnectDatabase()
|
||||
if liveDB == nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to connect to live database")
|
||||
return
|
||||
}
|
||||
// Setup test database
|
||||
var testDb *gorm.DB
|
||||
var err error
|
||||
|
||||
testDb, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
// Live-Datenbank in SQLite-Datei kopieren
|
||||
backupFile := preparedTestDB
|
||||
err := copyLiveDatabaseToFile(liveDB, backupFile)
|
||||
if err != nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to connect to test database testDb: "+err.Error())
|
||||
respondWithError(c, http.StatusInternalServerError, fmt.Sprintf("Failed to copy database: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Step 1: Migrate all structures to test database
|
||||
err = migrateAllStructures(testDb)
|
||||
if err != nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to migrate structures to test DB: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Step 2: Copy all data from live database to test database
|
||||
copyStats, err := copyAllDataToTestDB(db, testDb)
|
||||
if err != nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to copy data to test DB: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Step 3: Save test database to file for reuse
|
||||
err = saveTestDatabaseToFile(testDb, preparedTestDB)
|
||||
if err != nil {
|
||||
respondWithError(c, http.StatusInternalServerError, "Failed to save test DB to file: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// databases are set up and data copied
|
||||
|
||||
// cleanup test database
|
||||
sqlDB, err := testDb.DB()
|
||||
if err == nil {
|
||||
sqlDB.Close()
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Test data creation from live DB completed successfully",
|
||||
"statistics": copyStats,
|
||||
"test_data_file": preparedTestDB,
|
||||
"file_size_info": "Check file system for actual size",
|
||||
"message": "Live database copied to file successfully",
|
||||
"test_data_file": backupFile,
|
||||
})
|
||||
}
|
||||
|
||||
// copyAllDataToTestDB copies all data from the live database to the test database
|
||||
// TODO can we make it more dynamic? Maybe use reflection to get all models?
|
||||
func copyAllDataToTestDB(liveDB, testDB *gorm.DB) (map[string]int, error) {
|
||||
stats := make(map[string]int)
|
||||
|
||||
// Define all the model types that need to be copied
|
||||
// Order matters due to foreign key constraints!
|
||||
|
||||
// Step 1: Copy base/lookup data first (no foreign keys)
|
||||
count, err := copyTableDataWithCount(liveDB, testDB, &user.User{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
// copyLiveDatabaseToFile kopiert die MariaDB-Datenbank in eine SQLite-Datei
|
||||
func copyLiveDatabaseToFile(liveDB *gorm.DB, targetFile string) error {
|
||||
// Verzeichnis erstellen falls es nicht existiert
|
||||
dir := filepath.Dir(targetFile)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
stats["users"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Skill{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
// Backup der existierenden Datei erstellen
|
||||
if _, err := os.Stat(targetFile); err == nil {
|
||||
backupFile := targetFile + ".backup"
|
||||
os.Remove(backupFile) // Alte Backup entfernen
|
||||
if err := os.Rename(targetFile, backupFile); err != nil {
|
||||
return fmt.Errorf("failed to backup existing file: %w", err)
|
||||
}
|
||||
}
|
||||
stats["gsmaster_skills"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.WeaponSkill{})
|
||||
// SQLite-Zieldatenbank erstellen
|
||||
targetDB, err := gorm.Open(sqlite.Open(targetFile), &gorm.Config{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
return fmt.Errorf("failed to create target SQLite database: %w", err)
|
||||
}
|
||||
stats["gsmaster_weapon_skills"] = count
|
||||
defer func() {
|
||||
if sqlDB, err := targetDB.DB(); err == nil {
|
||||
sqlDB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Spell{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
// Strukturen in SQLite-DB migrieren
|
||||
if err := migrateAllStructures(targetDB); err != nil {
|
||||
return fmt.Errorf("failed to migrate structures to SQLite: %w", err)
|
||||
}
|
||||
stats["gsmaster_spells"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Equipment{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
// Daten von MariaDB zu SQLite kopieren
|
||||
if err := copyMariaDBToSQLite(liveDB, targetDB); err != nil {
|
||||
return fmt.Errorf("failed to copy data from MariaDB to SQLite: %w", err)
|
||||
}
|
||||
stats["gsmaster_equipment"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Weapon{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["gsmaster_weapons"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Container{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["gsmaster_containers"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Transportation{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["gsmaster_transportation"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Believe{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["gsmaster_believes"] = count
|
||||
|
||||
// Step 2: Copy character data (depends on nothing)
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Char{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["characters"] = count
|
||||
|
||||
// Step 3: Copy character-dependent data
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Eigenschaft{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_eigenschaften"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Lp{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_lp"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Ap{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_ap"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.B{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_b"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Merkmale{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_merkmale"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Erfahrungsschatz{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_erfahrungsschatz"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Bennies{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_bennies"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.Vermoegen{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["character_vermoegen"] = count
|
||||
|
||||
// Step 4: Copy skills (depends on characters)
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.SkFertigkeit{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["skills_fertigkeiten"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.SkWaffenfertigkeit{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["skills_waffenfertigkeiten"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.SkZauber{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["skills_zauber"] = count
|
||||
|
||||
// Step 5: Copy equipment (depends on characters)
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.EqAusruestung{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["equipment_ausruestung"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.EqWaffe{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["equipment_waffen"] = count
|
||||
|
||||
count, err = copyTableDataWithCount(liveDB, testDB, &models.EqContainer{})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats["equipment_containers"] = count
|
||||
|
||||
return stats, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyTableDataWithCount copies all records from one table to another and returns the count
|
||||
func copyTableDataWithCount(liveDB, testDB *gorm.DB, model interface{}) (int, error) {
|
||||
// Get all records from live database using the actual model type
|
||||
// This prevents GORM from modifying our data during Create operations
|
||||
// copyMariaDBToSQLite kopiert alle Daten von MariaDB zu SQLite
|
||||
func copyMariaDBToSQLite(mariaDB, sqliteDB *gorm.DB) error {
|
||||
// Vollständige Liste aller Strukturen mit GORM-Tags in der richtigen Reihenfolge
|
||||
// (Basis-Tabellen zuerst wegen Foreign Key-Abhängigkeiten)
|
||||
tables := []interface{}{
|
||||
// Basis-Strukturen (keine Abhängigkeiten)
|
||||
&user.User{},
|
||||
|
||||
// Learning Costs System - Basis
|
||||
&models.Source{},
|
||||
&models.CharacterClass{},
|
||||
&models.SkillCategory{},
|
||||
&models.SkillDifficulty{},
|
||||
&models.SpellSchool{},
|
||||
|
||||
// Learning Costs System - Abhängige Tabellen
|
||||
&models.ClassCategoryEPCost{},
|
||||
&models.ClassSpellSchoolEPCost{},
|
||||
&models.SpellLevelLECost{},
|
||||
&models.SkillCategoryDifficulty{},
|
||||
&models.SkillImprovementCost{},
|
||||
|
||||
// GSMaster Basis-Daten
|
||||
&models.LookupList{}, // Basis für Skills, Spells, Equipment
|
||||
&models.Skill{},
|
||||
&models.WeaponSkill{},
|
||||
&models.Spell{},
|
||||
&models.Equipment{},
|
||||
&models.Weapon{},
|
||||
&models.Container{},
|
||||
&models.Transportation{},
|
||||
&models.Believe{},
|
||||
|
||||
// Charaktere (Basis)
|
||||
&models.Char{},
|
||||
|
||||
// Charakter-Eigenschaften (abhängig von Char)
|
||||
&models.Eigenschaft{},
|
||||
&models.Lp{},
|
||||
&models.Ap{},
|
||||
&models.B{},
|
||||
&models.Merkmale{},
|
||||
&models.Erfahrungsschatz{},
|
||||
&models.Bennies{},
|
||||
&models.Vermoegen{},
|
||||
|
||||
// Charakter-Skills (abhängig von Char und Skills)
|
||||
&models.SkFertigkeit{},
|
||||
&models.SkWaffenfertigkeit{},
|
||||
&models.SkAngeboreneFertigkeit{},
|
||||
&models.SkZauber{},
|
||||
|
||||
// Charakter-Equipment (abhängig von Char und Equipment)
|
||||
&models.EqAusruestung{},
|
||||
&models.EqWaffe{},
|
||||
&models.EqContainer{},
|
||||
|
||||
// View-Strukturen ohne eigene Tabellen werden nicht kopiert:
|
||||
// SkillLearningInfo, SpellLearningInfo, CharList, FeChar, etc.
|
||||
}
|
||||
|
||||
for _, model := range tables {
|
||||
if err := copyTableData(mariaDB, sqliteDB, model); err != nil {
|
||||
return fmt.Errorf("failed to copy table data for %T: %w", model, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyTableData kopiert alle Daten einer Tabelle von MariaDB zu SQLite
|
||||
func copyTableData(sourceDB, targetDB *gorm.DB, model interface{}) error {
|
||||
// Anzahl der Datensätze prüfen
|
||||
var count int64
|
||||
if err := liveDB.Model(model).Count(&count).Error; err != nil {
|
||||
return 0, err
|
||||
if err := sourceDB.Model(model).Count(&count).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If no records, skip (this is normal for some tables)
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
return nil // Keine Daten zu kopieren
|
||||
}
|
||||
|
||||
// Process records in batches to avoid memory issues with large tables
|
||||
// Daten in Blöcken kopieren (für große Tabellen)
|
||||
batchSize := 100
|
||||
totalProcessed := 0
|
||||
|
||||
for offset := 0; offset < int(count); offset += batchSize {
|
||||
// Get a fresh batch of records for each iteration
|
||||
var records []map[string]interface{}
|
||||
if err := liveDB.Model(model).Offset(offset).Limit(batchSize).Find(&records).Error; err != nil {
|
||||
return totalProcessed, err
|
||||
|
||||
// Batch aus MariaDB lesen
|
||||
if err := sourceDB.Model(model).Offset(offset).Limit(batchSize).Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Insert the batch into test database
|
||||
if err := testDB.Model(model).Create(&records).Error; err != nil {
|
||||
// If batch insert fails, try individual inserts to identify problematic records
|
||||
for _, record := range records {
|
||||
if err := testDB.Model(model).Create(&record).Error; err != nil {
|
||||
return totalProcessed, err
|
||||
}
|
||||
totalProcessed++
|
||||
}
|
||||
} else {
|
||||
totalProcessed += len(records)
|
||||
}
|
||||
}
|
||||
|
||||
return totalProcessed, nil
|
||||
}
|
||||
|
||||
// saveTestDatabaseToFile saves the in-memory test database to a file
|
||||
func saveTestDatabaseToFile(testDB *gorm.DB, filename string) error {
|
||||
// Create the directory if it doesn't exist
|
||||
dir := filepath.Dir(filename)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Handle existing file by creating a backup
|
||||
backupFilename := filename + ".backup"
|
||||
fileExists := false
|
||||
|
||||
// Check if the target file exists
|
||||
if _, err := os.Stat(filename); err == nil {
|
||||
fileExists = true
|
||||
// Remove any existing backup file first
|
||||
if _, err := os.Stat(backupFilename); err == nil {
|
||||
if err := os.Remove(backupFilename); err != nil {
|
||||
return fmt.Errorf("failed to remove existing backup file %s: %w", backupFilename, err)
|
||||
}
|
||||
}
|
||||
// Rename existing file to backup
|
||||
if err := os.Rename(filename, backupFilename); err != nil {
|
||||
return fmt.Errorf("failed to create backup of existing file %s: %w", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
// For SQLite in-memory databases, we need to backup to a file
|
||||
// We'll use SQLite's backup API through a raw SQL command
|
||||
backupSQL := fmt.Sprintf("VACUUM INTO '%s'", filename)
|
||||
if err := testDB.Exec(backupSQL).Error; err != nil {
|
||||
// If VACUUM INTO fails and we created a backup, restore it
|
||||
if fileExists {
|
||||
if restoreErr := os.Rename(backupFilename, filename); restoreErr != nil {
|
||||
return fmt.Errorf("failed to backup database to file: %w (and failed to restore backup: %v)", err, restoreErr)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("failed to backup database to file: %w", err)
|
||||
}
|
||||
|
||||
// VACUUM INTO succeeded, remove the backup file if it exists
|
||||
if fileExists {
|
||||
if err := os.Remove(backupFilename); err != nil {
|
||||
// Log the error but don't fail the operation since the main task succeeded
|
||||
fmt.Printf("Warning: failed to remove backup file %s: %v\n", backupFilename, err)
|
||||
// Batch in SQLite einfügen
|
||||
if err := targetDB.Model(model).Create(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user