Inter-App Data Migration (Migrating Data to New App Version)

coredata - move to app group target

In case someone wants the solution in swift just add below function in didFinishLaunchingWithOptions.

 func migratePersistentStore(){

    let coordinator = NSPersistentStoreCoordinator(managedObjectModel: self.managedObjectModel)
    var storeOptions = [AnyHashable : Any]()
    storeOptions[NSMigratePersistentStoresAutomaticallyOption] = true
    storeOptions[NSInferMappingModelAutomaticallyOption] = true
    let oldStoreUrl = self.applicationDocumentsDirectory.appendingPathComponent("YourApp.sqlite")!
    let newStoreUrl = self.applicationGroupDirectory.appendingPathComponent("YourApp.sqlite")!
    var targetUrl : URL? = nil
    var needMigrate = false
    var needDeleteOld = false

    if FileManager.default.fileExists(atPath: oldStoreUrl.path){
        needMigrate = true
        targetUrl = oldStoreUrl
    }

    if FileManager.default.fileExists(atPath: newStoreUrl.path){
        needMigrate = false
        targetUrl = newStoreUrl

        if FileManager.default.fileExists(atPath: oldStoreUrl.path){
            needDeleteOld = true
        }
    }
    if targetUrl == nil {
        targetUrl = newStoreUrl
    }
    if needMigrate {
        do {
            try coordinator.addPersistentStore(ofType: NSSQLiteStoreType, configurationName: nil, at: targetUrl!, options: storeOptions)
            if let store = coordinator.persistentStore(for: targetUrl!) 
             {
                do {
                    try coordinator.migratePersistentStore(store, to: newStoreUrl, options: storeOptions, withType: NSSQLiteStoreType)

                } catch let error {
                    print("migrate failed with error : \(error)")
                }
            }
        } catch let error {
            CrashlyticsHelper.reportCrash(err: error as NSError, strMethodName: "migrateStore")
        }
    }
  if needDeleteOld {
        DBHelper.deleteDocumentAtUrl(url: oldStoreUrl)
        guard let shmDocumentUrl = self.applicationDocumentsDirectory.appendingPathComponent("NoddApp.sqlite-shm") else { return }
        DBHelper.deleteDocumentAtUrl(url: shmDocumentUrl)
        guard let walDocumentUrl = self.applicationDocumentsDirectory.appendingPathComponent("NoddApp.sqlite-wal") else { return }
        DBHelper.deleteDocumentAtUrl(url: walDocumentUrl)
    }
}

My PersistentStoreCoordinator Looks like this:

lazy var persistentStoreCoordinator: NSPersistentStoreCoordinator = {
    let coordinator = NSPersistentStoreCoordinator(managedObjectModel: self.managedObjectModel)
    let url = self.applicationGroupDirectory.appendingPathComponent("YourApp.sqlite")
    var storeOptions = [AnyHashable : Any]()
    storeOptions[NSMigratePersistentStoresAutomaticallyOption] = true
    storeOptions[NSInferMappingModelAutomaticallyOption] = true
    var failureReason = "There was an error creating or loading the application's saved data."
    do {
        try coordinator.addPersistentStore(ofType: NSSQLiteStoreType, configurationName: nil, at: url, options:storeOptions)
    } catch {
        // Report any error we got.
        var dict = [String: AnyObject]()
        dict[NSLocalizedDescriptionKey] = "Failed to initialize the application's saved data" as AnyObject?
        dict[NSLocalizedFailureReasonErrorKey] = failureReason as AnyObject?

        dict[NSUnderlyingErrorKey] = error as NSError
        let wrappedError = NSError(domain: "YOUR_ERROR_DOMAIN", code: 9999, userInfo: dict)
        NSLog("Unresolved error \(wrappedError), \(wrappedError.userInfo)")
        abort()
    }
    return coordinator
}()

This is the case when there is already an app in appstore and you want to migrate the coreData persistent store file from your default store location to your App Group Location.

Edit : For Deleting the file from the old location it is recommended that we use NSFileCoordinator to perform the task.

static func deleteDocumentAtUrl(url: URL){
    let fileCoordinator = NSFileCoordinator(filePresenter: nil)
    fileCoordinator.coordinate(writingItemAt: url, options: .forDeleting, error: nil, byAccessor: {
        (urlForModifying) -> Void in
        do {
            try FileManager.default.removeItem(at: urlForModifying)
        }catch let error {
            print("Failed to remove item with error: \(error.localizedDescription)")
        }
    })
}

Please note the reason why NSFileCoordinator is used to delete the file is because NSFileCoordinator allows us to ensure that file related tasks such as opening reading writing are done in such a way that wont interfere with anyother task on the system trying to work with the same file.Eg if you want to open a file and at the same time it gets deleted ,you dont want both the actions to happen at the same time.

Please call the above function after the store is successfully migrated.

Migrate Core Data database from one app to another

In the end I managed to find a workable solution. I have still not figured out how to successfully export the database, but since I will only perform this migration once for every app update (or less frequent) I can manually copy the data and verify that the database is not corrupt by simply testing it before submitting to the App Store.

Regarding the code for the import of the database, this is what I ended up using:

lazy var persistentContainer: NSPersistentContainer = {
        let databaseName = "GeoPointDB"

        let container = NSPersistentContainer(name: databaseName)

        var persistentStoreDescriptions: NSPersistentStoreDescription

        let storeUrl = FileManager.default.urls(for: .documentDirectory, in:.userDomainMask).first!.appendingPathComponent(databaseName + ".sqlite")
        let storeUrlFolder = FileManager.default.urls(for: .documentDirectory, in:.userDomainMask).first!

        if !FileManager.default.fileExists(atPath: (storeUrl.path)) {
            let seededDataUrl = Bundle.main.url(forResource: databaseName, withExtension: "sqlite")
            let seededDataUrl2 = Bundle.main.url(forResource: databaseName, withExtension: "sqlite-shm")
            let seededDataUrl3 = Bundle.main.url(forResource: databaseName, withExtension: "sqlite-wal")

            try! FileManager.default.copyItem(at: seededDataUrl!, to: storeUrl)
            try! FileManager.default.copyItem(at: seededDataUrl2!, to: storeUrlFolder.appendingPathComponent(databaseName + ".sqlite-shm"))
            try! FileManager.default.copyItem(at: seededDataUrl3!, to: storeUrlFolder.appendingPathComponent(databaseName + ".sqlite-wal"))

        }

        container.persistentStoreDescriptions = [NSPersistentStoreDescription(url: storeUrl)]
        container.loadPersistentStores(completionHandler: { (storeDescription, error) in
            if let error = error {

                fatalError("Unresolved error \(error),")
            }
        })

        return container
    }()

The two things I essentially changed was to make sure that the NSPersistantStoreDescription was pointing directly at the SQL file (rather than it's directory as above) and that I copied all three files (.sqlite, .sqlite-shm and .sqlite-wal) to the documents folder.

Data Migration from Legacy Data Structure to New Data Structure

EDIT: More explanation of synchronization using bi-directional triggers; updates for syntax, language and clarity.

Preamble

I have faced similar problems on a data model upgrade on a large web application I worked on for 7 years, so I feel your pain. From this experience, I would propose the something a bit different - but hopefully one that will be a lot easier to implement. But first, an observation:

Value to the organisation is the data - data will long outlive all your current applications. The business will constantly invent new ways of getting value out of the data it has captured which will engender new reports, applications and ways of doing business.

So getting the new data structure right should be your most important goal. Don't trade getting the structure right against against other short term development goals, especially:

Operational goals such as rolling out a new service
Report performance (use materialized views, triggers or batch jobs instead)

This structure will change over time so your architecture must allow for frequent additions and infrequent normalizations to it. This means that your data structure and any shared APIs to it (including RESTful services) must be properly versioned.

Why RESTful web services?

You mention that your will "Move all functionality to a RESTful service so no application has direct database access". I need to ask a very important question with respect to the legacy apps: Why is this important and what value has it brought?

I ask because:

You lose ACID transactions (each call is a single transaction unless you implement some horrifically complicated WS-* standards)
Performance degrades: Direct database connections will be faster (no web server work and translations to do) and have less latency (typically 1ms rather than 50-100ms) which will visibly reduce responsiveness in applications written for direct DB connections
The database structure is not abstracted from the RESTful service, because you acknowledge that with the database normalization you have to rewrite the web services and rewrite the applications calling them.

And the other cross-cutting concerns are unchanged:

Manageability: Direct database connections can be monitored and managed with many generic tools here
Security: direct connections are more secure than web services that your developers will write,
Authorization: The database permission model is very advanced and as fine-grained as you could want
Scaleability: The web service is a (only?) direct-connected database application and so scales only as much as the database

You can migrate the database and keep the legacy applications running by maintaining a legacy RESTful API. But what if we can keep the legacy apps without introducing a 'legacy' RESTful service.

Database versioning

Presumably the majority of the 'legacy' applications use SQL to directly access data tables; you may have a number of database views as well.

One approach to the data migration is that the new database (with the new normalized structure in a new schema) presents the old structure as views to the legacy applications, typically from a different schema.

This is actually quite easy to implement, but solves only reporting and read-only functionality. What about legacy application DML? DML can be solved using

Updatable views for simple transformations
Introducing stored procedures where updatable views not possible (eg "CALL insert_emp(?, ?, ?)" rather than "INSERT INTO EMP (col1, col2, col3) VALUES (?, ? ?)".
Have a 'legacy' table that synchronizes with the new database with triggers and DB links.

Having a legacy-format table with bi-directional synchronization to the new format table(s) using triggers is a brute-force solution and relatively ugly.

You end up with identical data in two different schemas (or databases) and the possibility of data going out-of-sync if the synchronization code has bugs - and then you have the classic issues of the "two master" problem. As such, treat this as a last resort, for example when:

The fundamental structure has changed (for example the changing the cardinality of a relation), or
The translation to the legacy format is a complex function (eg if the legacy column is the square of the new-format column value and is set to "4", an updatable view cannot determine if the correct value is +2 or -2).

When such changes are required in your data, there will be some significant change in code and logic somewhere. You could implement in a compatibility layer (advantage: no change to legacy code) or change the legacy app (advantage: data layer is clean). This is a technical decision by the engineering team.

Creating a compatibility database of the legacy structure using the approaches outlined above minimize changes to legacy applications (in some cases, the legacy application continues without any code change at all). This greatly reduces development and testing costs (for which there is no net functional gain to the business), and greatly reduces rollout risk.

It also allows you to concentrate on the real value to the organisation:

The new database structure
New RESTful web services
New applications (potentially build using the RESTful web services)

Positive aspect of web services

Please don't read the above as a diatribe against web services, especially RESTful web services. When used for the right reason, such as for enabling web applications or integration between disparate systems, this is a good architectural solution. However, it might not be the best solution for managing your legacy apps during the data migration.

iOS 11+ How to migrate existing Core Data to Shared App Group for use in extension?

I ended up getting it doing the following. The sqlite file was actually the name of my init plus .sqlite at the end.

+ (NSPersistentContainer*) GetPersistentContainer {
    //Init the store.
    NSPersistentContainer *_persistentContainer = [[NSPersistentContainer alloc] initWithName:@"Test_App"];

    //Define the store url that is located in the shared group.
    NSURL* storeURL = [[[NSFileManager defaultManager] containerURLForSecurityApplicationGroupIdentifier:@"group.Test_App"] URLByAppendingPathComponent:@"Test_App.sqlite"];

    //Determine if we already have a store saved in the default app location.
    BOOL hasDefaultAppLocation = [[NSFileManager defaultManager] fileExistsAtPath: _persistentContainer.persistentStoreDescriptions[0].URL.path];

    //Check if the store needs migration.
    BOOL storeNeedsMigration = hasDefaultAppLocation && ![_persistentContainer.persistentStoreDescriptions[0].URL.absoluteString isEqualToString:storeURL.absoluteString];

    //Check if the store in the default location does not exist.
    if (!hasDefaultAppLocation) {
        //Create a description to use for the app group store.
        NSPersistentStoreDescription *description = [[NSPersistentStoreDescription alloc] init];

        //set the automatic properties for the store.
        description.shouldMigrateStoreAutomatically = true;
        description.shouldInferMappingModelAutomatically = true;

        //Set the url for the store.
        description.URL = storeURL;

        //Replace the coordinator store description with this description.
        _persistentContainer.persistentStoreDescriptions = [NSArray arrayWithObjects:description, nil];
    }

    //Load the store.
    [_persistentContainer loadPersistentStoresWithCompletionHandler:^(NSPersistentStoreDescription *storeDescription, NSError *error) {
        //Check that we do not have an error.
        if (error == nil) {
            //Check if we need to migrate the store.
            if (storeNeedsMigration) {
                //Create errors to track migration and deleting errors.
                NSError *migrateError;
                NSError *deleteError;

                //Store the old location URL.
                NSURL *oldStoreURL = storeDescription.URL;

                //Get the store we want to migrate.
                NSPersistentStore *store = [_persistentContainer.persistentStoreCoordinator persistentStoreForURL: oldStoreURL];

                //Set the store options.
                NSDictionary *storeOptions = @{ NSSQLitePragmasOption : @{ @"journal_mode" : @"WAL" } };

                //Migrate the store.
                NSPersistentStore *newStore = [_persistentContainer.persistentStoreCoordinator migratePersistentStore: store toURL:storeURL options:storeOptions withType:NSSQLiteStoreType error:&migrateError];

                //Check that the store was migrated.
                if (newStore && !migrateError) {
                    //Remove the old SQLLite database.
                    [[[NSFileCoordinator alloc] init] coordinateWritingItemAtURL: oldStoreURL options: NSFileCoordinatorWritingForDeleting error: &deleteError byAccessor: ^(NSURL *urlForModifying) {
                        //Create a remove error.
                        NSError *removeError;

                        //Delete the file.
                        [[NSFileManager defaultManager] removeItemAtURL: urlForModifying error: &removeError];

                        //If there was an error. Output it.
                        if (removeError) {
                            NSLog(@"%@", [removeError localizedDescription]);
                        }
                    }
                     ];

                    //If there was an error. Output it.
                    if (deleteError) {
                        NSLog(@"%@", [deleteError localizedDescription]);
                    }
                }
            }
        } else {
            // Replace this implementation with code to handle the error appropriately.
            // abort() causes the application to generate a crash log and terminate. You should not use this function in a shipping application, although it may be useful during development.

            /*
             Typical reasons for an error here include:
             * The parent directory does not exist, cannot be created, or disallows writing.
             * The persistent store is not accessible, due to permissions or data protection when the device is locked.
             * The device is out of space.
             * The store could not be migrated to the current model version.
             Check the error message to determine what the actual problem was.
             */
            NSLog(@"Unresolved error %@, %@", error, error.userInfo);
            abort();
        }
    }];

    //Return the container.
    return _persistentContainer;
}

How to deal with changes when adding new field to a data class in Kotlin

You need to do a migration of your database. You can find all the instructions to do it there Migrating Room databases.

If your room version is 2.4.0-alpha01 or higher:

Replace in your database:

@Database(
  version = 1,
  entities = {User.class}
)

with:

@Database(
  version = 2,
  entities = {User.class},
  autoMigrations = {
    @AutoMigration (from = 1, to = 2)
  }
)

Else you will need to du the migration manually:

Add this to your Database class:

static final Migration MIGRATION_1_2 = new Migration(1, 2) {
  @Override
  public void migrate(SupportSQLiteDatabase database) {
    database.execSQL("ALTER TABLE users "
      + " ADD COLUMN gender TEXT");
  }
};

and add .addMigrations(MIGRATION_1_2) to your Room.databaseBuilder()