Feature document database migration #15

- Rewrites `DatabaseSeedingService` to work with a MongoDB `Db` instance.
- Removes table creation logic, as MongoDB collections are schemaless.
- Implements idempotent seeding using `bulkWrite` with `upsert` operations,
 preventing duplicate data on subsequent runs.
- Correctly handles the conversion of string IDs from fixtures to MongoDB
 `ObjectId` for the `_id` field.
- Ensures complex nested objects in fixtures are properly JSON-encoded
 before insertion.

Loading branch information

@fulleni

fulleni committed Jul 11, 2025

commit fe51a1e9bdee8048cf024ab9d3120de9b2d690c3

160 changes: 64 additions & 96 deletions lib/src/services/database_seeding_service.dart

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
		@@ -1,149 +1,117 @@
	import 'dart:convert';
	import 'dart:io';

		import 'package:ht_shared/ht_shared.dart';
		import 'package:logging/logging.dart';
		import 'package:mongo_dart/mongo_dart.dart';

		/// {@template database_seeding_service}
		/// A service responsible for seeding the MongoDB database with initial data.
		///
	/// This service reads data from local JSON fixture files and uses `upsert`
	/// operations to ensure that the seeding process is idempotent. It can be
	/// run multiple times without creating duplicate documents.
	/// This service reads data from predefined fixture lists in `ht_shared` and
	/// uses `upsert` operations to ensure that the seeding process is idempotent.
	/// It can be run multiple times without creating duplicate documents.
		/// {@endtemplate}
		class DatabaseSeedingService {
		/// {@macro database_seeding_service}
		const DatabaseSeedingService({required Db db, required Logger log})
	: _db = db,
	_log = log;
	: _db = db,
	_log = log;

		final Db _db;
		final Logger _log;

		/// The main entry point for seeding all necessary data.
		Future<void> seedInitialData() async {
		_log.info('Starting database seeding process...');
	await _seedCollection(

	await _seedCollection<Country>(
		collectionName: 'countries',
	fixturePath: 'lib/src/fixtures/countries.json',
	fixtureData: countriesFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
		);
	await _seedCollection(
	await _seedCollection<Source>(
		collectionName: 'sources',
	fixturePath: 'lib/src/fixtures/sources.json',
	fixtureData: sourcesFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
		);
	await _seedCollection(
	await _seedCollection<Topic>(
		collectionName: 'topics',
	fixturePath: 'lib/src/fixtures/topics.json',
	fixtureData: topicsFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
		);
	await _seedCollection(
	await _seedCollection<Headline>(
		collectionName: 'headlines',
	fixturePath: 'lib/src/fixtures/headlines.json',
	fixtureData: headlinesFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
	);
	await _seedCollection<User>(
	collectionName: 'users',
	fixtureData: usersFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
	);
	await _seedCollection<RemoteConfig>(
	collectionName: 'remote_configs',
	fixtureData: remoteConfigsFixturesData,
	getId: (item) => item.id,
	toJson: (item) => item.toJson(),
		);
	await _seedInitialAdminAndConfig();

		_log.info('Database seeding process completed.');
		}

	/// Seeds a specific collection from a given JSON fixture file.
	Future<void> _seedCollection({
	/// Seeds a specific collection from a given list of fixture data.
	Future<void> _seedCollection<T>({
		required String collectionName,
	required String fixturePath,
	required List<T> fixtureData,
	required String Function(T) getId,
	required Map<String, dynamic> Function(T) toJson,
		}) async {
	_log.info('Seeding collection: "$collectionName" from "$fixturePath"...');
	_log.info('Seeding collection: "$collectionName"...');
		try {
	final collection = _db.collection(collectionName);
	final file = File(fixturePath);
	if (!await file.exists()) {
	_log.warning('Fixture file not found: $fixturePath. Skipping.');
	return;
	}

	final content = await file.readAsString();
	final documents = jsonDecode(content) as List<dynamic>;

	if (documents.isEmpty) {
	if (fixtureData.isEmpty) {
		_log.info('No documents to seed for "$collectionName".');
		return;
		}

	final bulk = collection.initializeUnorderedBulkOperation();
	final collection = _db.collection(collectionName);
	final operations = <Map<String, Object>>[];

	for (final doc in documents) {
	final docMap = doc as Map<String, dynamic>;
	final id = docMap['id'] as String?;
	for (final item in fixtureData) {
	final id = getId(item);

	if (id == null \|\| !ObjectId.isValidHexId(id)) {
	_log.warning('Skipping document with invalid or missing ID: $doc');
	if (!ObjectId.isValidHexId(id)) {
	_log.warning('Skipping document with invalid ID format: $id');
		continue;
		}

		final objectId = ObjectId.fromHexString(id);
	// Remove the string 'id' field and use '_id' with ObjectId
	docMap.remove('id');
	final document = toJson(item)..remove('id');

	operations.add({
	'replaceOne': {
	'filter': {'_id': objectId},
	'replacement': document,
	'upsert': true,
	},
	});
	}

	bulk.find({'_id': objectId}).upsert().replaceOne(docMap);
	if (operations.isEmpty) {
	_log.info('No valid documents to write for "$collectionName".');
	return;
		}

	final result = await bulk.execute();
	final result = await collection.bulkWrite(operations);
		_log.info(
		'Seeding for "$collectionName" complete. '
		'Upserted: ${result.nUpserted}, Modified: ${result.nModified}.',
		);
		} on Exception catch (e, s) {
	_log.severe(
	'Failed to seed collection "$collectionName" from "$fixturePath".',
	e,
	s,
	);
	// Re-throwing to halt the startup process if seeding fails.
	rethrow;
	}
	}

	/// Seeds the initial admin user and remote config document.
	Future<void> _seedInitialAdminAndConfig() async {
	_log.info('Seeding initial admin user and remote config...');
	try {
	// --- Seed Admin User ---
	final usersCollection = _db.collection('users');
	final adminUser = User.fromJson(adminUserFixture);
	final adminDoc = adminUser.toJson()
	..['app_role'] = adminUser.appRole.name
	..['dashboard_role'] = adminUser.dashboardRole.name
	..['feed_action_status'] = jsonEncode(adminUser.feedActionStatus)
	..remove('id');

	await usersCollection.updateOne(
	where.id(ObjectId.fromHexString(adminUser.id)),
	modify.set(
	'email',
	adminDoc['email'],
	).setAll(adminDoc), // Use setAll to add/update all fields
	upsert: true,
	);
	_log.info('Admin user seeded successfully.');

	// --- Seed Remote Config ---
	final remoteConfigCollection = _db.collection('remote_config');
	final remoteConfig = RemoteConfig.fromJson(remoteConfigFixture);
	final remoteConfigDoc = remoteConfig.toJson()
	..['user_preference_limits'] =
	jsonEncode(remoteConfig.userPreferenceConfig.toJson())
	..['ad_config'] = jsonEncode(remoteConfig.adConfig.toJson())
	..['account_action_config'] =
	jsonEncode(remoteConfig.accountActionConfig.toJson())
	..['app_status'] = jsonEncode(remoteConfig.appStatus.toJson())
	..remove('id');

	await remoteConfigCollection.updateOne(
	where.id(ObjectId.fromHexString(remoteConfig.id)),
	modify.setAll(remoteConfigDoc),
	upsert: true,
	);
	_log.info('Remote config seeded successfully.');
	} on Exception catch (e, s) {
	_log.severe('Failed to seed admin user or remote config.', e, s);
	_log.severe('Failed to seed collection "$collectionName".', e, s);
		rethrow;
		}
		}
	}
	}

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Feature document database migration #15

Uh oh!

Feature document database migration #15

Filter by extension

Uh oh!

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing