From 7497ec1695f75b5f9687cfc9a71916dd37255402 Mon Sep 17 00:00:00 2001 From: JWMB Date: Fri, 12 Dec 2025 16:29:58 +0100 Subject: [PATCH 01/38] start mongodb implementation --- Directory.Packages.props | 1 + .../AzureTrainingRepositoryTests.cs | 4 +- .../ProblemSourceModule.csproj | 1 + .../AzureTableTrainingRepository.cs | 11 +- .../AzureTables/AzureTableUserRepository.cs | 7 +- .../Services/Storage/IRepository.cs | 15 +- .../Services/Storage/ITrainingRepository.cs | 11 +- .../MongoDb/MongoTrainingPlanRepository.cs | 202 ++++++++++++++++++ Tools/Program.cs | 1 + Tools/TrainingNormCreator.cs | 2 +- 10 files changed, 234 insertions(+), 21 deletions(-) create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 6598017..1450512 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -47,6 +47,7 @@ + diff --git a/ProblemSource/ProblemSourceModule.Tests/AzureTable/AzureTrainingRepositoryTests.cs b/ProblemSource/ProblemSourceModule.Tests/AzureTable/AzureTrainingRepositoryTests.cs index 248c5cd..7beaad4 100644 --- a/ProblemSource/ProblemSourceModule.Tests/AzureTable/AzureTrainingRepositoryTests.cs +++ b/ProblemSource/ProblemSourceModule.Tests/AzureTable/AzureTrainingRepositoryTests.cs @@ -17,7 +17,7 @@ public async Task AzureTableTrainingRepository_Add_Increment() var idThatsIgnored = -999; var item = new Training { Id = idThatsIgnored, TrainingPlanName = "avc" }; - var newId = await repo.Add(item); + var newId = await repo.AddGetId(item); item.Id.ShouldBe(newId); newId.ShouldBeGreaterThan(0); @@ -42,7 +42,7 @@ public async Task AzureTableTrainingRepository_GetByIds() var addedIds = new List(); foreach (var i in Enumerable.Range(0, numToAdd)) - addedIds.Add(await repo.Add(new Training { })); + addedIds.Add(await repo.AddGetId(new Training { })); var expectedRetrievedIds = addedIds.Skip(1); var idsToGet = expectedRetrievedIds.Concat(new[] { 999 }); diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj index e7428ff..e0ab65c 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj @@ -65,6 +65,7 @@ + diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs index 9dbf115..29acf68 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs @@ -1,7 +1,6 @@ using Azure.Data.Tables; using AzureTableGenerics; using ProblemSource.Services.Storage.AzureTables; -using ProblemSource.Services.Storage.AzureTables.TableEntities; using ProblemSourceModule.Models; namespace ProblemSourceModule.Services.Storage.AzureTables @@ -25,7 +24,10 @@ public AzureTableTrainingRepository(ITypedTableClientFactory tableClientFactory) private static int latestMax = 0; // TODO: ugly performance hack while waiting to port to SQL private object _lock = new object(); - public Task Add(Training item) + + public Task Add(Training item) => AddGetId(item); + + public Task AddGetId(Training item) { // Warning: multi-instance concurrency lock (_lock) @@ -39,9 +41,10 @@ public Task Add(Training item) } public async Task Update(Training item) => await repo.Update(item); - public async Task Upsert(Training item) => int.Parse(await repo.Upsert(item)); + //public async Task Upsert(Training item) => int.Parse(await repo.Upsert(item)); + public async Task Upsert(Training item) => await repo.Upsert(item); - public async Task Remove(Training item) => await repo.Remove(item); + public async Task Remove(Training item) => await repo.Remove(item); public async Task> GetAll() => await repo.GetAll(); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableUserRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableUserRepository.cs index a5c3679..9f7f112 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableUserRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableUserRepository.cs @@ -33,7 +33,7 @@ public AzureTableUserRepository(ITypedTableClientFactory tableClientFactory) //public async Task Get(string email) => await repo.Get(ConvertToKey(email)); private object _lock = new object(); - public Task Add(User item) + public Task Add(User item) { item.Email = User.NormalizeEmail(item.Email); // Warning: multi-instance concurrency @@ -43,7 +43,8 @@ public Task Add(User item) // + InnerException {"The specified entity already exists.\nRequestId:8570ae8a-d9ce-4523-ad65-3949c1a10b16\nTime:2023-01-05T11:51:05.5942785Z\r\nStatus: 409 (Conflict)\r\nErrorCode: EntityAlreadyExists\r\n\r\nContent:\r\n{\"odata.error\":{\"code\":\"EntityAlreadyExists\",\"message\":{\"lang\":\"sv-SE\",\"value\":\"The specified entity already exists.\\nRequestId:8570ae8a-d9ce-4523-ad65-3949c1a10b16\\nTime:2023-01-05T11:51:05.5942785Z\"}}}\r\n\r\nHeaders:\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\nServer: Windows-Azure-Table/1.0,Microsoft-HTTPAPI/2.0\r\nx-ms-request-id: 8570ae8a-d9ce-4523-ad65-3949c1a10b16\r\nx-ms-version: REDACTED\r\nX-Content-Type-Options: REDACTED\r\nPreference-Applied: REDACTED\r\nDate: Thu, 05 Jan 2023 11:51:05 GMT\r\nContent-Type: application/json; odata=minimalmetadata; streaming=true; charset=utf-8\r\n"} System.Exception {Azure.RequestFailedException} try { - return Task.FromResult(repo.Add(item).Result); + repo.Add(item); + return Task.CompletedTask; } catch (Exception ex) { @@ -59,7 +60,7 @@ public Task Add(User item) } } - public async Task Upsert(User item) => await repo.Upsert(item); + public async Task Upsert(User item) => await repo.Upsert(item); public async Task Update(User item) => await repo.Update(item); public async Task Remove(User item) => await repo.Remove(item); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/IRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/IRepository.cs index 0f08fa6..da86d1c 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/IRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/IRepository.cs @@ -4,14 +4,21 @@ public interface IRepository { Task> GetAll(); Task Get(TId id); - Task Add(TEntity item); - Task Upsert(TEntity item); - Task Update(TEntity item); + Task Add(TEntity item); + //Task Upsert(TEntity item); + Task Upsert(TEntity item); + Task Update(TEntity item); Task Remove(TEntity item); //Task Remove(TId id); } + public interface IAddGetId + { + Task AddGetId(TEntity item); + + } + - public static class IRepositoryExtensions + public static class IRepositoryExtensions { public static async Task RemoveByIdIfExists(this IRepository repo, TId id) { diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingRepository.cs index 2d7fb1f..ae23090 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingRepository.cs @@ -1,18 +1,15 @@ -using ProblemSource.Services.Storage; -using ProblemSource; -using ProblemSourceModule.Models; -using ProblemSource.Models; +using ProblemSourceModule.Models; namespace ProblemSourceModule.Services.Storage { - public interface ITrainingRepository : IRepository - { + public interface ITrainingRepository : IRepository, IAddGetId + { Task> GetByIds(IEnumerable ids); async Task Add(ITrainingUsernameService usernameService, Training training) { - var id = await Add(training); + var id = await AddGetId(training); training.Username = usernameService.FromId(id); await Update(training); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs new file mode 100644 index 0000000..bfabd32 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -0,0 +1,202 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using ProblemSource.Models; +using ProblemSource.Models.Aggregates; +using ProblemSource.Services.Storage; +using ProblemSourceModule.Models; +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + + public class MongoTools + { + public static async Task Initialize() + { + const string uri = "mongodb://localhost:27017/"; + var client = new MongoClient(uri); + var db = client.GetDatabase("training"); + //db.GetCollection<> + } + + public static string GetCollectionName() => GetCollectionName(typeof(T)); + public static string GetCollectionName(Type type) => type.Name; + } + + public class DbCollection + { + protected IMongoCollection collection; + private readonly string idField; + private readonly Func getId; + + public DbCollection(IMongoDatabase db, string idField, Func getId) + { + collection = db.GetCollection(MongoTools.GetCollectionName()); + this.idField = idField; + this.getId = getId; + } + public async Task InsertGetId(TDocument item) + { + await collection.InsertOneAsync(item); + //item.Id + return getId(item); + } + + public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); + public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); + public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); + + // public static FilterDefinition GetIdFilter(TId_ id, string idField) => Builders.Filter.Eq(idField, id); + //public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); + public static FilterDefinition GetIdFilter(TId id, string idField) => Builders.Filter.Eq(idField, id); + public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); + + public async Task Get(TId id) + { + //var filter = Builders.Filter.Eq(idField, id); + return await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); + //await collection.Find(o => o.Email == id).FirstOrDefaultAsync(); + } + + public async Task> GetAll() => await collection.Find(o => true).ToListAsync(); + + public async Task Remove(TDocument item) + { + var found = await collection.FindOneAndDeleteAsync(GetIdFilter(getId(item))); + } + + public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); + public async Task Upsert(TDocument item) => await Update(item); + + public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + { + return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); + } + public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + { + var result = await collection.DeleteManyAsync(filter, cancellationToken); + return (int)result.DeletedCount; + } + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) + { + var list = items.ToList(); + var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); + //InsertOneModel + var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); + var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); + + var upserted = upsertedIndices.Select(o => list[o]); + + return (list.Except(upserted), upserted); + } + //protected async Task> FindAsync(FilterDefinition filter, FindOptions? options = null, CancellationToken cancellationToken = default(CancellationToken)) + //{ + // if (options == null) throw new ArgumentNullException(nameof(options)); + // return await collection.FindAsync(filter, options, cancellationToken); + //} + + } + + public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider + { + private readonly IMongoDatabase db; + private readonly int trainingId; + + public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) + { + this.db = db; + this.trainingId = trainingId; + } + + public IBatchRepository Phases => new MongoBatchRepository(db, "", Phase.UniqueIdWithinUser, trainingId); + + public IBatchRepository TrainingDays => throw new NotImplementedException(); + + public IBatchRepository PhaseStatistics => throw new NotImplementedException(); + + public IBatchRepository TrainingSummaries => throw new NotImplementedException(); + + public IBatchRepository UserStates => throw new NotImplementedException(); + + public Task RemoveAll() + { + throw new NotImplementedException(); + } + } + + public class MongoDocumentWrapper + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = string.Empty; + + public required TDocument Document { get; set; } + } + + public class MongoBatchRepository : IBatchRepository + { + private readonly int trainingId; + private readonly string trainingIdField; + //private readonly Func getId; + private readonly DbCollection collection; + //private readonly DbCollection, TId> collection; + + public MongoBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") + { + collection = new DbCollection(db, idField, getId); + //collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); + + this.trainingId = trainingId; + this.trainingIdField = trainingIdField; + //this.getId = getId; + } + + //private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); + private FilterDefinition GetTrainingIdFilter() => DbCollection.GetIdFilter(trainingId, trainingIdField); + //public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + public async Task> GetAll() => await collection.ListAsync(GetTrainingIdFilter()); + + public async Task RemoveAll() + => await collection.RemoveAsync(GetTrainingIdFilter()); + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) + { + Func> createFilter = item => + Builders.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); + return await collection.Upsert(items, createFilter); + } + } + + + public class MongoUserRepository : DbCollection, IUserRepository + { + public MongoUserRepository(IMongoDatabase db) : base(db, nameof(User.Email), u => u.Email) + { } + public async Task Add(User item) => await InsertGetId(item); + //public async Task Get(string id) => await collection.Find(o => o.Email == id).FirstOrDefaultAsync(); + } + + public class MongoTrainingRepository : DbCollection, ITrainingRepository + { + public MongoTrainingRepository(IMongoDatabase db) : base(db, nameof(Training.Id), u => u.Id) + { } + + public Task Add(Training item) => AddGetId(item); + + public async Task AddGetId(Training item) => await InsertGetId(item); + + public async Task> GetByIds(IEnumerable ids) + { + var list = ids.ToList(); + return await (await collection.FindAsync(o => list.Contains(o.Id))).ToListAsync(); + } + //public Task Remove(Training item) => throw new NotImplementedException(); + //public Task Update(Training item) => throw new NotImplementedException(); + //public Task Upsert(Training item) => throw new NotImplementedException(); + //public async Task Get(int id) => await base.Get(id); + //public Task> GetAll() => throw new NotImplementedException(); + } + +} diff --git a/Tools/Program.cs b/Tools/Program.cs index d770baa..5704e2f 100644 --- a/Tools/Program.cs +++ b/Tools/Program.cs @@ -43,6 +43,7 @@ var path = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "WebProcessor_Files"); +//await TrainingMod.ModifyTimeSpent(42434, serviceProvider.GetRequiredService()); //await new FixAzureTableQuotedDateTime(serviceProvider.GetRequiredService().ConnectionString) // .Fix(new Dictionary> { // { diff --git a/Tools/TrainingNormCreator.cs b/Tools/TrainingNormCreator.cs index 2673d89..a431daa 100644 --- a/Tools/TrainingNormCreator.cs +++ b/Tools/TrainingNormCreator.cs @@ -46,7 +46,7 @@ private async Task RecreateTraining(Training training) else { var trainingRepo = new AzureTableTrainingRepository(tableClientFactory); - targetNormTrainingId = await trainingRepo.Add(training); + targetNormTrainingId = await trainingRepo.AddGetId(training); training.Id = targetNormTrainingId.Value; } } From f689714e883f1738cc11d1b4f6b073b092a0764c Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 13 Dec 2025 15:33:21 +0100 Subject: [PATCH 02/38] semi-working --- Directory.Packages.props | 2 + .../ProblemSourceModule.Tests/MongoDbTests.cs | 53 +++++++ .../ProblemSourceModule.Tests.csproj | 2 + .../MongoDb/MongoTrainingPlanRepository.cs | 144 +++++++++++++----- 4 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 1450512..01254fe 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,6 +17,7 @@ + @@ -47,6 +48,7 @@ + diff --git a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs new file mode 100644 index 0000000..7ad5d9d --- /dev/null +++ b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs @@ -0,0 +1,53 @@ +//using EphemeralMongo; +using Mongo2Go; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using ProblemSource.Models.Aggregates; +using ProblemSourceModule.Models; +using ProblemSourceModule.Services.Storage.MongoDb; +using Shouldly; + +namespace ProblemSourceModule.Tests +{ + public class MongoDbTests + { + [Fact] + public async Task X() + { + using var runner = MongoDbRunner.StartForDebugging(additionalMongodArguments: "--quiet --logpath /dev/null"); + + //var options = new MongoRunnerOptions + //{ + // UseSingleNodeReplicaSet = true, + // //StandardOuputLogger = Console.WriteLine, + // StandardOutputLogger = Console.WriteLine, + // StandardErrorLogger = Console.WriteLine, + //}; + + //BsonClassMap.RegisterClassMap>(x => + //{ + // x.AutoMap(); + // x.GetMemberMap(m => m.Id).SetIgnoreIfDefault(true); + //}); + + var db = new MongoClient(runner.ConnectionString).GetDatabase("_mydb"); + var sut = new MongoTrainingRepository(db); + var training = new Training { Username = "ABV", Created = DateTimeOffset.UtcNow }; + var id = await sut.AddGetId(training); + var retrieved = await sut.Get(id); + + retrieved?.Username.ShouldBe(training.Username); + + var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); + + var phase = new Phase { exercise = "a", phase_type = "TEST", training_day = 1, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; + var result = await ugdr.Phases.Upsert([phase]); + result.Added.Count().ShouldBe(1); + result.Updated.Count().ShouldBe(0); + + result = await ugdr.Phases.Upsert([phase]); + result.Added.Count().ShouldBe(0); + result.Updated.Count().ShouldBe(1); + } + } +} diff --git a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj index cc94997..de68acf 100644 --- a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj +++ b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj @@ -18,8 +18,10 @@ + + diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index bfabd32..b6db252 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -21,10 +21,45 @@ public static async Task Initialize() } public static string GetCollectionName() => GetCollectionName(typeof(T)); - public static string GetCollectionName(Type type) => type.Name; + public static string GetCollectionName(Type type) + { + if (type.GenericTypeArguments.Any()) + { + if (type.GetGenericTypeDefinition() == typeof(MongoDocumentWrapper<>)) + { + return type.GenericTypeArguments[0].Name; + } + } + return type.Name; + } + } + + public class DbWrappedCollection + { + protected DbCollection, TId> collection; + public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId)// : base(db, idField, getId) + { + collection = new DbCollection, TId>(db, idField, getId); + } + + public Task Remove(TDocument item) => collection.Remove(new MongoDocumentWrapper(item)); + public Task Update(TDocument item) => collection.Update(new MongoDocumentWrapper(item)); // TODO: is mongo ID handled properly? + public Task Upsert(TDocument item) => collection.Upsert(new MongoDocumentWrapper(item)); // TODO: is mongo ID handled properly? + public async Task Get(TId id) + { + var found = await collection.Get(id); + if (found != null) + return found.Document; + return default; + } + public async Task> Get(IEnumerable ids) => (await collection.Get(ids)).Select(o => o.Document).ToList(); + + + public async Task> GetAll() => (await collection.GetAll()).Select(o => o.Document).ToList(); + } - public class DbCollection + public class DbCollection { protected IMongoCollection collection; private readonly string idField; @@ -36,11 +71,11 @@ public DbCollection(IMongoDatabase db, string idField, Func getI this.idField = idField; this.getId = getId; } + public async Task InsertGetId(TDocument item) { - await collection.InsertOneAsync(item); - //item.Id - return getId(item); + await collection.InsertOneAsync(item); + return getId(item); } public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); @@ -52,12 +87,8 @@ public async Task InsertGetId(TDocument item) public static FilterDefinition GetIdFilter(TId id, string idField) => Builders.Filter.Eq(idField, id); public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); - public async Task Get(TId id) - { - //var filter = Builders.Filter.Eq(idField, id); - return await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); - //await collection.Find(o => o.Email == id).FirstOrDefaultAsync(); - } + public async Task Get(TId id) => await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); + public async Task> Get(IEnumerable ids) => await (await collection.FindAsync(GetIdFilter(ids))).ToListAsync(); public async Task> GetAll() => await collection.Find(o => true).ToListAsync(); @@ -67,7 +98,7 @@ public async Task Remove(TDocument item) } public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); - public async Task Upsert(TDocument item) => await Update(item); + public async Task Upsert(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) { @@ -91,18 +122,18 @@ public async Task RemoveAsync(FilterDefinition filter, Cancellat return (list.Except(upserted), upserted); } - //protected async Task> FindAsync(FilterDefinition filter, FindOptions? options = null, CancellationToken cancellationToken = default(CancellationToken)) - //{ - // if (options == null) throw new ArgumentNullException(nameof(options)); - // return await collection.FindAsync(filter, options, cancellationToken); - //} + public async Task CountDocumentsAsync(FilterDefinition? filter = null) + { + return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); + } } public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider { private readonly IMongoDatabase db; private readonly int trainingId; + private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) { @@ -110,7 +141,7 @@ public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingI this.trainingId = trainingId; } - public IBatchRepository Phases => new MongoBatchRepository(db, "", Phase.UniqueIdWithinUser, trainingId); + public IBatchRepository Phases => new MongoBatchRepository(db, Key, Phase.UniqueIdWithinUser, trainingId); public IBatchRepository TrainingDays => throw new NotImplementedException(); @@ -128,44 +159,62 @@ public Task RemoveAll() public class MongoDocumentWrapper { + public MongoDocumentWrapper() + { + Id = ObjectId.GenerateNewId(); + } + + [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] + public MongoDocumentWrapper(TDocument doc) : this() + { + Document = doc; + } + [BsonId] - [BsonRepresentation(BsonType.ObjectId)] - public string Id { get; set; } = string.Empty; + public ObjectId Id { get; set; } + + //[BsonRepresentation(BsonType.ObjectId)] + //[BsonId(IdGenerator = typeof(MongoDB.Bson.Serialization.IdGenerators.BsonObjectIdGenerator))] + //[BsonIgnoreIfDefault] + //public string Id { get; set; } = string.Empty; + + public string RowKey { get; set; } = string.Empty; public required TDocument Document { get; set; } - } + } public class MongoBatchRepository : IBatchRepository { private readonly int trainingId; private readonly string trainingIdField; //private readonly Func getId; - private readonly DbCollection collection; - //private readonly DbCollection, TId> collection; + //private readonly DbCollection collection; + private readonly DbCollection, TId> collection; public MongoBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") { - collection = new DbCollection(db, idField, getId); - //collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); + //collection = new DbCollection(db, idField, getId); + collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); this.trainingId = trainingId; this.trainingIdField = trainingIdField; //this.getId = getId; } - //private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); - private FilterDefinition GetTrainingIdFilter() => DbCollection.GetIdFilter(trainingId, trainingIdField); - //public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); - public async Task> GetAll() => await collection.ListAsync(GetTrainingIdFilter()); + private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); + public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + //private FilterDefinition GetTrainingIdFilter() => DbCollection.GetIdFilter(trainingId, trainingIdField); + //public async Task> GetAll() => await collection.ListAsync(GetTrainingIdFilter()); - public async Task RemoveAll() + public async Task RemoveAll() => await collection.RemoveAsync(GetTrainingIdFilter()); public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) { - Func> createFilter = item => - Builders.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); - return await collection.Upsert(items, createFilter); + Func, FilterDefinition>> createFilter = item => + Builders>.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); + var (added, upserted) = await collection.Upsert(items.Select(o => new MongoDocumentWrapper(o)), createFilter); + return (added.Select(o => o.Document), upserted.Select(o => o.Document)); } } @@ -178,25 +227,42 @@ public MongoUserRepository(IMongoDatabase db) : base(db, nameof(User.Email), u = //public async Task Get(string id) => await collection.Find(o => o.Email == id).FirstOrDefaultAsync(); } - public class MongoTrainingRepository : DbCollection, ITrainingRepository + public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository + + //public class MongoTrainingRepository : DbCollection, ITrainingRepository { - public MongoTrainingRepository(IMongoDatabase db) : base(db, nameof(Training.Id), u => u.Id) + private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); + + public MongoTrainingRepository(IMongoDatabase db) : base(db, $"{nameof(MongoDocumentWrapper.Document)}.{nameof(Training.Id)}", u => u.Document.Id) { } public Task Add(Training item) => AddGetId(item); - public async Task AddGetId(Training item) => await InsertGetId(item); + public async Task AddGetId(Training item) + { + await semaphore.WaitAsync(); + + var filter = Builders>.Filter.Empty; + var count = await collection.CountDocumentsAsync(); + item.Id = (int)count + 1; + //await collection.Upsert(new MongoDocumentWrapper(item)); + await Upsert(item); + //await InsertGetId(item); + + semaphore.Release(); + return item.Id; + } public async Task> GetByIds(IEnumerable ids) { var list = ids.ToList(); - return await (await collection.FindAsync(o => list.Contains(o.Id))).ToListAsync(); - } + return await Get(ids); + //return await (await collection.FindAsync(o => list.Contains(o.Id))).ToListAsync(); + } //public Task Remove(Training item) => throw new NotImplementedException(); //public Task Update(Training item) => throw new NotImplementedException(); //public Task Upsert(Training item) => throw new NotImplementedException(); //public async Task Get(int id) => await base.Get(id); //public Task> GetAll() => throw new NotImplementedException(); } - } From dce8b047a944fda93c49041092f906c009858f35 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 13 Dec 2025 16:02:12 +0100 Subject: [PATCH 03/38] split into files --- .../Services/Storage/MongoDb/DbCollection.cs | 74 ++++++ .../Storage/MongoDb/DbWrappedCollection.cs | 27 ++ .../Storage/MongoDb/MongoDocumentWrapper.cs | 31 +++ .../Services/Storage/MongoDb/MongoTools.cs | 28 ++ .../MongoDb/MongoTrainingPlanRepository.cs | 241 +----------------- ...ongoUserGeneratedDataRepositoryProvider.cs | 57 +++++ 6 files changed, 222 insertions(+), 236 deletions(-) create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs new file mode 100644 index 0000000..10b63c5 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs @@ -0,0 +1,74 @@ +using MongoDB.Driver; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class DbCollection + { + protected IMongoCollection collection; + private readonly string idField; + private readonly Func getId; + + public DbCollection(IMongoDatabase db, string idField, Func getId) + { + collection = db.GetCollection(MongoTools.GetCollectionName()); + this.idField = idField; + this.getId = getId; + } + + public async Task InsertGetId(TDocument item) + { + await collection.InsertOneAsync(item); + return getId(item); + } + + public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); + public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); + public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); + + // public static FilterDefinition GetIdFilter(TId_ id, string idField) => Builders.Filter.Eq(idField, id); + //public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); + public static FilterDefinition GetIdFilter(TId id, string idField) => Builders.Filter.Eq(idField, id); + public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); + + public async Task Get(TId id) => await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); + public async Task> Get(IEnumerable ids) => await (await collection.FindAsync(GetIdFilter(ids))).ToListAsync(); + + public async Task> GetAll() => await collection.Find(o => true).ToListAsync(); + + public async Task Remove(TDocument item) + { + var found = await collection.FindOneAndDeleteAsync(GetIdFilter(getId(item))); + } + + public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); + public async Task Upsert(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); + + public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + { + return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); + } + public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + { + var result = await collection.DeleteManyAsync(filter, cancellationToken); + return (int)result.DeletedCount; + } + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) + { + var list = items.ToList(); + var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); + //InsertOneModel + var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); + var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); + + var upserted = upsertedIndices.Select(o => list[o]); + + return (list.Except(upserted), upserted); + } + + public async Task CountDocumentsAsync(FilterDefinition? filter = null) + { + return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); + } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs new file mode 100644 index 0000000..2baac5c --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -0,0 +1,27 @@ +using MongoDB.Driver; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class DbWrappedCollection + { + protected DbCollection, TId> collection; + public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId) + { + collection = new DbCollection, TId>(db, idField, getId); + } + + public Task Remove(TDocument item) => collection.Remove(new MongoDocumentWrapper(item)); + public Task Update(TDocument item) => collection.Update(new MongoDocumentWrapper(item)); + public Task Upsert(TDocument item) => collection.Upsert(new MongoDocumentWrapper(item)); + public async Task Get(TId id) + { + var found = await collection.Get(id); + if (found != null) + return found.Document; + return default; + } + public async Task> Get(IEnumerable ids) => (await collection.Get(ids)).Select(o => o.Document).ToList(); + + public async Task> GetAll() => (await collection.GetAll()).Select(o => o.Document).ToList(); + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs new file mode 100644 index 0000000..7ed3be1 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs @@ -0,0 +1,31 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoDocumentWrapper + { + public MongoDocumentWrapper() + { + Id = ObjectId.GenerateNewId(); + } + + [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] + public MongoDocumentWrapper(TDocument doc) : this() + { + Document = doc; + } + + [BsonId] + public ObjectId Id { get; set; } + + //[BsonRepresentation(BsonType.ObjectId)] + //[BsonId(IdGenerator = typeof(MongoDB.Bson.Serialization.IdGenerators.BsonObjectIdGenerator))] + //[BsonIgnoreIfDefault] + //public string Id { get; set; } = string.Empty; + + public string RowKey { get; set; } = string.Empty; + + public required TDocument Document { get; set; } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs new file mode 100644 index 0000000..612ab06 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs @@ -0,0 +1,28 @@ +using MongoDB.Driver; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoTools + { + public static async Task Initialize() + { + const string uri = "mongodb://localhost:27017/"; + var client = new MongoClient(uri); + var db = client.GetDatabase("training"); + //db.GetCollection<> + } + + public static string GetCollectionName() => GetCollectionName(typeof(T)); + public static string GetCollectionName(Type type) + { + if (type.GenericTypeArguments.Any()) + { + if (type.GetGenericTypeDefinition() == typeof(MongoDocumentWrapper<>)) + { + return type.GenericTypeArguments[0].Name; + } + } + return type.Name; + } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index b6db252..4598a41 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -1,235 +1,16 @@ -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; -using MongoDB.Driver; -using ProblemSource.Models; -using ProblemSource.Models.Aggregates; -using ProblemSource.Services.Storage; +using MongoDB.Driver; using ProblemSourceModule.Models; -using ProblemSourceModule.Models.Aggregates; namespace ProblemSourceModule.Services.Storage.MongoDb { - - public class MongoTools - { - public static async Task Initialize() - { - const string uri = "mongodb://localhost:27017/"; - var client = new MongoClient(uri); - var db = client.GetDatabase("training"); - //db.GetCollection<> - } - - public static string GetCollectionName() => GetCollectionName(typeof(T)); - public static string GetCollectionName(Type type) - { - if (type.GenericTypeArguments.Any()) - { - if (type.GetGenericTypeDefinition() == typeof(MongoDocumentWrapper<>)) - { - return type.GenericTypeArguments[0].Name; - } - } - return type.Name; - } - } - - public class DbWrappedCollection - { - protected DbCollection, TId> collection; - public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId)// : base(db, idField, getId) - { - collection = new DbCollection, TId>(db, idField, getId); - } - - public Task Remove(TDocument item) => collection.Remove(new MongoDocumentWrapper(item)); - public Task Update(TDocument item) => collection.Update(new MongoDocumentWrapper(item)); // TODO: is mongo ID handled properly? - public Task Upsert(TDocument item) => collection.Upsert(new MongoDocumentWrapper(item)); // TODO: is mongo ID handled properly? - public async Task Get(TId id) - { - var found = await collection.Get(id); - if (found != null) - return found.Document; - return default; - } - public async Task> Get(IEnumerable ids) => (await collection.Get(ids)).Select(o => o.Document).ToList(); - - - public async Task> GetAll() => (await collection.GetAll()).Select(o => o.Document).ToList(); - - } - - public class DbCollection - { - protected IMongoCollection collection; - private readonly string idField; - private readonly Func getId; - - public DbCollection(IMongoDatabase db, string idField, Func getId) - { - collection = db.GetCollection(MongoTools.GetCollectionName()); - this.idField = idField; - this.getId = getId; - } - - public async Task InsertGetId(TDocument item) - { - await collection.InsertOneAsync(item); - return getId(item); - } - - public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); - public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); - public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); - - // public static FilterDefinition GetIdFilter(TId_ id, string idField) => Builders.Filter.Eq(idField, id); - //public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); - public static FilterDefinition GetIdFilter(TId id, string idField) => Builders.Filter.Eq(idField, id); - public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); - - public async Task Get(TId id) => await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); - public async Task> Get(IEnumerable ids) => await (await collection.FindAsync(GetIdFilter(ids))).ToListAsync(); - - public async Task> GetAll() => await collection.Find(o => true).ToListAsync(); - - public async Task Remove(TDocument item) - { - var found = await collection.FindOneAndDeleteAsync(GetIdFilter(getId(item))); - } - - public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); - public async Task Upsert(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); - - public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) - { - return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); - } - public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) - { - var result = await collection.DeleteManyAsync(filter, cancellationToken); - return (int)result.DeletedCount; - } - - public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) - { - var list = items.ToList(); - var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); - //InsertOneModel - var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); - var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); - - var upserted = upsertedIndices.Select(o => list[o]); - - return (list.Except(upserted), upserted); - } - - public async Task CountDocumentsAsync(FilterDefinition? filter = null) - { - return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); - } - } - - public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider + public class MongoUserRepository : DbWrappedCollection, IUserRepository { - private readonly IMongoDatabase db; - private readonly int trainingId; - private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; - - public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) - { - this.db = db; - this.trainingId = trainingId; - } - - public IBatchRepository Phases => new MongoBatchRepository(db, Key, Phase.UniqueIdWithinUser, trainingId); - - public IBatchRepository TrainingDays => throw new NotImplementedException(); - - public IBatchRepository PhaseStatistics => throw new NotImplementedException(); - - public IBatchRepository TrainingSummaries => throw new NotImplementedException(); - - public IBatchRepository UserStates => throw new NotImplementedException(); - - public Task RemoveAll() - { - throw new NotImplementedException(); - } - } - - public class MongoDocumentWrapper - { - public MongoDocumentWrapper() - { - Id = ObjectId.GenerateNewId(); - } - - [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] - public MongoDocumentWrapper(TDocument doc) : this() - { - Document = doc; - } - - [BsonId] - public ObjectId Id { get; set; } - - //[BsonRepresentation(BsonType.ObjectId)] - //[BsonId(IdGenerator = typeof(MongoDB.Bson.Serialization.IdGenerators.BsonObjectIdGenerator))] - //[BsonIgnoreIfDefault] - //public string Id { get; set; } = string.Empty; - - public string RowKey { get; set; } = string.Empty; - - public required TDocument Document { get; set; } - } - - public class MongoBatchRepository : IBatchRepository - { - private readonly int trainingId; - private readonly string trainingIdField; - //private readonly Func getId; - //private readonly DbCollection collection; - private readonly DbCollection, TId> collection; - - public MongoBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") - { - //collection = new DbCollection(db, idField, getId); - collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); - - this.trainingId = trainingId; - this.trainingIdField = trainingIdField; - //this.getId = getId; - } - - private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); - public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); - //private FilterDefinition GetTrainingIdFilter() => DbCollection.GetIdFilter(trainingId, trainingIdField); - //public async Task> GetAll() => await collection.ListAsync(GetTrainingIdFilter()); - - public async Task RemoveAll() - => await collection.RemoveAsync(GetTrainingIdFilter()); - - public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) - { - Func, FilterDefinition>> createFilter = item => - Builders>.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); - var (added, upserted) = await collection.Upsert(items.Select(o => new MongoDocumentWrapper(o)), createFilter); - return (added.Select(o => o.Document), upserted.Select(o => o.Document)); - } - } - - - public class MongoUserRepository : DbCollection, IUserRepository - { - public MongoUserRepository(IMongoDatabase db) : base(db, nameof(User.Email), u => u.Email) + public MongoUserRepository(IMongoDatabase db) : base(db, nameof(User.Email), u => u.Document.Email) { } - public async Task Add(User item) => await InsertGetId(item); - //public async Task Get(string id) => await collection.Find(o => o.Email == id).FirstOrDefaultAsync(); + public async Task Add(User item) => await Upsert(item); } public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository - - //public class MongoTrainingRepository : DbCollection, ITrainingRepository { private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); @@ -245,24 +26,12 @@ public async Task AddGetId(Training item) var filter = Builders>.Filter.Empty; var count = await collection.CountDocumentsAsync(); item.Id = (int)count + 1; - //await collection.Upsert(new MongoDocumentWrapper(item)); await Upsert(item); - //await InsertGetId(item); semaphore.Release(); return item.Id; } - public async Task> GetByIds(IEnumerable ids) - { - var list = ids.ToList(); - return await Get(ids); - //return await (await collection.FindAsync(o => list.Contains(o.Id))).ToListAsync(); - } - //public Task Remove(Training item) => throw new NotImplementedException(); - //public Task Update(Training item) => throw new NotImplementedException(); - //public Task Upsert(Training item) => throw new NotImplementedException(); - //public async Task Get(int id) => await base.Get(id); - //public Task> GetAll() => throw new NotImplementedException(); + public async Task> GetByIds(IEnumerable ids) => await Get(ids); } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs new file mode 100644 index 0000000..143ef7f --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs @@ -0,0 +1,57 @@ +using MongoDB.Driver; +using ProblemSource.Models; +using ProblemSource.Models.Aggregates; +using ProblemSource.Services.Storage; +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider + { + private readonly IMongoDatabase db; + private readonly int trainingId; + private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; + + public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) + { + this.db = db; + this.trainingId = trainingId; + } + + public IBatchRepository Phases => new MongoBatchRepository(db, Key, Phase.UniqueIdWithinUser, trainingId); + public IBatchRepository TrainingDays => new MongoBatchRepository(db, Key, item => item.TrainingDay, trainingId); + public IBatchRepository PhaseStatistics => new MongoBatchRepository(db, Key, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); + public IBatchRepository TrainingSummaries => new MongoBatchRepository(db, Key, item => "x", trainingId); + public IBatchRepository UserStates => new MongoBatchRepository(db, Key, item => "x", trainingId); + + public Task RemoveAll() => throw new NotImplementedException(); + } + + public class MongoBatchRepository : IBatchRepository + { + private readonly int trainingId; + private readonly string trainingIdField; + private readonly DbCollection, TId> collection; + + public MongoBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") + { + collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); + + this.trainingId = trainingId; + this.trainingIdField = trainingIdField; + } + + private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); + public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + + public async Task RemoveAll() => await collection.RemoveAsync(GetTrainingIdFilter()); + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) + { + Func, FilterDefinition>> createFilter = item => + Builders>.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); + var (added, upserted) = await collection.Upsert(items.Select(o => new MongoDocumentWrapper(o)), createFilter); + return (added.Select(o => o.Document), upserted.Select(o => o.Document)); + } + } +} From 4104153e33e6f4756ce7d15ef0fa0a28ab96885c Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 14 Dec 2025 14:45:53 +0100 Subject: [PATCH 04/38] experiments --- .../ProblemSourceModule.Tests/MongoDbTests.cs | 47 +++++++-- .../Services/Storage/MongoDb/DbCollection.cs | 55 +++++++++-- .../Storage/MongoDb/DbWrappedCollection.cs | 61 ++++++++++-- .../Services/Storage/MongoDb/DocumentBase.cs | 17 ++++ .../Storage/MongoDb/MongoDocumentWrapper.cs | 19 ++-- .../Services/Storage/MongoDb/MongoTools.cs | 18 ++-- .../MongoDb/MongoTrainingBatchRepository.cs | 65 +++++++++++++ .../MongoDb/MongoTrainingPlanRepository.cs | 5 +- ...ongoUserGeneratedDataRepositoryProvider.cs | 96 ++++++++----------- 9 files changed, 283 insertions(+), 100 deletions(-) create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DocumentBase.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs diff --git a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs index 7ad5d9d..756e9dc 100644 --- a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs +++ b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs @@ -30,7 +30,10 @@ public async Task X() // x.GetMemberMap(m => m.Id).SetIgnoreIfDefault(true); //}); - var db = new MongoClient(runner.ConnectionString).GetDatabase("_mydb"); + var client = new MongoClient(runner.ConnectionString); + var dbName = "_mydb"; + await client.DropDatabaseAsync(dbName); + var db = client.GetDatabase(dbName); var sut = new MongoTrainingRepository(db); var training = new Training { Username = "ABV", Created = DateTimeOffset.UtcNow }; var id = await sut.AddGetId(training); @@ -38,16 +41,48 @@ public async Task X() retrieved?.Username.ShouldBe(training.Username); - var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); + var collections = await (await db.ListCollectionsAsync()).ToListAsync(); - var phase = new Phase { exercise = "a", phase_type = "TEST", training_day = 1, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; - var result = await ugdr.Phases.Upsert([phase]); + var phaseA = new Phase { exercise = "a", phase_type = "TEST", training_day = 1, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; + + var batchRepo = new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, id); + var collectionName = batchRepo.GetCollection().CollectionNamespace.CollectionName; + await db.DropCollectionAsync(collectionName); + //collections = await (await db.ListCollectionsAsync()).ToListAsync(); + + var result = await batchRepo.Upsert([phaseA]); result.Added.Count().ShouldBe(1); result.Updated.Count().ShouldBe(0); - result = await ugdr.Phases.Upsert([phase]); - result.Added.Count().ShouldBe(0); + var underlyingCollection = batchRepo.GetCollection(); + var count = await underlyingCollection.CountDocumentsAsync(o => true); + count.ShouldBe(1); + + var phaseB = new Phase { exercise = "a", phase_type = "TEST", training_day = 2, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; + + result = await batchRepo.Upsert([phaseA, phaseB]); + result.Added.Count().ShouldBe(1); result.Updated.Count().ShouldBe(1); + count = await underlyingCollection.CountDocumentsAsync(o => true); + count.ShouldBe(2); + + + //await MongoTools.DropCollection>(db); + //var phaseColl = MongoTools.GetCollection>(db); + + //var allPhases = await (await phaseColl.FindAsync(o => true)).ToListAsync(); + + //var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); + + //var result = await ugdr.Phases.Upsert([phase]); + //result.Added.Count().ShouldBe(1); + //result.Updated.Count().ShouldBe(0); + + //result = await ugdr.Phases.Upsert([phase]); + //result.Added.Count().ShouldBe(0); + //result.Updated.Count().ShouldBe(1); } + + } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs index 10b63c5..aa88d8b 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs @@ -1,8 +1,10 @@ -using MongoDB.Driver; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; namespace ProblemSourceModule.Services.Storage.MongoDb { - public class DbCollection + public class DbCollection where TDocument : DocumentBase { protected IMongoCollection collection; private readonly string idField; @@ -21,6 +23,7 @@ public async Task InsertGetId(TDocument item) return getId(item); } + public IMongoCollection GetCollection() => collection; public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); @@ -55,18 +58,50 @@ public async Task RemoveAsync(FilterDefinition filter, Cancellat public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) { - var list = items.ToList(); - var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); - //InsertOneModel - var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); - var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); + var itemsWithId = items.Select(o => new { Id = getId(o), Item = o }).ToList(); + + var filter = Builders.Filter.In(idField, itemsWithId.Select(o => o.Id)); + //var projection = new FindExpressionProjectionDefinition(p => new X { Id = p.Id, SubId = getId(p) }); + //ProjectionDefinition projection = $$"""{ "Id": "Id", "SubId": "{{idField}}" }"""; + var tmpAll = await collection.Find(o => true).ToListAsync(); + var projection = Builders.Projection.Include(idField); //Include("Id"). + var tmpX = (await collection.Find(filter).Project(projection).ToListAsync()) + .Select(o => new { Id = o["_id"].AsObjectId, SubId = o[idField]?.ToString() }) + //.Select(o => BsonSerializer.Deserialize(o)) + .Where(o => o.SubId != null).ToList(); + if (tmpX == null) + throw new Exception("A"); + //var bsonIdById = tmpX.ToDictionary(o => o.SubId!, o => o.Id); + + var toInsert = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == false).ToList(); + var toReplace = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == true).ToList(); + //var toInsert = itemsWithId.Select(o => bsonIdById.GetValueOrDefault(o.Id!.ToString())).ToList(); + //var toReplace = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == true).ToList(); + + var models = toInsert.Select(o => (WriteModel)new InsertOneModel(o.Item)).ToList(); - var upserted = upsertedIndices.Select(o => list[o]); + models.AddRange(toReplace.Select(o => + { + var found = tmpX.Single(p => p.SubId?.Equals(o.Id) == true); + if (found == null) + throw new Exception("aaa"); + o.Item.Id = found.Id; + return new ReplaceOneModel(createFilter(o.Item), o.Item); + })); - return (list.Except(upserted), upserted); + var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); + // hm, we can't tell which were inserted and which were replaced?! + return (toInsert.Select(o => o.Item), toReplace.Select(o => o.Item)); + } + + private class X + { + public ObjectId Id { get; set; } + //public object? SubId { get; set; } + public TId? SubId { get; set; } } - public async Task CountDocumentsAsync(FilterDefinition? filter = null) + public async Task CountDocumentsAsync(FilterDefinition? filter = null) { return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs index 2baac5c..2a68620 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -5,14 +5,63 @@ namespace ProblemSourceModule.Services.Storage.MongoDb public class DbWrappedCollection { protected DbCollection, TId> collection; - public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId) - { - collection = new DbCollection, TId>(db, idField, getId); + private readonly Func getId; + private readonly Func> createWrapped; + + //private readonly Func, TId> getId; + + + //public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId) + //public DbWrappedCollection(IMongoDatabase db, string idField, Func getId) + public DbWrappedCollection(IMongoDatabase db, Func getId, Func> createWrapped) + { + collection = new DbCollection, TId>(db, nameof(MongoDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); + this.getId = getId; + this.createWrapped = createWrapped; + } + + public IMongoCollection> GetCollection() => collection.GetCollection(); + + private MongoDocumentWrapper CreateWrapped(TDocument item) => createWrapped(item); // new MongoDocumentWrapper(item, o => getId(o)?.ToString() ?? ""); + + public Task Remove(TDocument item) => collection.Remove(CreateWrapped(item)); + //public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + //{ + // var result = await collection.RemoveAsync(filter, cancellationToken); + // return (int)result.DeletedCount; + //} + //public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) + //{ + // return (await collection.ListAsync(filter)).Select(o => o.Document).ToList(); + // //return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); + //} + + //public string GetIdFilter(MongoDocumentWrapper item) + //{ + // return "AAA"; + //} + + //public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) + //{ + // return ([], []); + //} + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func, FilterDefinition>> createFilter) + { + var result = await collection.Upsert(items.Select(CreateWrapped), createFilter); + return (result.Added.Select(o => o.Document), result.Updated.Select(o => o.Document)); + //var list = items.ToList(); + //var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); + ////InsertOneModel + //var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); + //var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); + + //var upserted = upsertedIndices.Select(o => list[o]); + + //return (list.Except(upserted), upserted); } - public Task Remove(TDocument item) => collection.Remove(new MongoDocumentWrapper(item)); - public Task Update(TDocument item) => collection.Update(new MongoDocumentWrapper(item)); - public Task Upsert(TDocument item) => collection.Upsert(new MongoDocumentWrapper(item)); + public Task Update(TDocument item) => collection.Update(CreateWrapped(item)); + public Task Upsert(TDocument item) => collection.Upsert(CreateWrapped(item)); public async Task Get(TId id) { var found = await collection.Get(id); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DocumentBase.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DocumentBase.cs new file mode 100644 index 0000000..de999a0 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DocumentBase.cs @@ -0,0 +1,17 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class DocumentBase + { + //[BsonRepresentation(BsonType.ObjectId)] + //[BsonId(IdGenerator = typeof(MongoDB.Bson.Serialization.IdGenerators.BsonObjectIdGenerator))] + //[BsonIgnoreIfDefault] + //public string Id { get; set; } = string.Empty; + + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public ObjectId Id { get; set; } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs index 7ed3be1..00d0dcd 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs @@ -1,29 +1,22 @@ using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; namespace ProblemSourceModule.Services.Storage.MongoDb { - public class MongoDocumentWrapper - { + public class MongoDocumentWrapper : DocumentBase + { public MongoDocumentWrapper() { - Id = ObjectId.GenerateNewId(); + //Id = ObjectId.GenerateNewId(); } [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] - public MongoDocumentWrapper(TDocument doc) : this() + public MongoDocumentWrapper(TDocument doc, Func? getId = null) : this() { Document = doc; + if (getId != null) + RowKey = getId(doc); } - [BsonId] - public ObjectId Id { get; set; } - - //[BsonRepresentation(BsonType.ObjectId)] - //[BsonId(IdGenerator = typeof(MongoDB.Bson.Serialization.IdGenerators.BsonObjectIdGenerator))] - //[BsonIgnoreIfDefault] - //public string Id { get; set; } = string.Empty; - public string RowKey { get; set; } = string.Empty; public required TDocument Document { get; set; } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs index 612ab06..8491498 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs @@ -4,13 +4,19 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { public class MongoTools { - public static async Task Initialize() + // public static async Task Initialize() + // { + // const string uri = "mongodb://localhost:27017/"; + // var client = new MongoClient(uri); + // var db = client.GetDatabase("training"); + //} + + public static IMongoCollection GetCollection(IMongoDatabase db) => db.GetCollection(GetCollectionName(typeof(T))); + + public static async Task DropCollection(IMongoDatabase db) { - const string uri = "mongodb://localhost:27017/"; - var client = new MongoClient(uri); - var db = client.GetDatabase("training"); - //db.GetCollection<> - } + await db.DropCollectionAsync(GetCollectionName()); + } public static string GetCollectionName() => GetCollectionName(typeof(T)); public static string GetCollectionName(Type type) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs new file mode 100644 index 0000000..6123916 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs @@ -0,0 +1,65 @@ +using MongoDB.Driver; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoTrainingAssociatedDocumentWrapper : MongoDocumentWrapper // where TDocument : DocumentBase + { + public MongoTrainingAssociatedDocumentWrapper() { } + + [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] + public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Func? getId = null) : base(doc, getId) + { + TrainingId = trainingId; + } + public int TrainingId { get; set; } + } + + public class MongoTrainingBatchRepository //: IBatchRepository + { + private readonly Func getId; + private readonly int trainingId; + //private readonly string trainingIdField; + //private readonly DbCollection, TId> collection; + private readonly DbWrappedCollection collection; + //private readonly DbWrappedCollection collection; + + public IMongoCollection> GetCollection() => collection.GetCollection(); + + //public MongoTrainingBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") + public MongoTrainingBatchRepository(IMongoDatabase db, Func getId, int trainingId) //, string trainingIdField = "trainingId") + { + //collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); + //collection = new DbCollection, TId>(db, $"{idField}", item => getId(item.Document)); + collection = new DbWrappedCollection(db, getId, item => new MongoTrainingAssociatedDocumentWrapper(item, trainingId, o => getId(o)?.ToString() ?? "")); + this.getId = getId; + this.trainingId = trainingId; + //this.trainingIdField = trainingIdField; + } + + private FilterDefinition> GetTrainingIdFilter() + => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); + //private FilterDefinition> GetTrainingIdFilter() + // => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); + //private FilterDefinition> GetTrainingIdFilter() + // => DbCollection, int>.GetIdFilter(trainingId, nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)); //$"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); + + //public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + + //public async Task RemoveAll() => await collection.RemoveAsync(GetTrainingIdFilter()); + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) + { + Func, FilterDefinition>> createFilter = item => + Builders>.Filter.And( + GetTrainingIdFilter(), + Builders>.Filter.Eq(o => o.RowKey, getId(item.Document)?.ToString()) //nameof(MongoDocumentWrapper.RowKey), o => getId(o.Document)?.ToString() + ); + //var wrapped = items.Select(o => new MongoDocumentWrapper(o)).ToList(); + //foreach (var item in wrapped) + // item.RowKey = getId(wrapped); + var (added, upserted) = await collection.Upsert(items, createFilter); + //return (added.Select(o => o.Document), upserted.Select(o => o.Document)); + return (added, upserted); + } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index 4598a41..3263e62 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -5,7 +5,7 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { public class MongoUserRepository : DbWrappedCollection, IUserRepository { - public MongoUserRepository(IMongoDatabase db) : base(db, nameof(User.Email), u => u.Document.Email) + public MongoUserRepository(IMongoDatabase db) : base(db, u => u.Email, item => new MongoDocumentWrapper(item, o => o.Email)) //nameof(User.Email), u => u.Document.Email { } public async Task Add(User item) => await Upsert(item); } @@ -14,7 +14,8 @@ public class MongoTrainingRepository : DbWrappedCollection, ITrai { private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - public MongoTrainingRepository(IMongoDatabase db) : base(db, $"{nameof(MongoDocumentWrapper.Document)}.{nameof(Training.Id)}", u => u.Document.Id) + //public MongoTrainingRepository(IMongoDatabase db) : base(db, $"{nameof(MongoDocumentWrapper.Document)}.{nameof(Training.Id)}", u => u.Document.Id) + public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id, item => new MongoDocumentWrapper(item, o => o.Id.ToString())) { } public Task Add(Training item) => AddGetId(item); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs index 143ef7f..a812688 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs @@ -1,57 +1,39 @@ -using MongoDB.Driver; -using ProblemSource.Models; -using ProblemSource.Models.Aggregates; -using ProblemSource.Services.Storage; -using ProblemSourceModule.Models.Aggregates; - -namespace ProblemSourceModule.Services.Storage.MongoDb -{ - public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider - { - private readonly IMongoDatabase db; - private readonly int trainingId; - private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; - - public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) - { - this.db = db; - this.trainingId = trainingId; - } - - public IBatchRepository Phases => new MongoBatchRepository(db, Key, Phase.UniqueIdWithinUser, trainingId); - public IBatchRepository TrainingDays => new MongoBatchRepository(db, Key, item => item.TrainingDay, trainingId); - public IBatchRepository PhaseStatistics => new MongoBatchRepository(db, Key, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); - public IBatchRepository TrainingSummaries => new MongoBatchRepository(db, Key, item => "x", trainingId); - public IBatchRepository UserStates => new MongoBatchRepository(db, Key, item => "x", trainingId); - - public Task RemoveAll() => throw new NotImplementedException(); - } - - public class MongoBatchRepository : IBatchRepository - { - private readonly int trainingId; - private readonly string trainingIdField; - private readonly DbCollection, TId> collection; - - public MongoBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") - { - collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); - - this.trainingId = trainingId; - this.trainingIdField = trainingIdField; - } - - private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); - public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); - - public async Task RemoveAll() => await collection.RemoveAsync(GetTrainingIdFilter()); - - public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) - { - Func, FilterDefinition>> createFilter = item => - Builders>.Filter.And(GetTrainingIdFilter(), collection.GetIdFilter(item)); - var (added, upserted) = await collection.Upsert(items.Select(o => new MongoDocumentWrapper(o)), createFilter); - return (added.Select(o => o.Document), upserted.Select(o => o.Document)); - } - } -} +//using MongoDB.Driver; +//using ProblemSource.Models; +//using ProblemSource.Models.Aggregates; +//using ProblemSource.Services.Storage; +//using ProblemSourceModule.Models.Aggregates; + +//namespace ProblemSourceModule.Services.Storage.MongoDb +//{ +// public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider +// { +// private readonly IMongoDatabase db; +// private readonly int trainingId; +// private readonly string Key = $"{nameof(MongoDocumentWrapper.RowKey)}"; +// //private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; + +// public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) +// { +// this.db = db; +// this.trainingId = trainingId; +// } + +// public IBatchRepository Phases +// => new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, trainingId); //Key, + +// public IBatchRepository TrainingDays +// => new MongoTrainingBatchRepository(db, item => item.TrainingDay, trainingId); // "AccountId", + +// public IBatchRepository PhaseStatistics +// => new MongoTrainingBatchRepository(db, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); // "account_id", + +// public IBatchRepository TrainingSummaries +// => new MongoTrainingBatchRepository(db, item => "x", trainingId); // "AccountId", + +// public IBatchRepository UserStates +// => new MongoTrainingBatchRepository(db, item => "x", trainingId); // Key, + +// public Task RemoveAll() => throw new NotImplementedException(); +// } +//} From 3124b1967d264c6019fe7350cd8d6702a0b979ea Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 14 Dec 2025 16:09:20 +0100 Subject: [PATCH 05/38] semi-working --- .../ProblemSourceModule.Tests/MongoDbTests.cs | 35 +++---- .../ProblemSourceModule.cs | 34 ++++++- .../Services/IStatisticsProvider.cs | 27 +++--- .../AzureTableTrainingSummaryRepository.cs | 26 ++++++ .../Storage/ITrainingSummaryRepository.cs | 9 ++ .../Services/Storage/MongoDb/DbCollection.cs | 25 +++-- .../Storage/MongoDb/DbWrappedCollection.cs | 22 ++--- .../MongoDb/MongoTrainingBatchRepository.cs | 38 +++----- .../MongoDb/MongoTrainingSummaryRepository.cs | 18 ++++ ...ongoUserGeneratedDataRepositoryProvider.cs | 92 +++++++++++-------- Tools/Program.cs | 3 +- 11 files changed, 206 insertions(+), 123 deletions(-) create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingSummaryRepository.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/ITrainingSummaryRepository.cs create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs diff --git a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs index 756e9dc..827cacf 100644 --- a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs +++ b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs @@ -41,48 +41,41 @@ public async Task X() retrieved?.Username.ShouldBe(training.Username); + var training2 = new Training { Username = "ABXC", Created = DateTimeOffset.UtcNow }; + var id2 = await sut.AddGetId(training2); + (await sut.Get(id2))!.Username.ShouldBe(training2.Username); + var collections = await (await db.ListCollectionsAsync()).ToListAsync(); var phaseA = new Phase { exercise = "a", phase_type = "TEST", training_day = 1, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; - var batchRepo = new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, id); - var collectionName = batchRepo.GetCollection().CollectionNamespace.CollectionName; + var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); + var phasesRepo = ugdr.Phases; // new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, id); + + var mongoTyped = (MongoTrainingBatchRepository)phasesRepo; + var collectionName = mongoTyped.GetCollection().CollectionNamespace.CollectionName; await db.DropCollectionAsync(collectionName); //collections = await (await db.ListCollectionsAsync()).ToListAsync(); - var result = await batchRepo.Upsert([phaseA]); + var result = await phasesRepo.Upsert([phaseA]); result.Added.Count().ShouldBe(1); result.Updated.Count().ShouldBe(0); - var underlyingCollection = batchRepo.GetCollection(); + var underlyingCollection = mongoTyped.GetCollection(); var count = await underlyingCollection.CountDocumentsAsync(o => true); count.ShouldBe(1); var phaseB = new Phase { exercise = "a", phase_type = "TEST", training_day = 2, time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }; - result = await batchRepo.Upsert([phaseA, phaseB]); + result = await phasesRepo.Upsert([phaseA, phaseB]); result.Added.Count().ShouldBe(1); result.Updated.Count().ShouldBe(1); count = await underlyingCollection.CountDocumentsAsync(o => true); count.ShouldBe(2); - //await MongoTools.DropCollection>(db); - //var phaseColl = MongoTools.GetCollection>(db); - - //var allPhases = await (await phaseColl.FindAsync(o => true)).ToListAsync(); - - //var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); - - //var result = await ugdr.Phases.Upsert([phase]); - //result.Added.Count().ShouldBe(1); - //result.Updated.Count().ShouldBe(0); - - //result = await ugdr.Phases.Upsert([phase]); - //result.Added.Count().ShouldBe(0); - //result.Updated.Count().ShouldBe(1); + var ugdr2 = new MongoUserGeneratedDataRepositoryProvider(db, id2); + (await ugdr2.Phases.GetAll()).ShouldBeEmpty(); } - - } } diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 238a1c7..7ac903d 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using MongoDB.Driver; using PluginModuleBase; using ProblemSource.Services; using ProblemSource.Services.Storage; @@ -11,6 +12,7 @@ using ProblemSourceModule.Services; using ProblemSourceModule.Services.Storage; using ProblemSourceModule.Services.Storage.AzureTables; +using ProblemSourceModule.Services.Storage.MongoDb; using ProblemSourceModule.Services.TrainingAnalyzers; using System.Linq; @@ -57,8 +59,14 @@ public void ConfigureServices(IServiceCollection services) services.AddMemoryCache(); services.AddSingleton(); - ConfigureForAzureTables(services); - ConfigureUsernameHashing(services); + + var storageIsMongo = true; + if (storageIsMongo) + ConfigureForMongoDb(services); + else + ConfigureForAzureTables(services); + + ConfigureUsernameHashing(services); } public void ConfigureForAzureTables(IServiceCollection services, bool useCaching = true) @@ -67,7 +75,10 @@ public void ConfigureForAzureTables(IServiceCollection services, bool useCaching services.AddSingleton(); services.UpsertSingleton(sp => sp.GetRequiredService()); - if (useCaching) + services.AddSingleton(); + + + if (useCaching) services.AddSingleton(); else services.AddSingleton(); @@ -75,7 +86,22 @@ public void ConfigureForAzureTables(IServiceCollection services, bool useCaching services.AddSingleton(); } - public void ConfigureUsernameHashing(IServiceCollection services) + public void ConfigureForMongoDb(IServiceCollection services) + { + var connectionString = "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; + var database = "_Training"; + var client = new MongoClient(connectionString); + services.AddSingleton(sp => client.GetDatabase(database)); + + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + + services.AddSingleton(); + } + + public void ConfigureUsernameHashing(IServiceCollection services) { services.AddSingleton(new MnemoJapanese(2)); services.AddSingleton(sp => new UsernameHashing(sp.GetRequiredService(), 2)); diff --git a/ProblemSource/ProblemSourceModule/Services/IStatisticsProvider.cs b/ProblemSource/ProblemSourceModule/Services/IStatisticsProvider.cs index eb37175..32fde10 100644 --- a/ProblemSource/ProblemSourceModule/Services/IStatisticsProvider.cs +++ b/ProblemSource/ProblemSourceModule/Services/IStatisticsProvider.cs @@ -4,6 +4,7 @@ using ProblemSource.Services.Storage; using ProblemSource.Services.Storage.AzureTables; using ProblemSourceModule.Models.Aggregates; +using ProblemSourceModule.Services.Storage; namespace ProblemSource.Services { @@ -18,12 +19,15 @@ public interface IStatisticsProvider public class StatisticsProvider : IStatisticsProvider { private readonly IUserGeneratedDataRepositoryProviderFactory userGeneratedDataRepositoryProviderFactory; - private readonly ITypedTableClientFactory typedTableClientFactory; + private readonly ITrainingSummaryRepository trainingSummaryRepository; - public StatisticsProvider(IUserGeneratedDataRepositoryProviderFactory userGeneratedDataRepositoryProviderFactory, ITypedTableClientFactory typedTableClientFactory) + //private readonly ITypedTableClientFactory typedTableClientFactory; + + public StatisticsProvider(IUserGeneratedDataRepositoryProviderFactory userGeneratedDataRepositoryProviderFactory, ITrainingSummaryRepository trainingSummaryRepository) //ITypedTableClientFactory typedTableClientFactory) { this.userGeneratedDataRepositoryProviderFactory = userGeneratedDataRepositoryProviderFactory; - this.typedTableClientFactory = typedTableClientFactory; + this.trainingSummaryRepository = trainingSummaryRepository; + //this.typedTableClientFactory = typedTableClientFactory; } private IUserGeneratedDataRepositoryProvider GetDataProvider(int trainingId) => @@ -49,14 +53,15 @@ public async Task> GetTrainingDays(int trainingI public async Task> GetAllTrainingSummaries() { - var q = typedTableClientFactory.TrainingSummaries.QueryAsync(""); - var converter = new ExpandableTableEntityConverter(t => new TableFilter("none")); - var result = new List(); - await foreach (var item in q) - { - result.Add(converter.ToPoco(item)); - } - return result; + return await trainingSummaryRepository.GetAll(); + //var q = typedTableClientFactory.TrainingSummaries.QueryAsync(""); + //var converter = new ExpandableTableEntityConverter(t => new TableFilter("none")); + //var result = new List(); + //await foreach (var item in q) + //{ + // result.Add(converter.ToPoco(item)); + //} + //return result; } } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingSummaryRepository.cs new file mode 100644 index 0000000..d17aed8 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingSummaryRepository.cs @@ -0,0 +1,26 @@ +using Azure.Data.Tables; +using AzureTableGenerics; +using ProblemSource.Services.Storage.AzureTables; +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage.AzureTables +{ + public class AzureTableTrainingSummaryRepository : ITrainingSummaryRepository + { + private readonly ITypedTableClientFactory typedTableClientFactory; + + public AzureTableTrainingSummaryRepository(ITypedTableClientFactory typedTableClientFactory) + { + this.typedTableClientFactory = typedTableClientFactory; + } + public async Task> GetAll() + { + var q = typedTableClientFactory.TrainingSummaries.QueryAsync(""); + var converter = new ExpandableTableEntityConverter(t => new TableFilter("none")); + var result = new List(); + await foreach (var item in q) + result.Add(converter.ToPoco(item)); + return result; + } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingSummaryRepository.cs new file mode 100644 index 0000000..faa305b --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/ITrainingSummaryRepository.cs @@ -0,0 +1,9 @@ +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage +{ + public interface ITrainingSummaryRepository + { + Task> GetAll(); + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs index aa88d8b..8876cf5 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs @@ -44,7 +44,12 @@ public async Task Remove(TDocument item) } public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); - public async Task Upsert(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); + public async Task Upsert(TDocument item) + { + await Upsert([item]); + //var filter = GetFilter([item]); + //await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); + } public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) { @@ -56,16 +61,24 @@ public async Task RemoveAsync(FilterDefinition filter, Cancellat return (int)result.DeletedCount; } - public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) + private FilterDefinition GetFilter(IEnumerable items, FilterDefinition? globalFilter = null) + { + var filter = Builders.Filter.In(idField, items.Select(getId)); + if (globalFilter != null) + filter = Builders.Filter.And(globalFilter, filter); + return filter; + } + + // Func> createFilter, + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, FilterDefinition? globalFilter = null) { var itemsWithId = items.Select(o => new { Id = getId(o), Item = o }).ToList(); - var filter = Builders.Filter.In(idField, itemsWithId.Select(o => o.Id)); //var projection = new FindExpressionProjectionDefinition(p => new X { Id = p.Id, SubId = getId(p) }); //ProjectionDefinition projection = $$"""{ "Id": "Id", "SubId": "{{idField}}" }"""; - var tmpAll = await collection.Find(o => true).ToListAsync(); + //var tmpAll = await collection.Find(o => true).ToListAsync(); var projection = Builders.Projection.Include(idField); //Include("Id"). - var tmpX = (await collection.Find(filter).Project(projection).ToListAsync()) + var tmpX = (await collection.Find(GetFilter(items, globalFilter)).Project(projection).ToListAsync()) .Select(o => new { Id = o["_id"].AsObjectId, SubId = o[idField]?.ToString() }) //.Select(o => BsonSerializer.Deserialize(o)) .Where(o => o.SubId != null).ToList(); @@ -86,7 +99,7 @@ public async Task RemoveAsync(FilterDefinition filter, Cancellat if (found == null) throw new Exception("aaa"); o.Item.Id = found.Id; - return new ReplaceOneModel(createFilter(o.Item), o.Item); + return new ReplaceOneModel(Builders.Filter.Eq(p => p.Id, found.Id), o.Item); //createFilter(o.Item) })); var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs index 2a68620..ce2cb84 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -45,23 +45,14 @@ public DbWrappedCollection(IMongoDatabase db, Func getId, Func Added, IEnumerable Updated)> Upsert(IEnumerable items, Func, FilterDefinition>> createFilter) + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, FilterDefinition>? globalFilter = null) //, Func, FilterDefinition>> createFilter) { - var result = await collection.Upsert(items.Select(CreateWrapped), createFilter); + var result = await collection.Upsert(items.Select(CreateWrapped), globalFilter); //createFilter return (result.Added.Select(o => o.Document), result.Updated.Select(o => o.Document)); - //var list = items.ToList(); - //var models = items.Select(o => new ReplaceOneModel(createFilter(o), o) { IsUpsert = true }); - ////InsertOneModel - //var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); - //var upsertedIndices = results.Upserts.Select(o => o.Index).ToList(); - - //var upserted = upsertedIndices.Select(o => list[o]); - - //return (list.Except(upserted), upserted); } - public Task Update(TDocument item) => collection.Update(CreateWrapped(item)); - public Task Upsert(TDocument item) => collection.Upsert(CreateWrapped(item)); + public Task Update(TDocument item) => Upsert([item]); // TODO: throw if not already existing collection.Update(CreateWrapped(item)); + public Task Upsert(TDocument item) => Upsert([item]); // collection.Upsert(CreateWrapped(item)); public async Task Get(TId id) { var found = await collection.Get(id); @@ -72,5 +63,10 @@ public DbWrappedCollection(IMongoDatabase db, Func getId, Func> Get(IEnumerable ids) => (await collection.Get(ids)).Select(o => o.Document).ToList(); public async Task> GetAll() => (await collection.GetAll()).Select(o => o.Document).ToList(); + public async Task> GetAll(FilterDefinition> filter) + => (await collection.ListAsync(filter)).Select(o => o.Document).ToList(); + public async Task RemoveAll(FilterDefinition> filter) + => await collection.RemoveAsync(filter); + } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs index 6123916..c1876f3 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs @@ -1,4 +1,5 @@ using MongoDB.Driver; +using ProblemSource.Services.Storage; namespace ProblemSourceModule.Services.Storage.MongoDb { @@ -14,51 +15,34 @@ public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Fun public int TrainingId { get; set; } } - public class MongoTrainingBatchRepository //: IBatchRepository + public class MongoTrainingBatchRepository : IBatchRepository { private readonly Func getId; private readonly int trainingId; - //private readonly string trainingIdField; - //private readonly DbCollection, TId> collection; private readonly DbWrappedCollection collection; - //private readonly DbWrappedCollection collection; public IMongoCollection> GetCollection() => collection.GetCollection(); - //public MongoTrainingBatchRepository(IMongoDatabase db, string idField, Func getId, int trainingId, string trainingIdField = "trainingId") - public MongoTrainingBatchRepository(IMongoDatabase db, Func getId, int trainingId) //, string trainingIdField = "trainingId") + public MongoTrainingBatchRepository(IMongoDatabase db, Func getId, int trainingId) { - //collection = new DbCollection, TId>(db, $"{nameof(MongoDocumentWrapper.Document)}.{idField}", item => getId(item.Document)); - //collection = new DbCollection, TId>(db, $"{idField}", item => getId(item.Document)); collection = new DbWrappedCollection(db, getId, item => new MongoTrainingAssociatedDocumentWrapper(item, trainingId, o => getId(o)?.ToString() ?? "")); this.getId = getId; this.trainingId = trainingId; - //this.trainingIdField = trainingIdField; } private FilterDefinition> GetTrainingIdFilter() => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); - //private FilterDefinition> GetTrainingIdFilter() - // => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); - //private FilterDefinition> GetTrainingIdFilter() - // => DbCollection, int>.GetIdFilter(trainingId, nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)); //$"{nameof(MongoDocumentWrapper.Document)}.{trainingIdField}"); - - //public async Task> GetAll() => (await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); - - //public async Task RemoveAll() => await collection.RemoveAsync(GetTrainingIdFilter()); + public async Task> GetAll() => await collection.GetAll(GetTrainingIdFilter()); //(await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + public async Task RemoveAll() => await collection.RemoveAll(GetTrainingIdFilter()); public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) { - Func, FilterDefinition>> createFilter = item => - Builders>.Filter.And( - GetTrainingIdFilter(), - Builders>.Filter.Eq(o => o.RowKey, getId(item.Document)?.ToString()) //nameof(MongoDocumentWrapper.RowKey), o => getId(o.Document)?.ToString() - ); - //var wrapped = items.Select(o => new MongoDocumentWrapper(o)).ToList(); - //foreach (var item in wrapped) - // item.RowKey = getId(wrapped); - var (added, upserted) = await collection.Upsert(items, createFilter); - //return (added.Select(o => o.Document), upserted.Select(o => o.Document)); + //Func, FilterDefinition>> createFilter = item => + // Builders>.Filter.And( + // GetTrainingIdFilter(), + // Builders>.Filter.Eq(o => o.RowKey, getId(item.Document)?.ToString()) //nameof(MongoDocumentWrapper.RowKey), o => getId(o.Document)?.ToString() + // ); + var (added, upserted) = await collection.Upsert(items, GetTrainingIdFilter()); //createFilter return (added, upserted); } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs new file mode 100644 index 0000000..35ca2e1 --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs @@ -0,0 +1,18 @@ +using MongoDB.Driver; +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoTrainingSummaryRepository : /* DbWrappedCollection,*/ ITrainingSummaryRepository + { + private DbWrappedCollection collection; + + public MongoTrainingSummaryRepository(IMongoDatabase db) //: base(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")) + { + collection = new DbWrappedCollection(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")); + } + + public async Task> GetAll() => (await collection.GetAll()).ToList(); + //Task> ITrainingSummaryRepository.GetAll() => base.GetAll(); + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs index a812688..cf92065 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs @@ -1,39 +1,53 @@ -//using MongoDB.Driver; -//using ProblemSource.Models; -//using ProblemSource.Models.Aggregates; -//using ProblemSource.Services.Storage; -//using ProblemSourceModule.Models.Aggregates; - -//namespace ProblemSourceModule.Services.Storage.MongoDb -//{ -// public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider -// { -// private readonly IMongoDatabase db; -// private readonly int trainingId; -// private readonly string Key = $"{nameof(MongoDocumentWrapper.RowKey)}"; -// //private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; - -// public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) -// { -// this.db = db; -// this.trainingId = trainingId; -// } - -// public IBatchRepository Phases -// => new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, trainingId); //Key, - -// public IBatchRepository TrainingDays -// => new MongoTrainingBatchRepository(db, item => item.TrainingDay, trainingId); // "AccountId", - -// public IBatchRepository PhaseStatistics -// => new MongoTrainingBatchRepository(db, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); // "account_id", - -// public IBatchRepository TrainingSummaries -// => new MongoTrainingBatchRepository(db, item => "x", trainingId); // "AccountId", - -// public IBatchRepository UserStates -// => new MongoTrainingBatchRepository(db, item => "x", trainingId); // Key, - -// public Task RemoveAll() => throw new NotImplementedException(); -// } -//} +using MongoDB.Driver; +using ProblemSource.Models; +using ProblemSource.Models.Aggregates; +using ProblemSource.Services.Storage; +using ProblemSourceModule.Models.Aggregates; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoUserGeneratedDataRepositoryProviderFactory : IUserGeneratedDataRepositoryProviderFactory + { + private readonly IMongoDatabase db; + + public MongoUserGeneratedDataRepositoryProviderFactory(IMongoDatabase db) + { + this.db = db; + } + public IUserGeneratedDataRepositoryProvider Create(int userId) + { + return new MongoUserGeneratedDataRepositoryProvider(db, userId); + } + } + + public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataRepositoryProvider + { + private readonly IMongoDatabase db; + private readonly int trainingId; + //private readonly string Key = $"{nameof(MongoDocumentWrapper.RowKey)}"; + //private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; + + public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) + { + this.db = db; + this.trainingId = trainingId; + } + + public IBatchRepository Phases + => new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, trainingId); //Key, + + public IBatchRepository TrainingDays + => new MongoTrainingBatchRepository(db, item => item.TrainingDay, trainingId); // "AccountId", + + public IBatchRepository PhaseStatistics + => new MongoTrainingBatchRepository(db, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); // "account_id", + + public IBatchRepository TrainingSummaries + => new MongoTrainingBatchRepository(db, item => "x", trainingId); // "AccountId", + + public IBatchRepository UserStates + => new MongoTrainingBatchRepository(db, item => "x", trainingId); // Key, + + public Task RemoveAll() => throw new NotImplementedException(); + } +} diff --git a/Tools/Program.cs b/Tools/Program.cs index 5704e2f..c14a91d 100644 --- a/Tools/Program.cs +++ b/Tools/Program.cs @@ -19,8 +19,6 @@ var config = CreateConfig(); -//var azureTableSection = config.GetRequiredSection("AppSettings:AzureTable"); -//var tableConfig = TypedConfiguration.Bind(azureTableSection); var serviceProvider = InititalizeServices(config); Console.WriteLine("Run tooling?"); @@ -173,6 +171,7 @@ //var emails = BatchMail.ReadEmailFile(Path.Combine(path, "TeacherEmailsWithRejections.txt")); var emails = @" +jonas.beckeman+123@gmail.com ".Split('\n').SelectMany(o => o.Split(';')).Select(o => o.Trim().ToLower()).Where(o => o.Any()); var creator = serviceProvider.CreateInstance(); From a2caec628ebfc8a66f0efe60f95853527cf31993 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 14 Dec 2025 16:12:48 +0100 Subject: [PATCH 06/38] cleanup --- .../Services/Storage/MongoDb/DbCollection.cs | 11 ++--------- .../Storage/MongoDb/DbWrappedCollection.cs | 19 ------------------- .../Services/Storage/MongoDb/MongoTools.cs | 7 ------- .../MongoDb/MongoTrainingPlanRepository.cs | 1 - .../MongoDb/MongoTrainingSummaryRepository.cs | 1 - ...ongoUserGeneratedDataRepositoryProvider.cs | 2 -- 6 files changed, 2 insertions(+), 39 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs index 8876cf5..6cbb111 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs @@ -24,12 +24,10 @@ public async Task InsertGetId(TDocument item) } public IMongoCollection GetCollection() => collection; - public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); + //public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); - // public static FilterDefinition GetIdFilter(TId_ id, string idField) => Builders.Filter.Eq(idField, id); - //public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); public static FilterDefinition GetIdFilter(TId id, string idField) => Builders.Filter.Eq(idField, id); public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); @@ -44,12 +42,7 @@ public async Task Remove(TDocument item) } public async Task Update(TDocument item) => await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item); - public async Task Upsert(TDocument item) - { - await Upsert([item]); - //var filter = GetFilter([item]); - //await collection.FindOneAndReplaceAsync(GetIdFilter(getId(item)), item, new FindOneAndReplaceOptions { IsUpsert = true }); - } + public Task Upsert(TDocument item) => Upsert([item]); public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) { diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs index ce2cb84..debcfe1 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -25,26 +25,7 @@ public DbWrappedCollection(IMongoDatabase db, Func getId, Func CreateWrapped(TDocument item) => createWrapped(item); // new MongoDocumentWrapper(item, o => getId(o)?.ToString() ?? ""); public Task Remove(TDocument item) => collection.Remove(CreateWrapped(item)); - //public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) - //{ - // var result = await collection.RemoveAsync(filter, cancellationToken); - // return (int)result.DeletedCount; - //} - //public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) - //{ - // return (await collection.ListAsync(filter)).Select(o => o.Document).ToList(); - // //return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); - //} - //public string GetIdFilter(MongoDocumentWrapper item) - //{ - // return "AAA"; - //} - - //public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, Func> createFilter) - //{ - // return ([], []); - //} public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, FilterDefinition>? globalFilter = null) //, Func, FilterDefinition>> createFilter) { var result = await collection.Upsert(items.Select(CreateWrapped), globalFilter); //createFilter diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs index 8491498..d652ee7 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs @@ -4,13 +4,6 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { public class MongoTools { - // public static async Task Initialize() - // { - // const string uri = "mongodb://localhost:27017/"; - // var client = new MongoClient(uri); - // var db = client.GetDatabase("training"); - //} - public static IMongoCollection GetCollection(IMongoDatabase db) => db.GetCollection(GetCollectionName(typeof(T))); public static async Task DropCollection(IMongoDatabase db) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index 3263e62..7dba061 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -14,7 +14,6 @@ public class MongoTrainingRepository : DbWrappedCollection, ITrai { private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - //public MongoTrainingRepository(IMongoDatabase db) : base(db, $"{nameof(MongoDocumentWrapper.Document)}.{nameof(Training.Id)}", u => u.Document.Id) public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id, item => new MongoDocumentWrapper(item, o => o.Id.ToString())) { } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs index 35ca2e1..1a7af8d 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs @@ -13,6 +13,5 @@ public class MongoTrainingSummaryRepository : /* DbWrappedCollection> GetAll() => (await collection.GetAll()).ToList(); - //Task> ITrainingSummaryRepository.GetAll() => base.GetAll(); } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs index cf92065..1ca6988 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs @@ -24,8 +24,6 @@ public class MongoUserGeneratedDataRepositoryProvider : IUserGeneratedDataReposi { private readonly IMongoDatabase db; private readonly int trainingId; - //private readonly string Key = $"{nameof(MongoDocumentWrapper.RowKey)}"; - //private readonly string Key = $"{nameof(MongoDocumentWrapper)}.{nameof(MongoDocumentWrapper.RowKey)}"; public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingId) { From 30fef0d82489b9132f74459d0ab1f325aaca6654 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 14 Dec 2025 20:43:07 +0100 Subject: [PATCH 07/38] cleanup --- .../Services/Storage/MongoDb/DbCollection.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs index 6cbb111..3023cc3 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs @@ -24,7 +24,6 @@ public async Task InsertGetId(TDocument item) } public IMongoCollection GetCollection() => collection; - //public FilterDefinition GetIdFilter(TDocument id) => GetIdFilter(getId(id), idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(TId id) => GetIdFilter(id, idField); // Builders.Filter.Eq(idField, id); public FilterDefinition GetIdFilter(IEnumerable ids) => GetIdFilter(ids, idField); // Builders.Filter.AnyIn(idField, ids); @@ -67,9 +66,6 @@ private FilterDefinition GetFilter(IEnumerable items, Filt { var itemsWithId = items.Select(o => new { Id = getId(o), Item = o }).ToList(); - //var projection = new FindExpressionProjectionDefinition(p => new X { Id = p.Id, SubId = getId(p) }); - //ProjectionDefinition projection = $$"""{ "Id": "Id", "SubId": "{{idField}}" }"""; - //var tmpAll = await collection.Find(o => true).ToListAsync(); var projection = Builders.Projection.Include(idField); //Include("Id"). var tmpX = (await collection.Find(GetFilter(items, globalFilter)).Project(projection).ToListAsync()) .Select(o => new { Id = o["_id"].AsObjectId, SubId = o[idField]?.ToString() }) @@ -77,12 +73,9 @@ private FilterDefinition GetFilter(IEnumerable items, Filt .Where(o => o.SubId != null).ToList(); if (tmpX == null) throw new Exception("A"); - //var bsonIdById = tmpX.ToDictionary(o => o.SubId!, o => o.Id); var toInsert = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == false).ToList(); var toReplace = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == true).ToList(); - //var toInsert = itemsWithId.Select(o => bsonIdById.GetValueOrDefault(o.Id!.ToString())).ToList(); - //var toReplace = itemsWithId.Where(o => tmpX.Any(p => p.SubId?.Equals(o.Id) == true) == true).ToList(); var models = toInsert.Select(o => (WriteModel)new InsertOneModel(o.Item)).ToList(); From 8ece083584a2e597a225155aae400cda3d05a40c Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 24 Jan 2026 10:22:48 +0100 Subject: [PATCH 08/38] migration tooling --- .../ProblemSourceModule.Tests/MongoDbTests.cs | 2 +- .../Models/UserFullState.cs | 7 +- .../ProblemSourceModule.cs | 25 ++-- ...{DbCollection.cs => DbCollectionWithId.cs} | 68 ++++++++- .../Storage/MongoDb/DbWrappedCollection.cs | 64 +++++++-- .../Services/Storage/MongoDb/MongoTools.cs | 8 +- .../MongoTrainingAssociatedBatchRepository.cs | 67 +++++++++ .../MongoDb/MongoTrainingBatchRepository.cs | 49 ------- .../MongoDb/MongoTrainingSummaryRepository.cs | 10 +- ...ongoUserGeneratedDataRepositoryProvider.cs | 10 +- Tools/MigrateToMongoDb.cs | 135 ++++++++++++++++++ Tools/Program.cs | 20 ++- 12 files changed, 376 insertions(+), 89 deletions(-) rename ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/{DbCollection.cs => DbCollectionWithId.cs} (66%) create mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs delete mode 100644 ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs create mode 100644 Tools/MigrateToMongoDb.cs diff --git a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs index 827cacf..955ddca 100644 --- a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs +++ b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs @@ -52,7 +52,7 @@ public async Task X() var ugdr = new MongoUserGeneratedDataRepositoryProvider(db, id); var phasesRepo = ugdr.Phases; // new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, id); - var mongoTyped = (MongoTrainingBatchRepository)phasesRepo; + var mongoTyped = (MongoTrainingAssociatedBatchRepository)phasesRepo; var collectionName = mongoTyped.GetCollection().CollectionNamespace.CollectionName; await db.DropCollectionAsync(collectionName); //collections = await (await db.ListCollectionsAsync()).ToListAsync(); diff --git a/ProblemSource/ProblemSourceModule/Models/UserFullState.cs b/ProblemSource/ProblemSourceModule/Models/UserFullState.cs index aa92656..bdbd93d 100644 --- a/ProblemSource/ProblemSourceModule/Models/UserFullState.cs +++ b/ProblemSource/ProblemSourceModule/Models/UserFullState.cs @@ -1,5 +1,7 @@ -using Newtonsoft.Json; +using MongoDB.Bson.Serialization.Attributes; +using Newtonsoft.Json; using ProblemSourceModule.Models; +using ProblemSourceModule.Services.Storage.MongoDb; using System.Text.Json; namespace ProblemSource.Models @@ -252,7 +254,8 @@ public class ExerciseStats public long lastTimeStamp { get; set; } = 0; public Dictionary triggerData { get; set; } = new(); public List gameRuns { get; set; } = new List(); - public object? metaphorData { get; set; } + [BsonSerializer(typeof(XObjectCustomSerializer))] + public object? metaphorData { get; set; } public TrainingPlanSettings trainingPlanSettings { get; set; } = new TrainingPlanSettings(); public Dictionary gameCustomData { get; set; } = new Dictionary(); diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 7ac903d..33f40b5 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -4,6 +4,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using MongoDB.Bson.Serialization; using MongoDB.Driver; using PluginModuleBase; using ProblemSource.Services; @@ -60,7 +61,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); - var storageIsMongo = true; + var storageIsMongo = false; if (storageIsMongo) ConfigureForMongoDb(services); else @@ -91,9 +92,11 @@ public void ConfigureForMongoDb(IServiceCollection services) var connectionString = "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; var database = "_Training"; var client = new MongoClient(connectionString); - services.AddSingleton(sp => client.GetDatabase(database)); + services.AddSingleton(sp => client.GetDatabase(database)); - services.AddSingleton(); + //BsonSerializer.RegisterSerializer(new JObjectCustomSerializer()); + + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -109,17 +112,23 @@ public void ConfigureUsernameHashing(IServiceCollection services) } public void Configure(IApplicationBuilder app) + => Configure(app, true); + + public void Configure(IApplicationBuilder app, bool initAzureStorage) { var serviceProvider = app.ApplicationServices; serviceProvider.GetService()? .Register("problemsource", serviceProvider.GetRequiredService()); - // Initializing TableClientFactory on startup, in order to get an early error: - var tableClientFactory = serviceProvider.GetService() as TypedTableClientFactory; - tableClientFactory?.Init().Wait(); + if (initAzureStorage) + { + // Initializing TableClientFactory on startup, in order to get an early error: + var tableClientFactory = serviceProvider.GetService() as TypedTableClientFactory; + tableClientFactory?.Init().Wait(); - var queueEventDispatcher = serviceProvider.GetService() as AzureQueueEventDispatcher; - queueEventDispatcher?.Init().Wait(); + var queueEventDispatcher = serviceProvider.GetService() as AzureQueueEventDispatcher; + queueEventDispatcher?.Init().Wait(); + } } } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs similarity index 66% rename from ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs rename to ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs index 3023cc3..31b4f0f 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs @@ -1,16 +1,17 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; using MongoDB.Driver; namespace ProblemSourceModule.Services.Storage.MongoDb { - public class DbCollection where TDocument : DocumentBase + public class DbCollectionWithId where TDocument : DocumentBase { protected IMongoCollection collection; private readonly string idField; private readonly Func getId; - public DbCollection(IMongoDatabase db, string idField, Func getId) + public DbCollectionWithId(IMongoDatabase db, string idField, Func getId) { collection = db.GetCollection(MongoTools.GetCollectionName()); this.idField = idField; @@ -45,7 +46,8 @@ public async Task Remove(TDocument item) public async Task> ListAsync(FilterDefinition filter, CancellationToken cancellationToken = default) { - return await (await collection.FindAsync(filter, null, cancellationToken)).ToListAsync(cancellationToken); + var cursor = await collection.FindAsync(filter, null, cancellationToken); + return await cursor.ToListAsync(cancellationToken); } public async Task RemoveAsync(FilterDefinition filter, CancellationToken cancellationToken = default) { @@ -88,6 +90,8 @@ private FilterDefinition GetFilter(IEnumerable items, Filt return new ReplaceOneModel(Builders.Filter.Eq(p => p.Id, found.Id), o.Item); //createFilter(o.Item) })); + if (models.Any() == false) + return ([], []); var results = await collection.BulkWriteAsync(models, new BulkWriteOptions { }); // hm, we can't tell which were inserted and which were replaced?! return (toInsert.Select(o => o.Item), toReplace.Select(o => o.Item)); @@ -105,4 +109,62 @@ public async Task CountDocumentsAsync(FilterDefinition? filter return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); } } + + + public class XObjectCustomSerializer : SerializerBase + { + public override object? Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + //var name = context.Reader.ReadName(MongoDB.Bson.IO.Utf8NameDecoder.Instance); + + if (context.Reader.State == MongoDB.Bson.IO.BsonReaderState.Value) + { + //context.Reader.ReadNull(); + var rr = context.Reader.GetCurrentBsonType(); + if (rr == BsonType.Null) + { + context.Reader.ReadNull(); + return null; + } + } + var ntype = args.NominalType; + if (context.Reader.CurrentBsonType == BsonType.Null) + { + context.Reader.ReadNull(); + return null; + } + var json = context.Reader.ReadString(); + var value = System.Text.Json.JsonSerializer.Deserialize(json); + return value; + } + + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object? value) + { + var serializerOptions = new System.Text.Json.JsonSerializerOptions { WriteIndented = true }; + if (value == null) + { + context.Writer.WriteNull(); + return; + } + + var json = System.Text.Json.JsonSerializer.Serialize(value, serializerOptions); + context.Writer.WriteString(json); + } + } + public class JObjectCustomSerializer : SerializerBase + { + public override Newtonsoft.Json.Linq.JObject Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + var json = context.Reader.ReadString(); + var value = System.Text.Json.JsonSerializer.Deserialize(json); + return value!; + } + + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, Newtonsoft.Json.Linq.JObject value) + { + var serializerOptions = new System.Text.Json.JsonSerializerOptions { WriteIndented = true }; + var json = System.Text.Json.JsonSerializer.Serialize(value, serializerOptions); + context.Writer.WriteString(json); + } + } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs index debcfe1..e416fe9 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -2,27 +2,66 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { - public class DbWrappedCollection - { - protected DbCollection, TId> collection; - private readonly Func getId; - private readonly Func> createWrapped; + //public class DbWrappedCollectionOfT + // where T : DbWrappedCollection + // where TDocument : MongoDocumentWrapper + + public class DbTrainingAssociatedWrappedCollection + { + protected DbCollectionWithId, TId> collection; + private readonly Func getId; + private readonly Func> createWrapped; + + public DbTrainingAssociatedWrappedCollection(IMongoDatabase db, Func getId, Func> createWrapped) + { + collection = new DbCollectionWithId, TId>(db, nameof(MongoTrainingAssociatedDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); + this.getId = getId; + this.createWrapped = createWrapped; + } + + public IMongoCollection> GetCollection() => collection.GetCollection(); + + private MongoTrainingAssociatedDocumentWrapper CreateWrapped(TDocument item) => createWrapped(item); // new MongoDocumentWrapper(item, o => getId(o)?.ToString() ?? ""); + + public Task Remove(TDocument item) => collection.Remove(CreateWrapped(item)); + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items, FilterDefinition>? globalFilter = null) //, Func, FilterDefinition>> createFilter) + { + var result = await collection.Upsert(items.Select(CreateWrapped), globalFilter); //createFilter + return (result.Added.Select(o => o.Document), result.Updated.Select(o => o.Document)); + } - //private readonly Func, TId> getId; + public Task Update(TDocument item) => Upsert([item]); // TODO: throw if not already existing collection.Update(CreateWrapped(item)); + public Task Upsert(TDocument item) => Upsert([item]); // collection.Upsert(CreateWrapped(item)); + public async Task Get(TId id) + { + var found = await collection.Get(id); + if (found != null) + return found.Document; + return default; + } + public async Task> Get(IEnumerable ids) => (await collection.Get(ids)).Select(o => o.Document).ToList(); + public async Task> GetAll() => (await collection.GetAll()).Select(o => o.Document).ToList(); + public async Task> GetAll(FilterDefinition> filter) + => (await collection.ListAsync(filter)).Select(o => o.Document).ToList(); + public async Task RemoveAll(FilterDefinition> filter) + => await collection.RemoveAsync(filter); + } + public class DbWrappedCollection + { + protected DbCollectionWithId, TId> collection; + private readonly Func> createWrapped; - //public DbWrappedCollection(IMongoDatabase db, string idField, Func, TId> getId) - //public DbWrappedCollection(IMongoDatabase db, string idField, Func getId) public DbWrappedCollection(IMongoDatabase db, Func getId, Func> createWrapped) { - collection = new DbCollection, TId>(db, nameof(MongoDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); - this.getId = getId; + collection = new DbCollectionWithId, TId>(db, nameof(MongoDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); this.createWrapped = createWrapped; } public IMongoCollection> GetCollection() => collection.GetCollection(); - private MongoDocumentWrapper CreateWrapped(TDocument item) => createWrapped(item); // new MongoDocumentWrapper(item, o => getId(o)?.ToString() ?? ""); + private MongoDocumentWrapper CreateWrapped(TDocument item) => createWrapped(item); public Task Remove(TDocument item) => collection.Remove(CreateWrapped(item)); @@ -33,7 +72,7 @@ public DbWrappedCollection(IMongoDatabase db, Func getId, Func Upsert([item]); // TODO: throw if not already existing collection.Update(CreateWrapped(item)); - public Task Upsert(TDocument item) => Upsert([item]); // collection.Upsert(CreateWrapped(item)); + public Task Upsert(TDocument item) => Upsert([item]); public async Task Get(TId id) { var found = await collection.Get(id); @@ -48,6 +87,5 @@ public async Task> GetAll(FilterDefinition (await collection.ListAsync(filter)).Select(o => o.Document).ToList(); public async Task RemoveAll(FilterDefinition> filter) => await collection.RemoveAsync(filter); - } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs index d652ee7..2b4ceb2 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs @@ -17,11 +17,11 @@ public static string GetCollectionName(Type type) if (type.GenericTypeArguments.Any()) { if (type.GetGenericTypeDefinition() == typeof(MongoDocumentWrapper<>)) - { return type.GenericTypeArguments[0].Name; - } - } - return type.Name; + if (type.GetGenericTypeDefinition() == typeof(MongoTrainingAssociatedDocumentWrapper<>)) + return type.GenericTypeArguments[0].Name; + } + return type.Name; } } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs new file mode 100644 index 0000000..e2c126b --- /dev/null +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs @@ -0,0 +1,67 @@ +using MongoDB.Driver; +using ProblemSource.Services.Storage; + +namespace ProblemSourceModule.Services.Storage.MongoDb +{ + public class MongoTrainingAssociatedDocumentWrapper : MongoDocumentWrapper // where TDocument : DocumentBase + { + // An additional property TrainingId for easy joining + public MongoTrainingAssociatedDocumentWrapper() { } + + [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] + public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Func? getId = null) : base(doc, getId) + { + TrainingId = trainingId; + } + public int TrainingId { get; set; } + } + + public class MongoTrainingAssociatedBatchRepository : IBatchRepository + { + //private readonly Func getId; + private readonly int trainingId; + private readonly DbTrainingAssociatedWrappedCollection collection; + //private readonly DbWrappedCollection collection; + + public IMongoCollection> GetCollection() => collection.GetCollection(); + + public MongoTrainingAssociatedBatchRepository(IMongoDatabase db, Func getId, int trainingId) + { + collection = new DbTrainingAssociatedWrappedCollection(db, getId, item => new MongoTrainingAssociatedDocumentWrapper(item, trainingId, o => getId(o)?.ToString() ?? "")); + //this.getId = getId; + this.trainingId = trainingId; + } + + private FilterDefinition> GetTrainingIdFilter() + => DbCollectionWithId, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); + //private FilterDefinition> GetTrainingIdFilter() + //{ + // return DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); + //} + + public async Task> GetAll() //=> await collection.GetAll(GetTrainingIdFilter()) + { + var filter = GetTrainingIdFilter(); + + var serializerRegistry = MongoDB.Bson.Serialization.BsonSerializer.SerializerRegistry; + var documentSerializer = serializerRegistry.GetSerializer>(); + var tmp = filter.Render(new RenderArgs>(documentSerializer, serializerRegistry)); + return await collection.GetAll(filter); + } + //(await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); + + public async Task RemoveAll() + => await collection.RemoveAll(GetTrainingIdFilter()); + + public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) + { + //Func, FilterDefinition>> createFilter = item => + // Builders>.Filter.And( + // GetTrainingIdFilter(), + // Builders>.Filter.Eq(o => o.RowKey, getId(item.Document)?.ToString()) //nameof(MongoDocumentWrapper.RowKey), o => getId(o.Document)?.ToString() + // ); + var (added, upserted) = await collection.Upsert(items, GetTrainingIdFilter()); //createFilter + return (added, upserted); + } + } +} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs deleted file mode 100644 index c1876f3..0000000 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingBatchRepository.cs +++ /dev/null @@ -1,49 +0,0 @@ -using MongoDB.Driver; -using ProblemSource.Services.Storage; - -namespace ProblemSourceModule.Services.Storage.MongoDb -{ - public class MongoTrainingAssociatedDocumentWrapper : MongoDocumentWrapper // where TDocument : DocumentBase - { - public MongoTrainingAssociatedDocumentWrapper() { } - - [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] - public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Func? getId = null) : base(doc, getId) - { - TrainingId = trainingId; - } - public int TrainingId { get; set; } - } - - public class MongoTrainingBatchRepository : IBatchRepository - { - private readonly Func getId; - private readonly int trainingId; - private readonly DbWrappedCollection collection; - - public IMongoCollection> GetCollection() => collection.GetCollection(); - - public MongoTrainingBatchRepository(IMongoDatabase db, Func getId, int trainingId) - { - collection = new DbWrappedCollection(db, getId, item => new MongoTrainingAssociatedDocumentWrapper(item, trainingId, o => getId(o)?.ToString() ?? "")); - this.getId = getId; - this.trainingId = trainingId; - } - - private FilterDefinition> GetTrainingIdFilter() - => DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); - public async Task> GetAll() => await collection.GetAll(GetTrainingIdFilter()); //(await collection.ListAsync(GetTrainingIdFilter())).Select(o => o.Document).ToList(); - public async Task RemoveAll() => await collection.RemoveAll(GetTrainingIdFilter()); - - public async Task<(IEnumerable Added, IEnumerable Updated)> Upsert(IEnumerable items) - { - //Func, FilterDefinition>> createFilter = item => - // Builders>.Filter.And( - // GetTrainingIdFilter(), - // Builders>.Filter.Eq(o => o.RowKey, getId(item.Document)?.ToString()) //nameof(MongoDocumentWrapper.RowKey), o => getId(o.Document)?.ToString() - // ); - var (added, upserted) = await collection.Upsert(items, GetTrainingIdFilter()); //createFilter - return (added, upserted); - } - } -} diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs index 1a7af8d..a2673c1 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs @@ -3,11 +3,17 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { - public class MongoTrainingSummaryRepository : /* DbWrappedCollection,*/ ITrainingSummaryRepository + public interface IMongoRepoWithCollection + { + DbWrappedCollection Collection { get; } + } + public class MongoTrainingSummaryRepository : /* DbWrappedCollection,*/ ITrainingSummaryRepository, IMongoRepoWithCollection { private DbWrappedCollection collection; - public MongoTrainingSummaryRepository(IMongoDatabase db) //: base(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")) + public DbWrappedCollection Collection => collection; + + public MongoTrainingSummaryRepository(IMongoDatabase db) //: base(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")) { collection = new DbWrappedCollection(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")); } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs index 1ca6988..26e139a 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoUserGeneratedDataRepositoryProvider.cs @@ -32,19 +32,19 @@ public MongoUserGeneratedDataRepositoryProvider(IMongoDatabase db, int trainingI } public IBatchRepository Phases - => new MongoTrainingBatchRepository(db, Phase.UniqueIdWithinUser, trainingId); //Key, + => new MongoTrainingAssociatedBatchRepository(db, Phase.UniqueIdWithinUser, trainingId); //Key, public IBatchRepository TrainingDays - => new MongoTrainingBatchRepository(db, item => item.TrainingDay, trainingId); // "AccountId", + => new MongoTrainingAssociatedBatchRepository(db, item => item.TrainingDay, trainingId); // "AccountId", public IBatchRepository PhaseStatistics - => new MongoTrainingBatchRepository(db, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); // "account_id", + => new MongoTrainingAssociatedBatchRepository(db, ProblemSource.Models.Aggregates.PhaseStatistics.UniqueIdWithinUser, trainingId); // "account_id", public IBatchRepository TrainingSummaries - => new MongoTrainingBatchRepository(db, item => "x", trainingId); // "AccountId", + => new MongoTrainingAssociatedBatchRepository(db, item => "x", trainingId); // "AccountId", public IBatchRepository UserStates - => new MongoTrainingBatchRepository(db, item => "x", trainingId); // Key, + => new MongoTrainingAssociatedBatchRepository(db, item => "x", trainingId); // Key, public Task RemoveAll() => throw new NotImplementedException(); } diff --git a/Tools/MigrateToMongoDb.cs b/Tools/MigrateToMongoDb.cs new file mode 100644 index 0000000..dc5644f --- /dev/null +++ b/Tools/MigrateToMongoDb.cs @@ -0,0 +1,135 @@ +using MongoDB.Driver; +using ProblemSource.Models; +using ProblemSource.Models.Aggregates; +using ProblemSource.Services.Storage; +using ProblemSource.Services.Storage.AzureTables; +using ProblemSourceModule.Models; +using ProblemSourceModule.Services.Storage; +using ProblemSourceModule.Services.Storage.MongoDb; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tools +{ + internal class MigrateToMongoDb + { + private readonly ITypedTableClientFactory tableClientFactory; + private readonly ITrainingRepository trainingRepository; + private readonly IMongoDatabase db; + + //private readonly IUserGeneratedDataRepositoryProvider userGeneratedDataRepositoryProvider; + + // , IUserGeneratedDataRepositoryProvider userGeneratedDataRepositoryProvider + public MigrateToMongoDb(ITypedTableClientFactory tableClientFactory, ITrainingRepository trainingRepository, IMongoDatabase db) + { + this.tableClientFactory = tableClientFactory; + this.trainingRepository = trainingRepository; + this.db = db; + //this.userGeneratedDataRepositoryProvider = userGeneratedDataRepositoryProvider; + } + + private async Task Test() + { + var mdbRepoFactory = new MongoUserGeneratedDataRepositoryProviderFactory(db); + + var mdbRepos = mdbRepoFactory.Create(1); + var jobj = new Newtonsoft.Json.Linq.JObject(); + var state = new UserGeneratedState + { + exercise_stats = new ExerciseStats + { + trainingPlanSettings = new TrainingPlanSettings + { + changes = [new TrainingPlanChange { change = new Newtonsoft.Json.Linq.JObject(), timestamp = 1, type = "t" }] + }, + gameCustomData = new Dictionary(), + gameRuns = [], + planetInfos = null, + //metaphorData = new Newtonsoft.Json.Linq.JObject(), + }, + //syncInfo = new Newtonsoft.Json.Linq.JObject(), + user_data = null //new Newtonsoft.Json.Linq.JObject() //jobj + }; + await mdbRepos.UserStates.Upsert([state]); + + var tmp = await mdbRepos.UserStates.GetAll(); + } + + public async Task MigrateTest() + { + var training = new Training { Id = 7, Username = "A", TrainingPlanName = "aa", Settings = new TrainingSettings() }; + //var trainings = new[] { }; + var mdbRepoFactory = new MongoUserGeneratedDataRepositoryProviderFactory(db); + var mdbTrainings = new MongoTrainingRepository(db); + + await mdbTrainings.Upsert(training); //Add + var mdbRepos = mdbRepoFactory.Create(training.Id); + + var summary = (await mdbRepos.TrainingSummaries.GetAll()).SingleOrDefault(); + } + + public async Task Migrate() + { + var mdbRepoFactory = new MongoUserGeneratedDataRepositoryProviderFactory(db); + + //var mdbTrainings = new DbWrappedCollection(db, d => d.Id, d => new MongoDocumentWrapper(d)); + var mdbTrainings = new MongoTrainingRepository(db); + Console.WriteLine("Reading trainings..."); + var allMongoTrainings = (await mdbTrainings.GetCollection().Find(o => true).Project(o => o.RowKey).ToListAsync()).Select(int.Parse).ToList(); + + //var trainings = (await trainingRepository.GetAll()).ToList(); + var trainings = await trainingRepository.GetByIds([3, 7]); + Console.WriteLine($"{trainings.Count()} trainings found, {allMongoTrainings.Count} in MongoDB"); + + //var aaq = new MongoTrainingBatchRepository(db, d => d.Id, trainingId); + //collection = new DbWrappedCollection(db, getId, item => new MongoTrainingAssociatedDocumentWrapper(item, trainingId, o => getId(o)?.ToString() ?? "")); + + foreach (var (index, training) in trainings.Index()) + { + var repos = new AzureTableUserGeneratedDataRepositoryProvider(tableClientFactory, training.Id); + + var trainingSummaries = await repos.TrainingSummaries.GetAll(); + var trainedDays = trainingSummaries.SingleOrDefault()?.TrainedDays; + var skip = trainedDays == null || trainedDays <= 2; + Console.WriteLine($"{index}/{trainings.Count()} Id={training.Id} TrainedDays={trainingSummaries.SingleOrDefault()?.TrainedDays} {(skip ? "SKIP!" : "")}"); + if (skip) + continue; + if (allMongoTrainings.Contains(training.Id)) + { + Console.WriteLine("Training already in mongo - skip"); + continue; + } + + await mdbTrainings.Upsert(training); //Add + var mdbRepos = mdbRepoFactory.Create(training.Id); + + var summary = (await mdbRepos.TrainingSummaries.GetAll()).SingleOrDefault(); + if (summary != null) + { + Console.WriteLine("Summary already in mongo - skip"); + continue; + } + + await mdbRepos.UserStates.Upsert(await repos.UserStates.GetAll()); + await mdbRepos.TrainingSummaries.Upsert(trainingSummaries); + await mdbRepos.TrainingDays.Upsert(await repos.TrainingDays.GetAll()); + await mdbRepos.PhaseStatistics.Upsert(await repos.PhaseStatistics.GetAll()); + await mdbRepos.Phases.Upsert(await repos.Phases.GetAll()); + } + } + + // private async Task X(IBatchRepository repo, IMongoDatabase db) + // { + // var rows = await repo.GetAll(); + // var docs = rows.Select(o => new MongoDocumentWrapper(o)).ToList(); + // new DbWrappedCollection collection + // //new DbCollection(db, "", () => "asd"); + + // var collection = MongoTools.GetCollection(db); + // await collection.InsertManyAsync(docs); + //} + } +} diff --git a/Tools/Program.cs b/Tools/Program.cs index c14a91d..15ec230 100644 --- a/Tools/Program.cs +++ b/Tools/Program.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using MongoDB.Driver; using ProblemSource; using ProblemSource.Services; using ProblemSource.Services.Storage; @@ -41,6 +42,20 @@ var path = Path.Join(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "WebProcessor_Files"); +IMongoDatabase? mongoDb; +{ + var connectionString = "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; + var database = "_Training"; + var client = new MongoClient(connectionString); + mongoDb = client.GetDatabase(database); + + MongoDB.Bson.Serialization.BsonSerializer.RegisterSerializer(new ProblemSourceModule.Services.Storage.MongoDb.XObjectCustomSerializer()); + //MongoDB.Bson.Serialization.BsonSerializer.RegisterSerializer(new ProblemSourceModule.Services.Storage.MongoDb.JObjectCustomSerializer()); + + var migrator = new MigrateToMongoDb(serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService(), mongoDb); + await migrator.Migrate(); +} + //await TrainingMod.ModifyTimeSpent(42434, serviceProvider.GetRequiredService()); //await new FixAzureTableQuotedDateTime(serviceProvider.GetRequiredService().ConnectionString) // .Fix(new Dictionary> { @@ -240,8 +255,9 @@ IServiceProvider InititalizeServices(IConfigurationRoot config) var module = new ProblemSource.ProblemSourceModule(); module.ConfigureServices(services); var serviceProvider = services.BuildServiceProvider(); - module.Configure(new App(serviceProvider)); - return serviceProvider; + module.Configure(new App(serviceProvider), initAzureStorage: false); + + return serviceProvider; } class App : IApplicationBuilder From 8ec4d6775f2561bd2ebda3e87c919e54b1c1176f Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 24 Jan 2026 17:06:31 +0100 Subject: [PATCH 09/38] revert to actual migration --- Tools/MigrateToMongoDb.cs | 4 ++-- Tools/Program.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/MigrateToMongoDb.cs b/Tools/MigrateToMongoDb.cs index dc5644f..8cf6adb 100644 --- a/Tools/MigrateToMongoDb.cs +++ b/Tools/MigrateToMongoDb.cs @@ -80,8 +80,8 @@ public async Task Migrate() Console.WriteLine("Reading trainings..."); var allMongoTrainings = (await mdbTrainings.GetCollection().Find(o => true).Project(o => o.RowKey).ToListAsync()).Select(int.Parse).ToList(); - //var trainings = (await trainingRepository.GetAll()).ToList(); - var trainings = await trainingRepository.GetByIds([3, 7]); + var trainings = (await trainingRepository.GetAll()).ToList(); + //var trainings = await trainingRepository.GetByIds([3, 7]); Console.WriteLine($"{trainings.Count()} trainings found, {allMongoTrainings.Count} in MongoDB"); //var aaq = new MongoTrainingBatchRepository(db, d => d.Id, trainingId); diff --git a/Tools/Program.cs b/Tools/Program.cs index 15ec230..50bbf3d 100644 --- a/Tools/Program.cs +++ b/Tools/Program.cs @@ -255,7 +255,7 @@ IServiceProvider InititalizeServices(IConfigurationRoot config) var module = new ProblemSource.ProblemSourceModule(); module.ConfigureServices(services); var serviceProvider = services.BuildServiceProvider(); - module.Configure(new App(serviceProvider), initAzureStorage: false); + module.Configure(new App(serviceProvider), initAzureStorage: true); return serviceProvider; } From 46ae24f2d4c7054fc6387413448934f36dd72fe3 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 25 Jan 2026 12:22:45 +0100 Subject: [PATCH 10/38] migrate users, config --- Common.Web/ServiceConfiguration.cs | 4 +-- Directory.Packages.props | 1 + PluginModuleBase/IPluginModule.cs | 5 ++-- PluginModuleBase/PluginModuleBase.csproj | 1 + .../ProblemSourceModule.cs | 26 +++++++++++++------ ProblemSource/TrainingApi/Startup.cs | 7 ++--- ProblemSource/TrainingApi/appsettings.json | 7 +++++ Tools/MigrateToMongoDb.cs | 14 ++++++++-- Tools/Program.cs | 7 +++-- 9 files changed, 53 insertions(+), 19 deletions(-) diff --git a/Common.Web/ServiceConfiguration.cs b/Common.Web/ServiceConfiguration.cs index 332e057..3b13c91 100644 --- a/Common.Web/ServiceConfiguration.cs +++ b/Common.Web/ServiceConfiguration.cs @@ -10,7 +10,7 @@ namespace Common.Web { public static class ServiceConfiguration { - public static void ConfigureProcessingPipelineServices(IServiceCollection services, IEnumerable pluginModules) + public static void ConfigureProcessingPipelineServices(IServiceCollection services, IConfiguration config, IEnumerable pluginModules) { services.AddSingleton(); //(sp => new TableClientFactory("vektor") services.AddSingleton(); @@ -18,7 +18,7 @@ public static void ConfigureProcessingPipelineServices(IServiceCollection servic services.AddSingleton(); foreach (var plugin in pluginModules) - plugin.ConfigureServices(services); + plugin.ConfigureServices(services, config); } public static void ConfigurePlugins(IApplicationBuilder app, IEnumerable pluginModules) diff --git a/Directory.Packages.props b/Directory.Packages.props index 01254fe..3839254 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -37,6 +37,7 @@ + diff --git a/PluginModuleBase/IPluginModule.cs b/PluginModuleBase/IPluginModule.cs index 50de965..92d0c01 100644 --- a/PluginModuleBase/IPluginModule.cs +++ b/PluginModuleBase/IPluginModule.cs @@ -1,11 +1,12 @@ using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; namespace PluginModuleBase { public interface IPluginModule { - void ConfigureServices(IServiceCollection services); - void Configure(IApplicationBuilder app); + void ConfigureServices(IServiceCollection services, IConfiguration config); + void Configure(IApplicationBuilder app); } } diff --git a/PluginModuleBase/PluginModuleBase.csproj b/PluginModuleBase/PluginModuleBase.csproj index 0c56997..f405d14 100644 --- a/PluginModuleBase/PluginModuleBase.csproj +++ b/PluginModuleBase/PluginModuleBase.csproj @@ -6,6 +6,7 @@ + diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 33f40b5..c729886 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -21,7 +21,7 @@ namespace ProblemSource { public class ProblemSourceModule : IPluginModule { - public void ConfigureServices(IServiceCollection services) + public void ConfigureServices(IServiceCollection services, IConfiguration config) { services.AddSingleton(); services.AddSingleton(); @@ -61,9 +61,16 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); - var storageIsMongo = false; + var storageIsMongo = false; + config ??= services.Select(o => o.ImplementationInstance).OfType().Single(); + { + if (config != null) + { + storageIsMongo = config["AppSettings:Storage:Type"] == "MongoDB"; + } + } if (storageIsMongo) - ConfigureForMongoDb(services); + ConfigureForMongoDb(services, config!); else ConfigureForAzureTables(services); @@ -87,14 +94,17 @@ public void ConfigureForAzureTables(IServiceCollection services, bool useCaching services.AddSingleton(); } - public void ConfigureForMongoDb(IServiceCollection services) + public void ConfigureForMongoDb(IServiceCollection services, IConfiguration config) { - var connectionString = "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; - var database = "_Training"; - var client = new MongoClient(connectionString); + var section = config.GetSection("AppSettings:Storage:MongoDB"); + + var connectionString = section["ConnectionString"]; // "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; + var database = section["Database"]; //"_Training"; + + var client = new MongoClient(connectionString); services.AddSingleton(sp => client.GetDatabase(database)); - //BsonSerializer.RegisterSerializer(new JObjectCustomSerializer()); + BsonSerializer.RegisterSerializer(new XObjectCustomSerializer()); services.AddSingleton(); services.AddSingleton(); diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index b35e9c3..d22cea9 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -163,7 +163,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) var referer = context.Request.GetTypedHeaders().Referer; // when from swagger and localhost var autologin = referer?.AbsolutePath.Contains("/swagger/") == true - || referer?.AbsoluteUri.StartsWith("http://localhost:") == true; + || referer?.AbsoluteUri.StartsWith("http://localhost:") == true + || referer?.AbsoluteUri.StartsWith("https://localhost") == true; if (!autologin) { @@ -216,8 +217,8 @@ private IPluginModule[] ConfigureProblemSource(IServiceCollection services, ICon ConfigureAuthentication(services, config, env); var plugins = new IPluginModule[] { new ProblemSource.ProblemSourceModule() }; - services.AddSingleton(); - ServiceConfiguration.ConfigureProcessingPipelineServices(services, plugins); + //services.AddSingleton(); + ServiceConfiguration.ConfigureProcessingPipelineServices(services, config, plugins); return plugins; } diff --git a/ProblemSource/TrainingApi/appsettings.json b/ProblemSource/TrainingApi/appsettings.json index 0b49faf..0cd5915 100644 --- a/ProblemSource/TrainingApi/appsettings.json +++ b/ProblemSource/TrainingApi/appsettings.json @@ -23,6 +23,13 @@ "AzureQueue": { "ConnectionString": "UseDevelopmentStorage=true" }, + "Storage": { + "Type": "MongoDB", + "MongoDB": { + "ConnectionString": "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500", + "Database": "_Training" + } + }, "RealTime": { "Enabled": true, "AzureQueueConfig": { diff --git a/Tools/MigrateToMongoDb.cs b/Tools/MigrateToMongoDb.cs index 8cf6adb..e684aab 100644 --- a/Tools/MigrateToMongoDb.cs +++ b/Tools/MigrateToMongoDb.cs @@ -5,6 +5,7 @@ using ProblemSource.Services.Storage.AzureTables; using ProblemSourceModule.Models; using ProblemSourceModule.Services.Storage; +using ProblemSourceModule.Services.Storage.AzureTables; using ProblemSourceModule.Services.Storage.MongoDb; using System; using System.Collections.Generic; @@ -18,15 +19,17 @@ internal class MigrateToMongoDb { private readonly ITypedTableClientFactory tableClientFactory; private readonly ITrainingRepository trainingRepository; + private readonly IUserRepository userRepository; private readonly IMongoDatabase db; //private readonly IUserGeneratedDataRepositoryProvider userGeneratedDataRepositoryProvider; // , IUserGeneratedDataRepositoryProvider userGeneratedDataRepositoryProvider - public MigrateToMongoDb(ITypedTableClientFactory tableClientFactory, ITrainingRepository trainingRepository, IMongoDatabase db) + public MigrateToMongoDb(ITypedTableClientFactory tableClientFactory, ITrainingRepository trainingRepository, IUserRepository userRepository, IMongoDatabase db) { this.tableClientFactory = tableClientFactory; this.trainingRepository = trainingRepository; + this.userRepository = userRepository; this.db = db; //this.userGeneratedDataRepositoryProvider = userGeneratedDataRepositoryProvider; } @@ -75,12 +78,19 @@ public async Task Migrate() { var mdbRepoFactory = new MongoUserGeneratedDataRepositoryProviderFactory(db); + Console.WriteLine("Reading users..."); + var users = await userRepository.GetAll(); + var mdbUsers = new MongoUserRepository(db); + await mdbUsers.Upsert(users); + //var mdbTrainings = new DbWrappedCollection(db, d => d.Id, d => new MongoDocumentWrapper(d)); var mdbTrainings = new MongoTrainingRepository(db); Console.WriteLine("Reading trainings..."); var allMongoTrainings = (await mdbTrainings.GetCollection().Find(o => true).Project(o => o.RowKey).ToListAsync()).Select(int.Parse).ToList(); - var trainings = (await trainingRepository.GetAll()).ToList(); + var trainings = (await trainingRepository.GetAll()) + .Where(o => o.Id > 32431) + .ToList(); //var trainings = await trainingRepository.GetByIds([3, 7]); Console.WriteLine($"{trainings.Count()} trainings found, {allMongoTrainings.Count} in MongoDB"); diff --git a/Tools/Program.cs b/Tools/Program.cs index 50bbf3d..8cafb01 100644 --- a/Tools/Program.cs +++ b/Tools/Program.cs @@ -52,7 +52,10 @@ MongoDB.Bson.Serialization.BsonSerializer.RegisterSerializer(new ProblemSourceModule.Services.Storage.MongoDb.XObjectCustomSerializer()); //MongoDB.Bson.Serialization.BsonSerializer.RegisterSerializer(new ProblemSourceModule.Services.Storage.MongoDb.JObjectCustomSerializer()); - var migrator = new MigrateToMongoDb(serviceProvider.GetRequiredService(), serviceProvider.GetRequiredService(), mongoDb); + var migrator = new MigrateToMongoDb(serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + serviceProvider.GetRequiredService(), + mongoDb); await migrator.Migrate(); } @@ -253,7 +256,7 @@ IServiceProvider InititalizeServices(IConfigurationRoot config) services.AddSingleton(typeof(ILogger<>), typeof(Logger<>)); var module = new ProblemSource.ProblemSourceModule(); - module.ConfigureServices(services); + module.ConfigureServices(services, config); var serviceProvider = services.BuildServiceProvider(); module.Configure(new App(serviceProvider), initAzureStorage: true); From 37c930886032b0479c9432b91a51cc78dd991916 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 25 Jan 2026 21:49:48 +0100 Subject: [PATCH 11/38] cleanup --- .../ProblemSourceModule/ProblemSourceModule.cs | 9 +++++++++ .../Services/Storage/MongoDb/DbCollectionWithId.cs | 14 +++++--------- .../Storage/MongoDb/MongoDocumentWrapper.cs | 1 + .../MongoTrainingAssociatedBatchRepository.cs | 4 ---- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index c729886..57d79c6 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -104,6 +104,15 @@ public void ConfigureForMongoDb(IServiceCollection services, IConfiguration conf var client = new MongoClient(connectionString); services.AddSingleton(sp => client.GetDatabase(database)); + // DocumentBase MongoDocumentWrapper + //BsonClassMap.RegisterClassMap(cm => + //{ + // cm.AutoMap(); + // //cm.MapIdProperty(c => c.Id) + // // .SetIdGenerator(MongoDB.Bson.Serialization.IdGenerators.StringObjectIdGenerator.Instance) + // // .SetSerializer(new MongoDB.Bson.Serialization.Serializers.StringSerializer(MongoDB.Bson.BsonType.ObjectId)); + //}); + BsonSerializer.RegisterSerializer(new XObjectCustomSerializer()); services.AddSingleton(); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs index 31b4f0f..b84a055 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs @@ -32,8 +32,11 @@ public async Task InsertGetId(TDocument item) public static FilterDefinition GetIdFilter(IEnumerable ids, string idField) => Builders.Filter.AnyIn(idField, ids); public async Task Get(TId id) => await (await collection.FindAsync(GetIdFilter(id))).FirstOrDefaultAsync(); - public async Task> Get(IEnumerable ids) => await (await collection.FindAsync(GetIdFilter(ids))).ToListAsync(); - + public async Task> Get(IEnumerable ids) //=> await (await collection.FindAsync(GetIdFilter(ids))).ToListAsync(); + { + var filter = GetIdFilter(ids); + return await (await collection.FindAsync(filter)).ToListAsync(); + } public async Task> GetAll() => await collection.Find(o => true).ToListAsync(); public async Task Remove(TDocument item) @@ -97,13 +100,6 @@ private FilterDefinition GetFilter(IEnumerable items, Filt return (toInsert.Select(o => o.Item), toReplace.Select(o => o.Item)); } - private class X - { - public ObjectId Id { get; set; } - //public object? SubId { get; set; } - public TId? SubId { get; set; } - } - public async Task CountDocumentsAsync(FilterDefinition? filter = null) { return await collection.CountDocumentsAsync(filter ?? Builders.Filter.Empty, filter == null ? new CountOptions { Hint = "_id_" } : null); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs index 00d0dcd..c9716ca 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs @@ -17,6 +17,7 @@ public MongoDocumentWrapper(TDocument doc, Func? getId = null RowKey = getId(doc); } + //[BsonS] public string RowKey { get; set; } = string.Empty; public required TDocument Document { get; set; } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs index e2c126b..f2a2223 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs @@ -34,10 +34,6 @@ public MongoTrainingAssociatedBatchRepository(IMongoDatabase db, Func> GetTrainingIdFilter() => DbCollectionWithId, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); - //private FilterDefinition> GetTrainingIdFilter() - //{ - // return DbCollection, int>.GetIdFilter(trainingId, $"{nameof(MongoTrainingAssociatedDocumentWrapper.TrainingId)}"); - //} public async Task> GetAll() //=> await collection.GetAll(GetTrainingIdFilter()) { From 6e3407eb0185869546fb3bb8b6aac0f92bd35763 Mon Sep 17 00:00:00 2001 From: JWMB Date: Mon, 26 Jan 2026 09:42:06 +0100 Subject: [PATCH 12/38] MongoTrainingRepository: TId -> string --- .../Storage/MongoDb/MongoTrainingPlanRepository.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index 7dba061..49d1f85 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -10,11 +10,11 @@ public class MongoUserRepository : DbWrappedCollection, IUserRepos public async Task Add(User item) => await Upsert(item); } - public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository + public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository { private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id, item => new MongoDocumentWrapper(item, o => o.Id.ToString())) + public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id.ToString(), item => new MongoDocumentWrapper(item, o => o.Id.ToString())) { } public Task Add(Training item) => AddGetId(item); @@ -32,6 +32,8 @@ public async Task AddGetId(Training item) return item.Id; } - public async Task> GetByIds(IEnumerable ids) => await Get(ids); + public Task Get(int id) => Get(id.ToString()); + + public async Task> GetByIds(IEnumerable ids) => await Get(ids.Select(o => o.ToString())); } } From 963ee999fc17af407f4dd5aa7e1585898ded45c1 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 31 Jan 2026 09:42:02 +0100 Subject: [PATCH 13/38] mongodb working --- .../ProblemSourceModule.cs | 44 +++++++++++++++---- .../Storage/MongoDb/DbCollectionWithId.cs | 14 +++++- .../Storage/MongoDb/DbWrappedCollection.cs | 6 +-- .../Storage/MongoDb/MongoDocumentWrapper.cs | 10 ++--- .../MongoTrainingAssociatedBatchRepository.cs | 4 +- .../MongoDb/MongoTrainingPlanRepository.cs | 10 ++--- .../MongoDb/MongoTrainingSummaryRepository.cs | 4 +- 7 files changed, 64 insertions(+), 28 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 57d79c6..b98ccdd 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -6,10 +6,14 @@ using Microsoft.Extensions.DependencyInjection; using MongoDB.Bson.Serialization; using MongoDB.Driver; +using MongoDB.Driver.Core.Misc; using PluginModuleBase; +using ProblemSource.Models.Aggregates; using ProblemSource.Services; using ProblemSource.Services.Storage; using ProblemSource.Services.Storage.AzureTables; +using ProblemSourceModule.Models; +using ProblemSourceModule.Models.Aggregates; using ProblemSourceModule.Services; using ProblemSourceModule.Services.Storage; using ProblemSourceModule.Services.Storage.AzureTables; @@ -104,16 +108,38 @@ public void ConfigureForMongoDb(IServiceCollection services, IConfiguration conf var client = new MongoClient(connectionString); services.AddSingleton(sp => client.GetDatabase(database)); - // DocumentBase MongoDocumentWrapper - //BsonClassMap.RegisterClassMap(cm => - //{ - // cm.AutoMap(); - // //cm.MapIdProperty(c => c.Id) - // // .SetIdGenerator(MongoDB.Bson.Serialization.IdGenerators.StringObjectIdGenerator.Instance) - // // .SetSerializer(new MongoDB.Bson.Serialization.Serializers.StringSerializer(MongoDB.Bson.BsonType.ObjectId)); - //}); + // DocumentBase MongoDocumentWrapper - BsonSerializer.RegisterSerializer(new XObjectCustomSerializer()); + //BsonClassMap.RegisterClassMap>(cm => { }); + var wrappedTypes = new[] { typeof(User), typeof(Training) } + .Select(o => typeof(MongoDocumentWrapper<>).MakeGenericType(o)) + .Concat(new[] { typeof(TrainingSummary), typeof(TrainingDayAccount), typeof(Phase), typeof(PhaseStatistics) } + .Select(o => typeof(MongoTrainingAssociatedDocumentWrapper<>).MakeGenericType(o))); + // +typeof(TDocument) { Name = "MongoTrainingAssociatedDocumentWrapper`1" FullName = "ProblemSourceModule.Services.Storage.MongoDb.MongoTrainingAssociatedDocumentWrapper`1[[ProblemSourceModule.Models.Aggregates.TrainingSummary, ProblemSourceModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]]"} + + foreach (var wrappedType in wrappedTypes) + { + var cm = new BsonClassMap(wrappedType); + cm.AutoMap(); + cm.SetIgnoreExtraElements(true); + BsonClassMap.RegisterClassMap(cm); + } + + BsonClassMap.RegisterClassMap(cm => + { + cm.SetIgnoreExtraElements(true); + }); + + BsonClassMap.RegisterClassMap(cm => + { + cm.AutoMap(); + cm.SetIgnoreExtraElements(true); + //cm.MapIdProperty(c => c.Id) + // .SetIdGenerator(MongoDB.Bson.Serialization.IdGenerators.StringObjectIdGenerator.Instance) + // .SetSerializer(new MongoDB.Bson.Serialization.Serializers.StringSerializer(MongoDB.Bson.BsonType.ObjectId)); + }); + + BsonSerializer.RegisterSerializer(new XObjectCustomSerializer()); services.AddSingleton(); services.AddSingleton(); diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs index b84a055..c41693f 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbCollectionWithId.cs @@ -130,8 +130,18 @@ public class XObjectCustomSerializer : SerializerBase return null; } var json = context.Reader.ReadString(); - var value = System.Text.Json.JsonSerializer.Deserialize(json); - return value; + //if (json.StartsWith("[")) + try + { + var parsed = Newtonsoft.Json.Linq.JToken.Parse(json); + //var value = System.Text.Json.JsonSerializer.Deserialize(json); + return parsed; + } + catch (Exception ex) + { + Console.WriteLine($"XObjectCustomSerializer {ex}"); + return null; + } } public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, object? value) diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs index e416fe9..1ebb0c0 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/DbWrappedCollection.cs @@ -14,7 +14,7 @@ public class DbTrainingAssociatedWrappedCollection public DbTrainingAssociatedWrappedCollection(IMongoDatabase db, Func getId, Func> createWrapped) { - collection = new DbCollectionWithId, TId>(db, nameof(MongoTrainingAssociatedDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); + collection = new DbCollectionWithId, TId>(db, "", wrapper => getId(wrapper.Document)); this.getId = getId; this.createWrapped = createWrapped; } @@ -53,9 +53,9 @@ public class DbWrappedCollection protected DbCollectionWithId, TId> collection; private readonly Func> createWrapped; - public DbWrappedCollection(IMongoDatabase db, Func getId, Func> createWrapped) + public DbWrappedCollection(IMongoDatabase db, Func getId, string idField, Func> createWrapped) { - collection = new DbCollectionWithId, TId>(db, nameof(MongoDocumentWrapper.RowKey), wrapper => getId(wrapper.Document)); + collection = new DbCollectionWithId, TId>(db, idField, wrapper => getId(wrapper.Document)); this.createWrapped = createWrapped; } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs index c9716ca..4c8fbc8 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoDocumentWrapper.cs @@ -10,15 +10,15 @@ public MongoDocumentWrapper() } [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] - public MongoDocumentWrapper(TDocument doc, Func? getId = null) : this() - { + public MongoDocumentWrapper(TDocument doc) : this() //Func? getId = null + { Document = doc; - if (getId != null) - RowKey = getId(doc); + //if (getId != null) + // RowKey = getId(doc); } //[BsonS] - public string RowKey { get; set; } = string.Empty; + //public string RowKey { get; set; } = string.Empty; public required TDocument Document { get; set; } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs index f2a2223..71cdc34 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingAssociatedBatchRepository.cs @@ -9,8 +9,8 @@ public class MongoTrainingAssociatedDocumentWrapper : MongoDocumentWr public MongoTrainingAssociatedDocumentWrapper() { } [System.Diagnostics.CodeAnalysis.SetsRequiredMembers] - public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Func? getId = null) : base(doc, getId) - { + public MongoTrainingAssociatedDocumentWrapper(TDocument doc, int trainingId, Func? getId = null) : base(doc) //getId + { TrainingId = trainingId; } public int TrainingId { get; set; } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs index 49d1f85..0fe6aad 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingPlanRepository.cs @@ -5,16 +5,16 @@ namespace ProblemSourceModule.Services.Storage.MongoDb { public class MongoUserRepository : DbWrappedCollection, IUserRepository { - public MongoUserRepository(IMongoDatabase db) : base(db, u => u.Email, item => new MongoDocumentWrapper(item, o => o.Email)) //nameof(User.Email), u => u.Document.Email + public MongoUserRepository(IMongoDatabase db) : base(db, u => u.Email, "Document.Email", item => new MongoDocumentWrapper(item)) //, o => o.Email nameof(User.Email), u => u.Document.Email { } public async Task Add(User item) => await Upsert(item); } - public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository + public class MongoTrainingRepository : DbWrappedCollection, ITrainingRepository { private SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); - public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id.ToString(), item => new MongoDocumentWrapper(item, o => o.Id.ToString())) + public MongoTrainingRepository(IMongoDatabase db) : base(db, u => u.Id, "Document._id", item => new MongoDocumentWrapper(item)) // o => o.Id.ToString() { } public Task Add(Training item) => AddGetId(item); @@ -32,8 +32,8 @@ public async Task AddGetId(Training item) return item.Id; } - public Task Get(int id) => Get(id.ToString()); + //public Task Get(int id) => Get(id); //.ToString() - public async Task> GetByIds(IEnumerable ids) => await Get(ids.Select(o => o.ToString())); + public async Task> GetByIds(IEnumerable ids) => await Get(ids); //.Select(o => o.ToString())); } } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs index a2673c1..44c6802 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTrainingSummaryRepository.cs @@ -15,8 +15,8 @@ public class MongoTrainingSummaryRepository : /* DbWrappedCollection 0, item => new MongoDocumentWrapper(item, o => "0")) { - collection = new DbWrappedCollection(db, item => 0, item => new MongoDocumentWrapper(item, o => "0")); - } + collection = new DbWrappedCollection(db, item => 0, "Document._id", item => new MongoDocumentWrapper(item)); //, o => "0" + } public async Task> GetAll() => (await collection.GetAll()).ToList(); } From 2bb1c982a580f8d11e5b26f19f08ed606278b80a Mon Sep 17 00:00:00 2001 From: JWMB Date: Sun, 1 Feb 2026 16:54:57 +0100 Subject: [PATCH 14/38] local MLPredictNumberlineLevelService, QueueListener: QueueClient -> IQueueListener --- .../IServiceProviderExtensions.cs | 2 +- .../ProblemSourceModule.cs | 31 ++++--- .../ProblemSourceProcessingMiddleware.cs | 7 +- .../TrainingAnalyzers/CategorizerDay5_23Q1.cs | 8 +- .../TrainingAnalyzers/CategorizerDay5_24Q1.cs | 2 +- .../TrainingApi/RealTime/QueueListener.cs | 84 +++++++++++++++---- .../TrainingApi/RealTime/RealTimeConfig.cs | 5 +- ProblemSource/TrainingApi/RealTime/Startup.cs | 18 ++-- ProblemSource/TrainingApi/appsettings.json | 2 +- Tools/MigrateToMongoDb.cs | 2 +- 10 files changed, 106 insertions(+), 55 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/IServiceProviderExtensions.cs b/ProblemSource/ProblemSourceModule/IServiceProviderExtensions.cs index b20244f..cdf494a 100644 --- a/ProblemSource/ProblemSourceModule/IServiceProviderExtensions.cs +++ b/ProblemSource/ProblemSourceModule/IServiceProviderExtensions.cs @@ -27,7 +27,7 @@ public static object CreateInstance(this IServiceProvider instance, Type type) var constructor = constructors.First(); var parameterInfo = constructor.GetParameters(); - var parameters = parameterInfo.Select(o => instance.GetRequiredService(o.ParameterType)).ToArray(); + var parameters = parameterInfo.Select(o => instance.GetService(o.ParameterType)).ToArray(); return constructor.Invoke(parameters); } diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index b98ccdd..dd153d9 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.DependencyInjection; using MongoDB.Bson.Serialization; using MongoDB.Driver; -using MongoDB.Driver.Core.Misc; using PluginModuleBase; using ProblemSource.Models.Aggregates; using ProblemSource.Services; @@ -19,7 +18,6 @@ using ProblemSourceModule.Services.Storage.AzureTables; using ProblemSourceModule.Services.Storage.MongoDb; using ProblemSourceModule.Services.TrainingAnalyzers; -using System.Linq; namespace ProblemSource { @@ -42,26 +40,29 @@ public void ConfigureServices(IServiceCollection services, IConfiguration config .Where(o => !o.IsInterface) .ToArray(); - //var pathToMLModel = "Resources/JuliaMLModel_Reg.zip"; - services.AddSingleton(sp => - new MLPredictNumberlineLevelService(new RemoteMLPredictor(sp.GetRequiredService().GetOrThrow("MLPredictionEndpoint"), sp.GetRequiredService())) - //new MLPredictNumberlineLevelService(new LocalMLPredictor( - // sp.GetRequiredService().ContentRootFileProvider.GetFileInfo(pathToMLModel)?.PhysicalPath ?? "" - //)) - ); - services.AddSingleton>(sp => analyzers.Select(o => (ITrainingAnalyzer)sp.GetOrCreateInstance(o))); + var pathToMLModel = "Resources/JuliaMLModel_Reg.zip"; // TODO: config + if (File.Exists(pathToMLModel)) + { + services.AddSingleton(sp => + //new MLPredictNumberlineLevelService(new RemoteMLPredictor(sp.GetRequiredService().GetOrThrow("MLPredictionEndpoint"), sp.GetRequiredService())) + new MLPredictNumberlineLevelService(new LocalMLPredictor( + sp.GetRequiredService().ContentRootFileProvider.GetFileInfo(pathToMLModel)?.PhysicalPath ?? "" + )) + ); + } + services.AddSingleton(sp => analyzers.Select(o => (ITrainingAnalyzer)sp.GetOrCreateInstance(o))); services.AddSingleton(); //services.AddSingleton(sp => new TrainingAnalyzerCollection(new[] { }, sp.GetRequiredService<>)); - if (services.Any(o => o.ServiceType == typeof(IEventDispatcher)) == false) - services.AddSingleton(); // AzureQueueEventDispatcher>(); - services.AddSingleton(); + // services.AddSingleton(sp => //// new QueueEventDispatcher(sp.GetRequiredService().GetOrThrow("AppSettings:AzureQueue:ConnectionString"), sp.GetRequiredService>())); + if (services.Any(o => o.ServiceType == typeof(IEventDispatcher)) == false) + services.AddSingleton(); // TODO: shouldn't be needed (nullable in ProblemSourceProcessingMiddleware) - services.AddMemoryCache(); + services.AddMemoryCache(); services.AddSingleton(); @@ -69,9 +70,7 @@ public void ConfigureServices(IServiceCollection services, IConfiguration config config ??= services.Select(o => o.ImplementationInstance).OfType().Single(); { if (config != null) - { storageIsMongo = config["AppSettings:Storage:Type"] == "MongoDB"; - } } if (storageIsMongo) ConfigureForMongoDb(services, config!); diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs b/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs index adb4cb4..e2f1e48 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs @@ -22,7 +22,7 @@ public class ProblemSourceProcessingMiddleware : IProcessingMiddleware private readonly ITrainingPlanRepository trainingPlanRepository; private readonly IClientSessionManager sessionManager; private readonly IDataSink dataSink; - private readonly IEventDispatcher eventDispatcher; + private readonly IEventDispatcher? eventDispatcher; private readonly IAggregationService aggregationService; private readonly IUserGeneratedDataRepositoryProviderFactory userGeneratedRepositoriesFactory; private readonly UsernameHashing usernameHashing; @@ -34,7 +34,7 @@ public class ProblemSourceProcessingMiddleware : IProcessingMiddleware //public bool SupportsMiddlewarePattern => throw new NotImplementedException(); public ProblemSourceProcessingMiddleware(ITrainingPlanRepository trainingPlanRepository, - IClientSessionManager sessionManager, IDataSink dataSink, IEventDispatcher eventDispatcher, IAggregationService aggregationService, + IClientSessionManager sessionManager, IDataSink dataSink, IEventDispatcher? eventDispatcher, IAggregationService aggregationService, IUserGeneratedDataRepositoryProviderFactory userGeneratedRepositoriesFactory, UsernameHashing usernameHashing, MnemoJapanese mnemoJapanese, ITrainingRepository trainingRepository, TrainingAnalyzerCollection trainingAnalyzers, ILogger log) @@ -217,6 +217,9 @@ private IUserGeneratedDataRepositoryProvider AssertSession(Training training, st private async Task DispatchIncoming(Training training, SyncInput root) { + if (eventDispatcher == null) + return; + try { // E.g. for real-time teacher view diff --git a/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_23Q1.cs b/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_23Q1.cs index 111a447..7b6deec 100644 --- a/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_23Q1.cs +++ b/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_23Q1.cs @@ -13,10 +13,10 @@ namespace ProblemSourceModule.Services.TrainingAnalyzers { public class CategorizerDay5_23Q1 : ITrainingAnalyzer { - protected readonly IPredictNumberlineLevelService modelService; + protected readonly IPredictNumberlineLevelService? modelService; protected readonly ILogger log; - public CategorizerDay5_23Q1(IPredictNumberlineLevelService modelService, ILogger log) + public CategorizerDay5_23Q1(IPredictNumberlineLevelService? modelService, ILogger log) { this.modelService = modelService; this.log = log; @@ -75,6 +75,10 @@ public async Task Analyze(Training training, IUserGeneratedDataRepositoryP public async Task Predict(Training training, IUserGeneratedDataRepositoryProvider provider) { + if (modelService == null) + { + return new PredictedNumberlineLevel(); + } var mlFeatures = await CreateFeatures(training, provider); var result = await modelService.Predict(mlFeatures); if (result.PredictedPerformanceTier == PredictedNumberlineLevel.PerformanceTier.Unknown) diff --git a/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_24Q1.cs b/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_24Q1.cs index 4f7f42d..93511bd 100644 --- a/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_24Q1.cs +++ b/ProblemSource/ProblemSourceModule/Services/TrainingAnalyzers/CategorizerDay5_24Q1.cs @@ -6,7 +6,7 @@ namespace ProblemSourceModule.Services.TrainingAnalyzers { public class CategorizerDay5_24Q1 : CategorizerDay5_23Q1 { - public CategorizerDay5_24Q1(IPredictNumberlineLevelService modelService, ILogger log) + public CategorizerDay5_24Q1(IPredictNumberlineLevelService? modelService, ILogger log) : base(modelService, log) { } diff --git a/ProblemSource/TrainingApi/RealTime/QueueListener.cs b/ProblemSource/TrainingApi/RealTime/QueueListener.cs index b3d11a8..b012db3 100644 --- a/ProblemSource/TrainingApi/RealTime/QueueListener.cs +++ b/ProblemSource/TrainingApi/RealTime/QueueListener.cs @@ -7,28 +7,76 @@ namespace TrainingApi.RealTime { - public class QueueListener + public interface IQueueMessage { - private readonly QueueClient client; - private readonly CommHubWrapper chatHub; - private readonly IAccessResolver accessResolver; - private readonly ILogger log; + string MessageId { get; } + string PopReceipt { get; } + BinaryData Body { get; } + } + + public interface IQueueClient + { + Task ReceiveMessagesAsync(int? maxMessages = null, TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default); + Task DeleteMessageAsync(string messageId, string popReceipt, CancellationToken cancellationToken = default); + } + + public class AzureQueueClient : IQueueClient + { + private QueueClient client; + + public AzureQueueClient(AzureQueueConfig config) + { + if (string.IsNullOrEmpty(config.ConnectionString)) + throw new ArgumentException("null or empty", nameof(config.ConnectionString)); + + if (string.IsNullOrEmpty(config.QueueName)) + throw new ArgumentException("null or empty", nameof(config.QueueName)); + + client = new QueueClient(config.ConnectionString, config.QueueName); // "UseDevelopmentStorage=true", "problemsource-sync"); + // TODO: move to some async Init + client.CreateIfNotExists(); + } + + public async Task DeleteMessageAsync(string messageId, string popReceipt, CancellationToken cancellationToken = default) + { + var result = await client.DeleteMessageAsync(messageId, popReceipt, cancellationToken); + return result.Status < 400; + } + + public async Task ReceiveMessagesAsync(int? maxMessages = null, TimeSpan? visibilityTimeout = null, CancellationToken cancellationToken = default) + { + var result = await client.ReceiveMessagesAsync(maxMessages, visibilityTimeout, cancellationToken); + if (result.Value == null) + throw new Exception(""); + return result.Value.Select(o => new WrappedMessage(o)).ToArray(); + } - public QueueListener(CommHubWrapper chatHub, IAccessResolver accessResolver, RealTimeConfig config, ILogger log) + public class WrappedMessage : IQueueMessage { - if (config.AzureQueueConfig == null) - throw new NullReferenceException($"{nameof(config.AzureQueueConfig)}"); + private readonly QueueMessage message; + + public WrappedMessage(QueueMessage message) + { + this.message = message; + } + public string MessageId => message.MessageId; + public string PopReceipt => message.PopReceipt; + public BinaryData Body => message.Body; - if (string.IsNullOrEmpty(config.AzureQueueConfig.ConnectionString)) - throw new ArgumentException("null or empty", nameof(config.AzureQueueConfig.ConnectionString)); + } - if (string.IsNullOrEmpty(config.AzureQueueConfig.QueueName)) - throw new ArgumentException("null or empty", nameof(config.AzureQueueConfig.QueueName)); + } - client = new QueueClient(config.AzureQueueConfig.ConnectionString, config.AzureQueueConfig.QueueName); // "UseDevelopmentStorage=true", "problemsource-sync"); + public class QueueListener + { + private readonly IQueueClient client; + private readonly CommHubWrapper chatHub; + private readonly IAccessResolver accessResolver; + private readonly ILogger log; - // TODO: move to some async Init - client.CreateIfNotExists(); + public QueueListener(CommHubWrapper chatHub, IAccessResolver accessResolver, RealTimeConfig config, IQueueClient queueClient, ILogger log) + { + client = queueClient; this.chatHub = chatHub; this.accessResolver = accessResolver; @@ -74,8 +122,8 @@ public async Task Receive(CancellationToken cancellationToken) try { - var response = await client.ReceiveMessagesAsync(32, cancellationToken: cancellationToken); - var msgs = response.Value; + var msgs = await client.ReceiveMessagesAsync(32, cancellationToken: cancellationToken); + //var msgs = response.Value; foreach (var msg in msgs) { if (cancellationToken.IsCancellationRequested) @@ -105,7 +153,7 @@ public async Task Receive(CancellationToken cancellationToken) isWorking = false; } - private async Task ProcessMessage(QueueMessage msg) + private async Task ProcessMessage(IQueueMessage msg) { JObject? jObj = null; try diff --git a/ProblemSource/TrainingApi/RealTime/RealTimeConfig.cs b/ProblemSource/TrainingApi/RealTime/RealTimeConfig.cs index e411aa3..93bfcba 100644 --- a/ProblemSource/TrainingApi/RealTime/RealTimeConfig.cs +++ b/ProblemSource/TrainingApi/RealTime/RealTimeConfig.cs @@ -1,10 +1,7 @@ -using ProblemSource.Services; - -namespace TrainingApi.RealTime +namespace TrainingApi.RealTime { public class RealTimeConfig { public bool Enabled { get; set; } = false; - public AzureQueueConfig? AzureQueueConfig { get; set; } } } diff --git a/ProblemSource/TrainingApi/RealTime/Startup.cs b/ProblemSource/TrainingApi/RealTime/Startup.cs index 2ff77d7..b2d7b62 100644 --- a/ProblemSource/TrainingApi/RealTime/Startup.cs +++ b/ProblemSource/TrainingApi/RealTime/Startup.cs @@ -1,7 +1,4 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using Microsoft.Extensions.DependencyInjection.Extensions; -using ProblemSource.Services; +using ProblemSource.Services; namespace TrainingApi.RealTime { @@ -9,19 +6,22 @@ public class Startup // TODO: can we somehow utilize IStartupFilter? { public void ConfigureServices(IServiceCollection services) { + throw new NotImplementedException("No non-Azure services implemented"); + services.AddSingleton(); + services.AddSignalR(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddHostedService(sp => new TimedHostedService(sp.GetRequiredService().Receive, sp.GetRequiredService>())); - var registered = services.FirstOrDefault(o => o.ServiceType == typeof(IEventDispatcher)); - if (registered != null) - services.Remove(registered); + //var registered = services.FirstOrDefault(o => o.ServiceType == typeof(IEventDispatcher)); + //if (registered != null) + // services.Remove(registered); services.AddSingleton(); - } + } - public void Configure(WebApplication app, string pathPattern) + public void Configure(WebApplication app, string pathPattern) { // TODO: read up on https://learn.microsoft.com/en-us/aspnet/core/signalr/hubs?view=aspnetcore-7.0 app.MapHub(pathPattern, options => diff --git a/ProblemSource/TrainingApi/appsettings.json b/ProblemSource/TrainingApi/appsettings.json index 0cd5915..cd298d7 100644 --- a/ProblemSource/TrainingApi/appsettings.json +++ b/ProblemSource/TrainingApi/appsettings.json @@ -31,7 +31,7 @@ } }, "RealTime": { - "Enabled": true, + "Enabled": false, "AzureQueueConfig": { "ConnectionString": "UseDevelopmentStorage=true", "QueueName": "vektorsync" diff --git a/Tools/MigrateToMongoDb.cs b/Tools/MigrateToMongoDb.cs index e684aab..954c5f1 100644 --- a/Tools/MigrateToMongoDb.cs +++ b/Tools/MigrateToMongoDb.cs @@ -86,7 +86,7 @@ public async Task Migrate() //var mdbTrainings = new DbWrappedCollection(db, d => d.Id, d => new MongoDocumentWrapper(d)); var mdbTrainings = new MongoTrainingRepository(db); Console.WriteLine("Reading trainings..."); - var allMongoTrainings = (await mdbTrainings.GetCollection().Find(o => true).Project(o => o.RowKey).ToListAsync()).Select(int.Parse).ToList(); + var allMongoTrainings = await mdbTrainings.GetCollection().Find(o => true).Project(o => o.Document.Id).ToListAsync(); var trainings = (await trainingRepository.GetAll()) .Where(o => o.Id > 32431) From 637a65015e214be40b4353b2291ae73119606c93 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 16:09:28 +0100 Subject: [PATCH 15/38] compose.yaml --- compose.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 compose.yaml diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3b8029d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,21 @@ +services: + api: + image: "localhost/trainingapi" + # image: "localhost/complimentgeneratorapi" + ports: + - "9090:80" + depends_on: + - mongodb + mongodb: + image: "docker.io/mongodb/mongodb-community-server" + environment: + - MONGO_INITDB_ROOT_USERNAME=user + - MONGO_INITDB_ROOT_PASSWORD=pass + volumes: + - db-data:/etc/data + # - type: bind + # source: ./data + # target: /data/db + +volumes: + db-data: \ No newline at end of file From ac79eaef19268b1dfecb4b5d0d652f8f12bf458e Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 16:40:23 +0100 Subject: [PATCH 16/38] dotnet 10, a few nugets --- Common.Web/Common.Web.csproj | 6 +++--- Common/Common.csproj | 2 +- Directory.Packages.props | 2 +- EmailServices/EmailServices.csproj | 4 ++-- ML.Helpers/ML.Helpers.csproj | 4 ++-- MLTools/ML.Dynamic.csproj | 2 +- Organization.Tests/Organization.Tests.csproj | 4 ++-- Organization/Organization.csproj | 4 ++-- PluginModuleBase/PluginModuleBase.csproj | 2 +- ProblemSource/OldDb/OldDb.csproj | 2 +- ProblemSource/OldDbAdapter/OldDbAdapter.csproj | 2 +- .../ProblemSourceModule.Tests/MongoDbTests.cs | 1 - .../ProblemSourceModule.Tests.csproj | 3 +-- .../ProblemSourceModule.csproj | 4 ++-- .../AzureTableTrainingRepository.cs | 18 ++++++++++-------- .../ProblemSourceTestClient.csproj | 4 ++-- .../TrainingApi.Tests/TrainingApi.Tests.csproj | 2 +- ProblemSource/TrainingApi/Startup.cs | 16 ++++++++-------- ProblemSource/TrainingApi/TrainingApi.csproj | 3 +-- ProblemSource/WebJob/WebJob.csproj | 2 +- Tools/Tools.csproj | 2 +- 21 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Common.Web/Common.Web.csproj b/Common.Web/Common.Web.csproj index 5292d0b..fedf74b 100644 --- a/Common.Web/Common.Web.csproj +++ b/Common.Web/Common.Web.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable @@ -9,7 +9,7 @@ - + diff --git a/Common/Common.csproj b/Common/Common.csproj index d8110da..8038139 100644 --- a/Common/Common.csproj +++ b/Common/Common.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable diff --git a/Directory.Packages.props b/Directory.Packages.props index 3839254..5cd9b55 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -50,7 +50,7 @@ - + diff --git a/EmailServices/EmailServices.csproj b/EmailServices/EmailServices.csproj index c37e7ac..6a3c040 100644 --- a/EmailServices/EmailServices.csproj +++ b/EmailServices/EmailServices.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable diff --git a/ML.Helpers/ML.Helpers.csproj b/ML.Helpers/ML.Helpers.csproj index fa7a238..0fa851b 100644 --- a/ML.Helpers/ML.Helpers.csproj +++ b/ML.Helpers/ML.Helpers.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable diff --git a/MLTools/ML.Dynamic.csproj b/MLTools/ML.Dynamic.csproj index 9d55849..425d5bd 100644 --- a/MLTools/ML.Dynamic.csproj +++ b/MLTools/ML.Dynamic.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable diff --git a/Organization.Tests/Organization.Tests.csproj b/Organization.Tests/Organization.Tests.csproj index c8bf9b4..9fda1c8 100644 --- a/Organization.Tests/Organization.Tests.csproj +++ b/Organization.Tests/Organization.Tests.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable false diff --git a/Organization/Organization.csproj b/Organization/Organization.csproj index bcbf88c..b46c433 100644 --- a/Organization/Organization.csproj +++ b/Organization/Organization.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable diff --git a/PluginModuleBase/PluginModuleBase.csproj b/PluginModuleBase/PluginModuleBase.csproj index f405d14..4d064d2 100644 --- a/PluginModuleBase/PluginModuleBase.csproj +++ b/PluginModuleBase/PluginModuleBase.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable diff --git a/ProblemSource/OldDb/OldDb.csproj b/ProblemSource/OldDb/OldDb.csproj index d20c515..b2e5bb8 100644 --- a/ProblemSource/OldDb/OldDb.csproj +++ b/ProblemSource/OldDb/OldDb.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable diff --git a/ProblemSource/OldDbAdapter/OldDbAdapter.csproj b/ProblemSource/OldDbAdapter/OldDbAdapter.csproj index 83f0fdd..994c09e 100644 --- a/ProblemSource/OldDbAdapter/OldDbAdapter.csproj +++ b/ProblemSource/OldDbAdapter/OldDbAdapter.csproj @@ -1,6 +1,6 @@ - net9.0 + net10.0 enable enable diff --git a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs index 955ddca..0bd1844 100644 --- a/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs +++ b/ProblemSource/ProblemSourceModule.Tests/MongoDbTests.cs @@ -1,6 +1,5 @@ //using EphemeralMongo; using Mongo2Go; -using MongoDB.Bson.Serialization; using MongoDB.Driver; using ProblemSource.Models.Aggregates; using ProblemSourceModule.Models; diff --git a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj index de68acf..0f5e857 100644 --- a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj +++ b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable false @@ -18,7 +18,6 @@ - diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj index e0ab65c..9f7de4b 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj @@ -1,6 +1,6 @@ - + - net9.0 + net10.0 enable enable diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs index 29acf68..f5ba953 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/AzureTables/AzureTableTrainingRepository.cs @@ -24,10 +24,7 @@ public AzureTableTrainingRepository(ITypedTableClientFactory tableClientFactory) private static int latestMax = 0; // TODO: ugly performance hack while waiting to port to SQL private object _lock = new object(); - - public Task Add(Training item) => AddGetId(item); - - public Task AddGetId(Training item) + public Task Add(Training item) { // Warning: multi-instance concurrency lock (_lock) @@ -41,10 +38,9 @@ public Task AddGetId(Training item) } public async Task Update(Training item) => await repo.Update(item); - //public async Task Upsert(Training item) => int.Parse(await repo.Upsert(item)); - public async Task Upsert(Training item) => await repo.Upsert(item); + public async Task Upsert(Training item) => int.Parse(await repo.Upsert(item)); - public async Task Remove(Training item) => await repo.Remove(item); + public async Task Remove(Training item) => await repo.Remove(item); public async Task> GetAll() => await repo.GetAll(); @@ -54,5 +50,11 @@ public async Task> GetByIds(IEnumerable ids) // Note: skips entries that were not found return values.Values.OfType(); } - } + + Task IRepository.Add(Training item) => Add(item); + + Task IRepository.Upsert(Training item) => Upsert(item); + + public Task AddGetId(Training item) => throw new NotImplementedException(); + } } diff --git a/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj b/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj index 73b329e..16b5e34 100644 --- a/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj +++ b/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj @@ -1,7 +1,7 @@ - + Exe - net9.0 + net10.0 enable enable diff --git a/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj b/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj index 1b96537..5e1ff14 100644 --- a/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj +++ b/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable false diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index d22cea9..67d69e0 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -22,7 +22,7 @@ namespace TrainingApi public class Startup { private IPluginModule[] plugins = Array.Empty(); - private OldDbAdapter.Startup? oldDbStartup = null; + //private OldDbAdapter.Startup? oldDbStartup = null; private RealTime.Startup? realTimeStartup; public void ConfigureServices(IServiceCollection services, ConfigurationManager configurationManager, IWebHostEnvironment env) @@ -57,8 +57,9 @@ public void ConfigureServices(IServiceCollection services, ConfigurationManager var oldDbEnabled = configurationManager.GetValue("OldDbEnabled"); if (oldDbEnabled && System.Diagnostics.Debugger.IsAttached) { - oldDbStartup = new OldDbAdapter.Startup(); - oldDbStartup.ConfigureServices(services); + throw new NotImplementedException(); + //oldDbStartup = new OldDbAdapter.Startup(); + //oldDbStartup.ConfigureServices(services); } services.AddControllers(); @@ -163,8 +164,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) var referer = context.Request.GetTypedHeaders().Referer; // when from swagger and localhost var autologin = referer?.AbsolutePath.Contains("/swagger/") == true - || referer?.AbsoluteUri.StartsWith("http://localhost:") == true - || referer?.AbsoluteUri.StartsWith("https://localhost") == true; + || referer?.AbsoluteUri.StartsWith("http://localhost:") == true; if (!autologin) { @@ -207,8 +207,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) ServiceConfiguration.ConfigureApplicationInsights(app, config, env.IsDevelopment()); - if (oldDbStartup != null) - oldDbStartup.Configure(app); + //if (oldDbStartup != null) + // oldDbStartup.Configure(app); } private IPluginModule[] ConfigureProblemSource(IServiceCollection services, IConfiguration config, IHostEnvironment env) @@ -217,7 +217,7 @@ private IPluginModule[] ConfigureProblemSource(IServiceCollection services, ICon ConfigureAuthentication(services, config, env); var plugins = new IPluginModule[] { new ProblemSource.ProblemSourceModule() }; - //services.AddSingleton(); + services.AddSingleton(); ServiceConfiguration.ConfigureProcessingPipelineServices(services, config, plugins); return plugins; } diff --git a/ProblemSource/TrainingApi/TrainingApi.csproj b/ProblemSource/TrainingApi/TrainingApi.csproj index 5674536..0344ce5 100644 --- a/ProblemSource/TrainingApi/TrainingApi.csproj +++ b/ProblemSource/TrainingApi/TrainingApi.csproj @@ -1,6 +1,6 @@  - net9.0 + net10.0 enable enable @@ -21,7 +21,6 @@ - diff --git a/ProblemSource/WebJob/WebJob.csproj b/ProblemSource/WebJob/WebJob.csproj index 23c04ca..4e69a55 100644 --- a/ProblemSource/WebJob/WebJob.csproj +++ b/ProblemSource/WebJob/WebJob.csproj @@ -1,7 +1,7 @@ Exe - net9.0 + net10.0 enable enable diff --git a/Tools/Tools.csproj b/Tools/Tools.csproj index e550f05..7f73019 100644 --- a/Tools/Tools.csproj +++ b/Tools/Tools.csproj @@ -1,7 +1,7 @@  Exe - net9.0 + net10.0 enable enable beba58a4-9d12-469a-8899-4792da28247b From 23356fa0c439182ae788f1b1f076845449414c85 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 17:06:19 +0100 Subject: [PATCH 17/38] nugets --- Directory.Packages.props | 80 ++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5cd9b55..c4d50ea 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,66 +5,66 @@ $(NoWarn);NU1507 - + - - - - - - + + + + + + - - - - - + + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - - - - - - - + + + + + + + - + - - - + + + \ No newline at end of file From 08efd100f8862c13f2d4e07f02e3afc85c00987d Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 17:10:51 +0100 Subject: [PATCH 18/38] forgot LightGbm --- Directory.Packages.props | 2 +- ProblemSource/TrainingApi/Startup.cs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index c4d50ea..8fffd13 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -47,7 +47,7 @@ - + diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index 67d69e0..35e1bf8 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.FileProviders; using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; -using Microsoft.OpenApi.Models; using PluginModuleBase; using ProblemSource.Services; using ProblemSourceModule.Services; @@ -86,8 +85,8 @@ public void ConfigureServices(IServiceCollection services, ConfigurationManager services.AddLogging(builder => { - builder.AddApplicationInsights(); - }); + builder.AddApplicationInsights(); // AddAzureWebAppDiagnostics + }); services.Configure(telemetryConfiguration => { From e571abecbab96e13277fb44cb887a2e2daf63dab Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 17:33:28 +0100 Subject: [PATCH 19/38] revert breaking nuget upgrades --- Directory.Packages.props | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 8fffd13..0ce35a6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,8 +20,8 @@ - - + + @@ -53,10 +53,10 @@ - - + + - + From 5b73706bfe652740f8e785b4f282cfe1e59022c2 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 17:34:28 +0100 Subject: [PATCH 20/38] fix build errors --- .../ProblemSourceProcessingMiddleware.cs | 9 +- ProblemSource/TrainingApi/RolesRequirement.cs | 2 +- .../Services/OldDbStatisticsProvider.cs | 190 +++++++++--------- ProblemSource/TrainingApi/Startup.cs | 1 + Tools/ClientUtils.cs | 11 +- 5 files changed, 104 insertions(+), 109 deletions(-) diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs b/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs index e2f1e48..c50d755 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceProcessingMiddleware.cs @@ -12,8 +12,6 @@ using ProblemSourceModule.Services; using ProblemSourceModule.Services.Storage; using System.Security.Claims; -using static Microsoft.ApplicationInsights.MetricDimensionNames.TelemetryContext; -using static ProblemSource.Services.LogEventsToPhases; namespace ProblemSource { @@ -22,7 +20,7 @@ public class ProblemSourceProcessingMiddleware : IProcessingMiddleware private readonly ITrainingPlanRepository trainingPlanRepository; private readonly IClientSessionManager sessionManager; private readonly IDataSink dataSink; - private readonly IEventDispatcher? eventDispatcher; + private readonly IEventDispatcher eventDispatcher; private readonly IAggregationService aggregationService; private readonly IUserGeneratedDataRepositoryProviderFactory userGeneratedRepositoriesFactory; private readonly UsernameHashing usernameHashing; @@ -34,7 +32,7 @@ public class ProblemSourceProcessingMiddleware : IProcessingMiddleware //public bool SupportsMiddlewarePattern => throw new NotImplementedException(); public ProblemSourceProcessingMiddleware(ITrainingPlanRepository trainingPlanRepository, - IClientSessionManager sessionManager, IDataSink dataSink, IEventDispatcher? eventDispatcher, IAggregationService aggregationService, + IClientSessionManager sessionManager, IDataSink dataSink, IEventDispatcher eventDispatcher, IAggregationService aggregationService, IUserGeneratedDataRepositoryProviderFactory userGeneratedRepositoriesFactory, UsernameHashing usernameHashing, MnemoJapanese mnemoJapanese, ITrainingRepository trainingRepository, TrainingAnalyzerCollection trainingAnalyzers, ILogger log) @@ -217,9 +215,6 @@ private IUserGeneratedDataRepositoryProvider AssertSession(Training training, st private async Task DispatchIncoming(Training training, SyncInput root) { - if (eventDispatcher == null) - return; - try { // E.g. for real-time teacher view diff --git a/ProblemSource/TrainingApi/RolesRequirement.cs b/ProblemSource/TrainingApi/RolesRequirement.cs index 1a59f41..38a0eec 100644 --- a/ProblemSource/TrainingApi/RolesRequirement.cs +++ b/ProblemSource/TrainingApi/RolesRequirement.cs @@ -1,5 +1,5 @@ using Microsoft.AspNetCore.Authorization; -using OldDb.Models; +//using OldDb.Models; namespace TrainingApi { diff --git a/ProblemSource/TrainingApi/Services/OldDbStatisticsProvider.cs b/ProblemSource/TrainingApi/Services/OldDbStatisticsProvider.cs index 536929e..e392b24 100644 --- a/ProblemSource/TrainingApi/Services/OldDbStatisticsProvider.cs +++ b/ProblemSource/TrainingApi/Services/OldDbStatisticsProvider.cs @@ -1,104 +1,104 @@ -using Microsoft.EntityFrameworkCore; -using OldDb.Models; -using ProblemSource.Models.Aggregates; -using ProblemSource.Services; -using ProblemSourceModule.Models.Aggregates; +//using Microsoft.EntityFrameworkCore; +//using OldDb.Models; +//using ProblemSource.Models.Aggregates; +//using ProblemSource.Services; +//using ProblemSourceModule.Models.Aggregates; -namespace TrainingApi.Services -{ - public class OldDbStatisticsProvider : IStatisticsProvider - { - private readonly TrainingDbContext dbContext; +//namespace TrainingApi.Services +//{ +// public class OldDbStatisticsProvider : IStatisticsProvider +// { +// private readonly TrainingDbContext dbContext; - public OldDbStatisticsProvider(TrainingDbContext dbContext) - { - this.dbContext = dbContext; - } +// public OldDbStatisticsProvider(TrainingDbContext dbContext) +// { +// this.dbContext = dbContext; +// } - public Task> GetAllTrainingSummaries() => throw new NotImplementedException(); +// public Task> GetAllTrainingSummaries() => throw new NotImplementedException(); - public async Task> GetPhaseStatistics(int trainingId) - { - var rows = await dbContext.AggregatedData.Where(o => o.AggregatorId == 1 && o.AccountId == trainingId) - .ToListAsync(); - return rows.Select(o => o.ToPhaseStatistics()).OfType(); - } +// public async Task> GetPhaseStatistics(int trainingId) +// { +// var rows = await dbContext.AggregatedData.Where(o => o.AggregatorId == 1 && o.AccountId == trainingId) +// .ToListAsync(); +// return rows.Select(o => o.ToPhaseStatistics()).OfType(); +// } - public async Task> GetTrainingDays(int trainingId) - { - var rows = await dbContext.AggregatedData.Where(o => o.AggregatorId == 2 && o.AccountId == trainingId) - .ToListAsync(); - return rows.Select(o => o.ToTyped()).OfType(); - } +// public async Task> GetTrainingDays(int trainingId) +// { +// var rows = await dbContext.AggregatedData.Where(o => o.AggregatorId == 2 && o.AccountId == trainingId) +// .ToListAsync(); +// return rows.Select(o => o.ToTyped()).OfType(); +// } - public Task> GetTrainingSummaries(IEnumerable trainingIds) => throw new NotImplementedException(); - } +// public Task> GetTrainingSummaries(IEnumerable trainingIds) => throw new NotImplementedException(); +// } - public static class OldAggregatesExtensions - { - public static PhaseStatistics? ToPhaseStatistics(this AggregatedDatum? row) - { - if (row?.Data == null) - return null; - var data = row.Data.Split('\t'); - // Id Account Training Day Exercise Phase-id PhaseType Timestamp Sequence No of questions No of Correct answers No of Incorrect answers No of correct on first try % correct on first try Lowest Level Highest level Average respone time Total response time ERR #removed dup answ ERR #prob w/o answ Score Target score Planet target score Won race Completed planet - var result = new PhaseStatistics - { - account_id = int.Parse(data[0]), - //ac = data[1], - training_day = int.Parse(data[2]), - exercise = data[3], - //phase_id = data[4], - phase_type = data[5], - timestamp = DateTime.Parse(data[6]), // new DateTime(1970, 1, 1).AddMilliseconds(long.Parse(data[6])), - sequence = int.Parse(data[7]), - num_questions = int.Parse(data[8]), - num_correct_answers = int.Parse(data[9]), - num_incorrect_answers = int.Parse(data[10]), - num_correct_first_try = int.Parse(data[11]), - //percent - level_min = data[13] == "" ? 0M : decimal.Parse(data[13], System.Globalization.CultureInfo.InvariantCulture), - level_max = data[14] == "" ? 0M : decimal.Parse(data[14], System.Globalization.CultureInfo.InvariantCulture), - response_time_avg = int.Parse(data[15]), - response_time_total = data[16] == "" ? 0 : int.Parse(data[16]), - }; +// public static class OldAggregatesExtensions +// { +// public static PhaseStatistics? ToPhaseStatistics(this AggregatedDatum? row) +// { +// if (row?.Data == null) +// return null; +// var data = row.Data.Split('\t'); +// // Id Account Training Day Exercise Phase-id PhaseType Timestamp Sequence No of questions No of Correct answers No of Incorrect answers No of correct on first try % correct on first try Lowest Level Highest level Average respone time Total response time ERR #removed dup answ ERR #prob w/o answ Score Target score Planet target score Won race Completed planet +// var result = new PhaseStatistics +// { +// account_id = int.Parse(data[0]), +// //ac = data[1], +// training_day = int.Parse(data[2]), +// exercise = data[3], +// //phase_id = data[4], +// phase_type = data[5], +// timestamp = DateTime.Parse(data[6]), // new DateTime(1970, 1, 1).AddMilliseconds(long.Parse(data[6])), +// sequence = int.Parse(data[7]), +// num_questions = int.Parse(data[8]), +// num_correct_answers = int.Parse(data[9]), +// num_incorrect_answers = int.Parse(data[10]), +// num_correct_first_try = int.Parse(data[11]), +// //percent +// level_min = data[13] == "" ? 0M : decimal.Parse(data[13], System.Globalization.CultureInfo.InvariantCulture), +// level_max = data[14] == "" ? 0M : decimal.Parse(data[14], System.Globalization.CultureInfo.InvariantCulture), +// response_time_avg = int.Parse(data[15]), +// response_time_total = data[16] == "" ? 0 : int.Parse(data[16]), +// }; - if (data.Length > 17) - { - // ERR #removed dup answ - // ERR #prob w/o answ - // Score Target score - // Planet target score - if (data.Length > 19) - { - result.won_race = bool.Parse(data[22]); - result.completed_planet = bool.Parse(data[23]); - } - } - return result; - } +// if (data.Length > 17) +// { +// // ERR #removed dup answ +// // ERR #prob w/o answ +// // Score Target score +// // Planet target score +// if (data.Length > 19) +// { +// result.won_race = bool.Parse(data[22]); +// result.completed_planet = bool.Parse(data[23]); +// } +// } +// return result; +// } - public static TrainingDayAccount? ToTyped(this AggregatedDatum? row) - { - if (row?.Data == null) - return null; - var data = row.Data.Split('\t'); - // id uuid trainingDay startTime endTimeStamp numRacesWon numRaces numPlanetsWon numCorrectAnswers numQuestions responseMinutes remainingMinutes - return new TrainingDayAccount - { - AccountId = int.Parse(data[0]), - AccountUuid = data[1], - TrainingDay = int.Parse(data[2]), - StartTime = DateTime.Parse(data[3]), //data[3] - EndTimeStamp = new DateTime(1970, 1, 1).AddMilliseconds(long.Parse(data[4])), - NumRacesWon = int.Parse(data[5]), - NumRaces = int.Parse(data[6]), - NumPlanetsWon = int.Parse(data[7]), - NumCorrectAnswers = int.Parse(data[8]), - NumQuestions = int.Parse(data[9]), - ResponseMinutes = int.Parse(data[10]), - RemainingMinutes = int.Parse(data[11]), - }; - } - } -} +// public static TrainingDayAccount? ToTyped(this AggregatedDatum? row) +// { +// if (row?.Data == null) +// return null; +// var data = row.Data.Split('\t'); +// // id uuid trainingDay startTime endTimeStamp numRacesWon numRaces numPlanetsWon numCorrectAnswers numQuestions responseMinutes remainingMinutes +// return new TrainingDayAccount +// { +// AccountId = int.Parse(data[0]), +// AccountUuid = data[1], +// TrainingDay = int.Parse(data[2]), +// StartTime = DateTime.Parse(data[3]), //data[3] +// EndTimeStamp = new DateTime(1970, 1, 1).AddMilliseconds(long.Parse(data[4])), +// NumRacesWon = int.Parse(data[5]), +// NumRaces = int.Parse(data[6]), +// NumPlanetsWon = int.Parse(data[7]), +// NumCorrectAnswers = int.Parse(data[8]), +// NumQuestions = int.Parse(data[9]), +// ResponseMinutes = int.Parse(data[10]), +// RemainingMinutes = int.Parse(data[11]), +// }; +// } +// } +//} diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index 35e1bf8..13ddc45 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.FileProviders; using Microsoft.IdentityModel.Tokens; using Microsoft.Net.Http.Headers; +using Microsoft.OpenApi.Models; using PluginModuleBase; using ProblemSource.Services; using ProblemSourceModule.Services; diff --git a/Tools/ClientUtils.cs b/Tools/ClientUtils.cs index 5748d78..dfcbbee 100644 --- a/Tools/ClientUtils.cs +++ b/Tools/ClientUtils.cs @@ -1,6 +1,5 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NumSharp.Utilities; namespace Tools { @@ -57,17 +56,17 @@ public static List CsvToJsonList(string path) { var lines = File.ReadAllLines(path); - var header = GetItems(lines.First()); + var header = GetItems(lines.First()).ToList(); while (header.Last().Trim() == "") - header = header.RemoveAt(header.Length - 1); + header.RemoveAt(header.Count - 1); //lowercase 1st - header = header.Select(o => $"{o.First()}".ToLower() + o.Substring(1)).ToArray(); + header = header.Select(o => $"{o.First()}".ToLower() + o.Substring(1)).ToList(); var types = header.Select(o => (Type?)null).ToList(); foreach (var line in lines.Skip(1).Take(100)) { var items = GetItems(line); - for (int i = 0; i < Math.Min(items.Length, header.Length); i++) + for (int i = 0; i < Math.Min(items.Length, header.Count); i++) { if (items[i].Length > 0) { @@ -93,7 +92,7 @@ public static List CsvToJsonList(string path) { var obj = new JObject(); var items = GetItems(line); - for (int i = 0; i < Math.Min(items.Length, header.Length); i++) + for (int i = 0; i < Math.Min(items.Length, header.Count); i++) { var item = items[i]; var type = types[i]; From a0919450e1e00cd48fd6ece122c43685e1565261 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 17:54:11 +0100 Subject: [PATCH 21/38] Startup: log instead of throw --- Common.Web/ServiceConfiguration.cs | 3 ++- ProblemSource/TrainingApi/Startup.cs | 32 +++++++++++++++++----------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/Common.Web/ServiceConfiguration.cs b/Common.Web/ServiceConfiguration.cs index 3b13c91..af186f4 100644 --- a/Common.Web/ServiceConfiguration.cs +++ b/Common.Web/ServiceConfiguration.cs @@ -36,7 +36,8 @@ public static void ConfigureApplicationInsights(IApplicationBuilder app, IConfig if (aiConn == "SECRET" || aiConn == string.Empty) { if (isDevelopment == false) - throw new ArgumentException("InstrumentationKey not set"); + Console.WriteLine($"Warning: InstrumentationKey not set ({aiConn})"); + //throw new ArgumentException("InstrumentationKey not set"); } else { diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index 13ddc45..0ceb0f9 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -137,20 +137,28 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) realTimeStartup?.Configure(webApp, "/realtime"); } - // TODO: separate into a method - // static files with fallback to index.html (entry point for admin interface) - var cacheMaxAge = TimeSpan.FromMinutes(10); - var fileProvider = new FallbackFileProvider("index.html", new PhysicalFileProvider(Path.Combine(env.ContentRootPath, "StaticFiles", "Admin")), "/admin"); - app.UseStaticFiles(new StaticFileOptions + try { - ServeUnknownFileTypes = true, - FileProvider = fileProvider, - RequestPath = fileProvider.RootPath, - OnPrepareResponse = ctx => + // TODO: separate into a method + // static files with fallback to index.html (entry point for admin interface) + var cacheMaxAge = TimeSpan.FromMinutes(10); + var fileProvider = new FallbackFileProvider("index.html", new PhysicalFileProvider(Path.Combine(env.ContentRootPath, "StaticFiles", "Admin")), "/admin"); + app.UseStaticFiles(new StaticFileOptions { - ctx.Context.Response.Headers.Append("Cache-Control", $"public, max-age={(int)cacheMaxAge.TotalSeconds}"); - } - }); + ServeUnknownFileTypes = true, + FileProvider = fileProvider, + RequestPath = fileProvider.RootPath, + OnPrepareResponse = ctx => + { + ctx.Context.Response.Headers.Append("Cache-Control", $"public, max-age={(int)cacheMaxAge.TotalSeconds}"); + } + }); + } + catch (DirectoryNotFoundException dEx) + { + Console.WriteLine(dEx.Message); + } + app.UseAuthentication(); From 536bb6e4851943814a4c7542b7d2964a9928f41f Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 19:19:58 +0100 Subject: [PATCH 22/38] remove unnecessary System.Text.Encodings.Web --- Common.Web/Common.Web.csproj | 1 - ML.AzureFunction/ML.AzureFunction.csproj | 2 +- PluginModuleBase/PluginModuleBase.csproj | 1 - ProblemSource/OldDb/OldDb.csproj | 2 +- ProblemSource/OldDbAdapter/OldDbAdapter.csproj | 2 +- .../ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj | 2 +- ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj | 2 +- .../ProblemSourceTestClient/ProblemSourceTestClient.csproj | 2 +- ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj | 2 +- ProblemSource/TrainingApi/ApiKeyAuthentication.cs | 1 - ProblemSource/WebJob/WebJob.csproj | 2 +- Tools/Tools.csproj | 2 +- 12 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Common.Web/Common.Web.csproj b/Common.Web/Common.Web.csproj index fedf74b..4f0b06a 100644 --- a/Common.Web/Common.Web.csproj +++ b/Common.Web/Common.Web.csproj @@ -10,7 +10,6 @@ - diff --git a/ML.AzureFunction/ML.AzureFunction.csproj b/ML.AzureFunction/ML.AzureFunction.csproj index 287e855..32549c5 100644 --- a/ML.AzureFunction/ML.AzureFunction.csproj +++ b/ML.AzureFunction/ML.AzureFunction.csproj @@ -20,7 +20,7 @@ - + diff --git a/PluginModuleBase/PluginModuleBase.csproj b/PluginModuleBase/PluginModuleBase.csproj index 4d064d2..4a238f7 100644 --- a/PluginModuleBase/PluginModuleBase.csproj +++ b/PluginModuleBase/PluginModuleBase.csproj @@ -8,6 +8,5 @@ - \ No newline at end of file diff --git a/ProblemSource/OldDb/OldDb.csproj b/ProblemSource/OldDb/OldDb.csproj index b2e5bb8..c53c80c 100644 --- a/ProblemSource/OldDb/OldDb.csproj +++ b/ProblemSource/OldDb/OldDb.csproj @@ -18,7 +18,7 @@ - + diff --git a/ProblemSource/OldDbAdapter/OldDbAdapter.csproj b/ProblemSource/OldDbAdapter/OldDbAdapter.csproj index 994c09e..3a12240 100644 --- a/ProblemSource/OldDbAdapter/OldDbAdapter.csproj +++ b/ProblemSource/OldDbAdapter/OldDbAdapter.csproj @@ -7,7 +7,7 @@ - + diff --git a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj index 0f5e857..a8e2482 100644 --- a/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj +++ b/ProblemSource/ProblemSourceModule.Tests/ProblemSourceModule.Tests.csproj @@ -24,7 +24,7 @@ - + diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj index 9f7de4b..2300fb1 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.csproj @@ -70,7 +70,7 @@ - + diff --git a/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj b/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj index 16b5e34..3c6cd0d 100644 --- a/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj +++ b/ProblemSource/ProblemSourceTestClient/ProblemSourceTestClient.csproj @@ -8,7 +8,7 @@ - + diff --git a/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj b/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj index 5e1ff14..40b6da9 100644 --- a/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj +++ b/ProblemSource/TrainingApi.Tests/TrainingApi.Tests.csproj @@ -14,7 +14,7 @@ - + diff --git a/ProblemSource/TrainingApi/ApiKeyAuthentication.cs b/ProblemSource/TrainingApi/ApiKeyAuthentication.cs index 6883022..2f43273 100644 --- a/ProblemSource/TrainingApi/ApiKeyAuthentication.cs +++ b/ProblemSource/TrainingApi/ApiKeyAuthentication.cs @@ -1,6 +1,5 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.Extensions.Options; -using System.Security.Claims; using System.Text.Encodings.Web; using TrainingApi.Services; diff --git a/ProblemSource/WebJob/WebJob.csproj b/ProblemSource/WebJob/WebJob.csproj index 4e69a55..82f1495 100644 --- a/ProblemSource/WebJob/WebJob.csproj +++ b/ProblemSource/WebJob/WebJob.csproj @@ -17,7 +17,7 @@ - + diff --git a/Tools/Tools.csproj b/Tools/Tools.csproj index 7f73019..39084b8 100644 --- a/Tools/Tools.csproj +++ b/Tools/Tools.csproj @@ -25,7 +25,7 @@ - + From 09e665542aed7ee726ded5e864232e4401ae2dd1 Mon Sep 17 00:00:00 2001 From: JWMB Date: Sat, 21 Feb 2026 19:20:05 +0100 Subject: [PATCH 23/38] ASPNETCORE_ENVIRONMENT: Development, allow fake login when Development --- ProblemSource/TrainingApi/Startup.cs | 6 +++--- compose.yaml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index 0ceb0f9..0b253ad 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -156,7 +156,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) } catch (DirectoryNotFoundException dEx) { - Console.WriteLine(dEx.Message); + Console.WriteLine($"Static files folder not found: {dEx.Message}"); } @@ -166,8 +166,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { app.Use(async (context, next) => { - if (System.Diagnostics.Debugger.IsAttached && context.User?.Claims.Any() == false) - { + if (context.User?.Claims.Any() == false) // System.Diagnostics.Debugger.IsAttached + { // For the lazy developer - swagger and test client are automatically authenticated var referer = context.Request.GetTypedHeaders().Referer; // when from swagger and localhost diff --git a/compose.yaml b/compose.yaml index 3b8029d..365971a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,8 @@ services: - "9090:80" depends_on: - mongodb + environment: + ASPNETCORE_ENVIRONMENT: Development mongodb: image: "docker.io/mongodb/mongodb-community-server" environment: From a222db8f3de180b32699864a3e3c280ae59d933a Mon Sep 17 00:00:00 2001 From: JWMB Date: Mon, 23 Feb 2026 08:43:10 +0100 Subject: [PATCH 24/38] working compose --- Common.Web/Extensions.cs | 16 ++++++++++++++++ Dockerfile.web | 10 +++++++--- .../ProblemSourceModule/ProblemSourceModule.cs | 2 ++ ProblemSource/TrainingApi/Startup.cs | 16 +++++++++++----- .../appsettings.Docker.Development.json | 14 ++++++++++++++ .../TrainingApi/appsettings.Docker.json | 14 ++++++++++++++ compose.yaml | 16 ++++++++++------ 7 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 Common.Web/Extensions.cs create mode 100644 ProblemSource/TrainingApi/appsettings.Docker.Development.json create mode 100644 ProblemSource/TrainingApi/appsettings.Docker.json diff --git a/Common.Web/Extensions.cs b/Common.Web/Extensions.cs new file mode 100644 index 0000000..b690fad --- /dev/null +++ b/Common.Web/Extensions.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.Hosting; + +namespace Common.Web +{ + public static class Extensions + { + public static bool HasEnvironmentPart(this IHostEnvironment hostEnvironment, string environmentPart) + { + ArgumentNullException.ThrowIfNull(hostEnvironment); + return hostEnvironment.EnvironmentName.Split('.') + .Any(part => string.Equals(environmentPart, part, StringComparison.OrdinalIgnoreCase)); + } + public static bool HasDevelopmentEnvironment(this IHostEnvironment hostEnvironment) + => hostEnvironment.HasEnvironmentPart(Environments.Development); + } +} diff --git a/Dockerfile.web b/Dockerfile.web index dfda06a..83a271a 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -5,7 +5,8 @@ WORKDIR /app # nothing copied? COPY ProblemSource/AdminApp/* / # works, but too much copied? -COPY ProblemSource/AdminApp/** / +COPY ProblemSource/AdminApp/** / + # COPY ./ProblemSource/AdminApp/** ./ # COPY ./ProblemSource/AdminApp/package.json ./ # COPY ./ProblemSource/AdminApp/src/** ./src/ @@ -16,11 +17,14 @@ COPY ProblemSource/AdminApp/** / RUN ls -ltR RUN npm install +# apk add curl COPY . . RUN npm run build -EXPOSE 5173 +EXPOSE 5171 + +CMD ["npm", "run", "dev", "--", "--host"] -CMD ["npm", "run", "dev", "--", "--host"] \ No newline at end of file +# podman build -t adminapp . -f Dockerfile.web \ No newline at end of file diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index dd153d9..9759713 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -104,6 +104,8 @@ public void ConfigureForMongoDb(IServiceCollection services, IConfiguration conf var connectionString = section["ConnectionString"]; // "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; var database = section["Database"]; //"_Training"; + Console.WriteLine($"connectionString={connectionString} database={database}"); + var client = new MongoClient(connectionString); services.AddSingleton(sp => client.GetDatabase(database)); diff --git a/ProblemSource/TrainingApi/Startup.cs b/ProblemSource/TrainingApi/Startup.cs index 0b253ad..7c8e9a2 100644 --- a/ProblemSource/TrainingApi/Startup.cs +++ b/ProblemSource/TrainingApi/Startup.cs @@ -2,7 +2,6 @@ using Common.Web.Services; using Microsoft.ApplicationInsights.AspNetCore.Extensions; using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.Extensions.FileProviders; @@ -108,7 +107,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) ServiceConfiguration.ConfigurePlugins(app, plugins); // Configure the HTTP request pipeline. - if (env.IsDevelopment()) + if (env.HasDevelopmentEnvironment()) { //app.UseSwagger(); //app.UseSwaggerUI(); @@ -129,7 +128,11 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) Secure = CookieSecurePolicy.Always }); - app.UseHttpsRedirection(); + Console.WriteLine($"EnvironmentName={env.EnvironmentName}"); + Console.WriteLine($"HasDevelopmentEnvironment={env.HasDevelopmentEnvironment()}"); + + if (!env.HasEnvironmentPart("Docker")) + app.UseHttpsRedirection(); if (app is WebApplication webApp) { @@ -162,7 +165,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseAuthentication(); - if (env.IsDevelopment()) + if (env.HasDevelopmentEnvironment()) { app.Use(async (context, next) => { @@ -172,7 +175,10 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) var referer = context.Request.GetTypedHeaders().Referer; // when from swagger and localhost var autologin = referer?.AbsolutePath.Contains("/swagger/") == true - || referer?.AbsoluteUri.StartsWith("http://localhost:") == true; + || referer?.AbsoluteUri.StartsWith("http://localhost:") == true + || referer == null; + + Console.WriteLine($"referer={referer}, autologin={autologin}"); if (!autologin) { diff --git a/ProblemSource/TrainingApi/appsettings.Docker.Development.json b/ProblemSource/TrainingApi/appsettings.Docker.Development.json new file mode 100644 index 0000000..195b289 --- /dev/null +++ b/ProblemSource/TrainingApi/appsettings.Docker.Development.json @@ -0,0 +1,14 @@ +{ + "AppSettings": { + "Storage": { + "Type": "MongoDB", + "MongoDB": { + "ConnectionString": "mongodb://mongo:27017/?maxPoolSize=500&waitQueueSize=2500", + "Database": "_Training" + } + } + }, + "CorsOrigins": "http://localhost:5171,https://localhost:5171,http://localhost:8080,http://localhost:8081", + "Cookies:SameSite": "", + "AllowedHosts": "*" +} \ No newline at end of file diff --git a/ProblemSource/TrainingApi/appsettings.Docker.json b/ProblemSource/TrainingApi/appsettings.Docker.json new file mode 100644 index 0000000..195b289 --- /dev/null +++ b/ProblemSource/TrainingApi/appsettings.Docker.json @@ -0,0 +1,14 @@ +{ + "AppSettings": { + "Storage": { + "Type": "MongoDB", + "MongoDB": { + "ConnectionString": "mongodb://mongo:27017/?maxPoolSize=500&waitQueueSize=2500", + "Database": "_Training" + } + } + }, + "CorsOrigins": "http://localhost:5171,https://localhost:5171,http://localhost:8080,http://localhost:8081", + "Cookies:SameSite": "", + "AllowedHosts": "*" +} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 365971a..7793e4f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,14 +5,18 @@ services: ports: - "9090:80" depends_on: - - mongodb + - mongo environment: - ASPNETCORE_ENVIRONMENT: Development - mongodb: + ASPNETCORE_ENVIRONMENT: Docker.Development + AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/?maxPoolSize=500&waitQueueSize=2500 + # AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/?directConnection=true&serverSelectionTimeoutMS=2000 + # AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/app_development + # mongodb://:27017 + mongo: image: "docker.io/mongodb/mongodb-community-server" - environment: - - MONGO_INITDB_ROOT_USERNAME=user - - MONGO_INITDB_ROOT_PASSWORD=pass + # environment: + # - MONGO_INITDB_ROOT_USERNAME=user + # - MONGO_INITDB_ROOT_PASSWORD=pass volumes: - db-data:/etc/data # - type: bind From 958b64a46c94bf48273bc9a0c47d1fb7369d8b22 Mon Sep 17 00:00:00 2001 From: JWMB Date: Tue, 24 Feb 2026 12:51:24 +0100 Subject: [PATCH 25/38] svelte app attempts --- Dockerfile.web | 7 ++++++- ProblemSource/AdminApp/svelte.config.js | 4 +++- ProblemSource/AdminApp/vite.config.ts | 9 +++++++-- ProblemSource/ProblemSourceModule/ProblemSourceModule.cs | 1 + compose.yaml | 9 +++++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Dockerfile.web b/Dockerfile.web index 83a271a..43a1bc4 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -17,6 +17,7 @@ COPY ProblemSource/AdminApp/** / RUN ls -ltR RUN npm install +# RUN npm install -g http-server # apk add curl COPY . . @@ -25,6 +26,10 @@ RUN npm run build EXPOSE 5171 -CMD ["npm", "run", "dev", "--", "--host"] +# https://stackoverflow.com/questions/61106423/how-to-put-a-svelte-app-in-a-docker-container +# CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] + +CMD ["npm", "run", "start"] +# CMD ["npm", "run", "dev", "--", "--host"] # podman build -t adminapp . -f Dockerfile.web \ No newline at end of file diff --git a/ProblemSource/AdminApp/svelte.config.js b/ProblemSource/AdminApp/svelte.config.js index 4f121c4..3baac99 100644 --- a/ProblemSource/AdminApp/svelte.config.js +++ b/ProblemSource/AdminApp/svelte.config.js @@ -9,6 +9,8 @@ const mdsvexOptions = { extensions: ['.md'] } +const basePath = true ? undefined : "/admin"; + /** @type {import('@sveltejs/kit').Config} */ const config = { // Consult https://github.com/sveltejs/svelte-preprocess @@ -22,7 +24,7 @@ const config = { adapter: adapter({ fallback: 'index.html' }), prerender: { entries: ['/help/en', '/help/en/first'] }, paths: { - base: "/admin", + base: basePath, // Not working: // https://github.com/sveltejs/kit/issues/2958 // https://github.com/sveltejs/kit/pull/7543 diff --git a/ProblemSource/AdminApp/vite.config.ts b/ProblemSource/AdminApp/vite.config.ts index 52ae729..4e94612 100644 --- a/ProblemSource/AdminApp/vite.config.ts +++ b/ProblemSource/AdminApp/vite.config.ts @@ -5,13 +5,17 @@ import path from 'path'; // TODO: import.meta does not contain 'env' (import.meta.env.VITE_HTTPS does not work) // TODO: process.env does not contain any variables from .env files (process.env.VITE_HTTPS does not work) -const useHttps = process.env.COMPUTERNAME !== "CND1387M7P"; +let useHttps = process.env.COMPUTERNAME !== "CND1387M7P"; +if (process.env.HTTPS === "false") useHttps = false; +const port = process.env.PORT ? parseInt(process.env.PORT) : 5171; +// const base = true ? "./" : undefined; + const config: UserConfig = { // plugins: [ sveltekit(), ].concat(useHttps ? [new Promise(res => res([basicSsl()]))] : []), server: { - port: 5171, + port: port, https: useHttps, // proxy: { // '/api': { @@ -22,6 +26,7 @@ const config: UserConfig = { // // } // } }, + // base: base, resolve: { alias: { src: path.resolve('./src') } } }; diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 9759713..65210ea 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -72,6 +72,7 @@ public void ConfigureServices(IServiceCollection services, IConfiguration config if (config != null) storageIsMongo = config["AppSettings:Storage:Type"] == "MongoDB"; } + if (storageIsMongo) ConfigureForMongoDb(services, config!); else diff --git a/compose.yaml b/compose.yaml index 7793e4f..5e3049c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,13 @@ services: + app: + image: "localhost/adminapp" + ports: + - "9091:8080" + depends_on: + - api + environment: + PUBLIC_LOCAL_SERVER_PATH: https://localhost:80 + VITE_HTTPS: false api: image: "localhost/trainingapi" # image: "localhost/complimentgeneratorapi" From 2606c9f47d3fe8b4a180151b49c2d40376e5c596 Mon Sep 17 00:00:00 2001 From: JWMB Date: Wed, 25 Feb 2026 22:04:53 +0100 Subject: [PATCH 26/38] http://api et.al --- Dockerfile.web | 17 +++++++++++++---- ProblemSource/AdminApp/vite.config.ts | 1 + compose.yaml | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Dockerfile.web b/Dockerfile.web index 43a1bc4..8d7981e 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,5 +1,8 @@ FROM node:24-alpine +# podman build -t adminapp . -f Dockerfile.web +# podman start adminapp -p 5171:5171 --name adminapp-1 + # WORKDIR ProblemSource/AdminApp WORKDIR /app @@ -17,19 +20,25 @@ COPY ProblemSource/AdminApp/** / RUN ls -ltR RUN npm install -# RUN npm install -g http-server +RUN npm install -g http-server # apk add curl COPY . . -RUN npm run build - EXPOSE 5171 +ENV PORT=5171 +# ENV PUBLIC_LOCAL_SERVER_PATH=https://kistudysync.azurewebsites.net + +# EXPOSE 80 +# ENV PORT=80 + +RUN npm run build # https://stackoverflow.com/questions/61106423/how-to-put-a-svelte-app-in-a-docker-container # CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] +CMD [ "http-server", "ProblemSource/AdminApp/build" ] -CMD ["npm", "run", "start"] +# CMD ["npm", "run", "start"] # CMD ["npm", "run", "dev", "--", "--host"] # podman build -t adminapp . -f Dockerfile.web \ No newline at end of file diff --git a/ProblemSource/AdminApp/vite.config.ts b/ProblemSource/AdminApp/vite.config.ts index 4e94612..bdf8661 100644 --- a/ProblemSource/AdminApp/vite.config.ts +++ b/ProblemSource/AdminApp/vite.config.ts @@ -7,6 +7,7 @@ import path from 'path'; // TODO: process.env does not contain any variables from .env files (process.env.VITE_HTTPS does not work) let useHttps = process.env.COMPUTERNAME !== "CND1387M7P"; if (process.env.HTTPS === "false") useHttps = false; +// console.log("useHttps", useHttps, process.env); const port = process.env.PORT ? parseInt(process.env.PORT) : 5171; // const base = true ? "./" : undefined; diff --git a/compose.yaml b/compose.yaml index 5e3049c..868718d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,7 +6,8 @@ services: depends_on: - api environment: - PUBLIC_LOCAL_SERVER_PATH: https://localhost:80 + PUBLIC_LOCAL_SERVER_PATH: http://api:9090/ + # https://localhost:80 VITE_HTTPS: false api: image: "localhost/trainingapi" From 35e93bc0fc3d6b7e9d381c80e180d6d6a4dfb763 Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 09:26:00 +0100 Subject: [PATCH 27/38] .dockerignore - add build (otherwise vite uses old files?!) --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 6e9e936..3b85058 100644 --- a/.dockerignore +++ b/.dockerignore @@ -332,3 +332,6 @@ # performance testing sandbox **/sandbox + +# Vite +**/build/ \ No newline at end of file From 1f5d7f1cd74cdd71d87450d382602e9328489d0d Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 09:26:28 +0100 Subject: [PATCH 28/38] adminapp logging --- ProblemSource/AdminApp/src/apiFacade.ts | 3 ++- ProblemSource/AdminApp/src/globalStore.ts | 4 ++-- ProblemSource/AdminApp/src/startup.ts | 1 + ProblemSource/AdminApp/vite.config.ts | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ProblemSource/AdminApp/src/apiFacade.ts b/ProblemSource/AdminApp/src/apiFacade.ts index 8d47881..440e04b 100644 --- a/ProblemSource/AdminApp/src/apiFacade.ts +++ b/ProblemSource/AdminApp/src/apiFacade.ts @@ -10,7 +10,8 @@ export class ApiFacade { impersonateUser: string | null = null; constructor(baseUrl: string) { - // console.log("baseUrl", baseUrl); + console.log("api baseUrl", baseUrl); + // baseUrl = "http://api:9090/"; const http = { fetch: (r: Request, init?: RequestInit) => { init = init || {}; diff --git a/ProblemSource/AdminApp/src/globalStore.ts b/ProblemSource/AdminApp/src/globalStore.ts index 6eb07be..e56582b 100644 --- a/ProblemSource/AdminApp/src/globalStore.ts +++ b/ProblemSource/AdminApp/src/globalStore.ts @@ -4,7 +4,7 @@ import type { LoginCredentials } from './apiClient'; import { ApiFacade } from './apiFacade'; import type { CurrentUserInfo } from './currentUserInfo'; import { Assistant } from './services/assistant'; -import { resolveLocalServerBaseUrl, Startup } from './startup'; +import { resolveLocalServerBaseUrl } from './startup'; import { SeverityLevel, type NotificationItem } from './types'; import type { TrainingUpdateMessage } from './types.js'; import { Realtime } from './services/realtime'; @@ -62,7 +62,7 @@ export const realtimeTrainingListener = (() => { signal.trigger(null); try { - await realtime.connect(Startup.resolveLocalServerBaseUrl(window.location)); + await realtime.connect(resolveLocalServerBaseUrl(window.location)); } catch (err) { console.error('error connecting', err); } diff --git a/ProblemSource/AdminApp/src/startup.ts b/ProblemSource/AdminApp/src/startup.ts index 18e685e..829d15a 100644 --- a/ProblemSource/AdminApp/src/startup.ts +++ b/ProblemSource/AdminApp/src/startup.ts @@ -23,5 +23,6 @@ export class Startup { } export function resolveLocalServerBaseUrl(location: Location) { + console.log("PUBLIC_LOCAL_SERVER_PATH", PUBLIC_LOCAL_SERVER_PATH); return PUBLIC_LOCAL_SERVER_PATH || location.origin; } diff --git a/ProblemSource/AdminApp/vite.config.ts b/ProblemSource/AdminApp/vite.config.ts index bdf8661..9c5ad4a 100644 --- a/ProblemSource/AdminApp/vite.config.ts +++ b/ProblemSource/AdminApp/vite.config.ts @@ -28,7 +28,8 @@ const config: UserConfig = { // // } }, // base: base, - resolve: { alias: { src: path.resolve('./src') } } + resolve: { alias: { src: path.resolve('./src') } }, + // build: { rollupOptions: { cache: false} } }; export default config; From bc75c7519082679a67dfdbf4a5c34171c3ee5b4b Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 09:27:05 +0100 Subject: [PATCH 29/38] compose: adminapp port --- compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 868718d..527e68f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,8 @@ services: app: image: "localhost/adminapp" ports: - - "9091:8080" + # - "9091:8080" + - "9091:5171" depends_on: - api environment: From 198f9c4508797d767023b9d877cfbd64d0d1e4e3 Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 10:52:34 +0100 Subject: [PATCH 30/38] http-server --proxy --- Dockerfile.web | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.web b/Dockerfile.web index 8d7981e..0e9d8df 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -10,6 +10,8 @@ WORKDIR /app # works, but too much copied? COPY ProblemSource/AdminApp/** / +# RUN rm -rf ProblemSource/AdminApp/build/_app/ + # COPY ./ProblemSource/AdminApp/** ./ # COPY ./ProblemSource/AdminApp/package.json ./ # COPY ./ProblemSource/AdminApp/src/** ./src/ @@ -35,8 +37,8 @@ ENV PORT=5171 RUN npm run build # https://stackoverflow.com/questions/61106423/how-to-put-a-svelte-app-in-a-docker-container -# CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] -CMD [ "http-server", "ProblemSource/AdminApp/build" ] +CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] +# CMD [ "http-server", "ProblemSource/AdminApp/build" ] # CMD ["npm", "run", "start"] # CMD ["npm", "run", "dev", "--", "--host"] From 8307779148a8ed46113ad1c97063d4c073a335d3 Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 18:08:52 +0100 Subject: [PATCH 31/38] svelte: attempts to read runtime env --- ProblemSource/AdminApp/.env | 7 ++++++- ProblemSource/AdminApp/src/apiFacade.ts | 3 +-- ProblemSource/AdminApp/src/startup.ts | 4 +++- ProblemSource/AdminApp/vite.config.ts | 6 +++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ProblemSource/AdminApp/.env b/ProblemSource/AdminApp/.env index 556a9d7..29a9bc6 100644 --- a/ProblemSource/AdminApp/.env +++ b/ProblemSource/AdminApp/.env @@ -1,3 +1,8 @@ VITE_HTTPS=true # Public -PUBLIC_LOCAL_SERVER_PATH="https://kistudysync.azurewebsites.net" +PUBLIC_LOCAL_SERVER_PATH=http://localhost:9090 +#PUBLIC_LOCAL_SERVER_PATH="https://kistudysync.azurewebsites.net" +PUBLIC_HTTPS=true +PUBLIC_PORT=5171 +PUBLIC_VITE_TEST1=abc +VITE_PUBLIC_TEST1=cde \ No newline at end of file diff --git a/ProblemSource/AdminApp/src/apiFacade.ts b/ProblemSource/AdminApp/src/apiFacade.ts index 440e04b..ca80956 100644 --- a/ProblemSource/AdminApp/src/apiFacade.ts +++ b/ProblemSource/AdminApp/src/apiFacade.ts @@ -10,8 +10,7 @@ export class ApiFacade { impersonateUser: string | null = null; constructor(baseUrl: string) { - console.log("api baseUrl", baseUrl); - // baseUrl = "http://api:9090/"; + // console.log("api baseUrl", baseUrl); const http = { fetch: (r: Request, init?: RequestInit) => { init = init || {}; diff --git a/ProblemSource/AdminApp/src/startup.ts b/ProblemSource/AdminApp/src/startup.ts index 829d15a..35003be 100644 --- a/ProblemSource/AdminApp/src/startup.ts +++ b/ProblemSource/AdminApp/src/startup.ts @@ -1,6 +1,7 @@ import { goto } from "$app/navigation"; import { base } from '$app/paths'; import { PUBLIC_LOCAL_SERVER_PATH } from '$env/static/public' +import { env as envDyn } from '$env/dynamic/public' import { ErrorHandling } from "./errorHandling"; export class Startup { @@ -24,5 +25,6 @@ export class Startup { export function resolveLocalServerBaseUrl(location: Location) { console.log("PUBLIC_LOCAL_SERVER_PATH", PUBLIC_LOCAL_SERVER_PATH); - return PUBLIC_LOCAL_SERVER_PATH || location.origin; + console.log("dyn", envDyn, "meta", import.meta.env); + return envDyn.PUBLIC_LOCAL_SERVER_PATH || PUBLIC_LOCAL_SERVER_PATH || location.origin; } diff --git a/ProblemSource/AdminApp/vite.config.ts b/ProblemSource/AdminApp/vite.config.ts index 9c5ad4a..8198928 100644 --- a/ProblemSource/AdminApp/vite.config.ts +++ b/ProblemSource/AdminApp/vite.config.ts @@ -2,14 +2,17 @@ import { sveltekit } from '@sveltejs/kit/vite'; import basicSsl from '@vitejs/plugin-basic-ssl' import type { UserConfig } from 'vite'; import path from 'path'; +// import { env as envDynPriv } from '$env/dynamic/private' +// import { env as envDynPub } from '$env/dynamic/public' // TODO: import.meta does not contain 'env' (import.meta.env.VITE_HTTPS does not work) // TODO: process.env does not contain any variables from .env files (process.env.VITE_HTTPS does not work) let useHttps = process.env.COMPUTERNAME !== "CND1387M7P"; if (process.env.HTTPS === "false") useHttps = false; -// console.log("useHttps", useHttps, process.env); +console.log("useHttps", useHttps, process.env); const port = process.env.PORT ? parseInt(process.env.PORT) : 5171; // const base = true ? "./" : undefined; +// envDynPub.PUBLIC_LOCAL_SERVER_PATH = envDynPriv["PUBLIC_LOCAL_SERVER_PATH"] || envDynPub.PUBLIC_LOCAL_SERVER_PATH; const config: UserConfig = { // plugins: [ @@ -29,6 +32,7 @@ const config: UserConfig = { // }, // base: base, resolve: { alias: { src: path.resolve('./src') } }, + envPrefix: "PUBLIC" // build: { rollupOptions: { cache: false} } }; From 2bab64ad60d47a02ac8235caa66b8e4784c72af8 Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 18:09:37 +0100 Subject: [PATCH 32/38] trainingapi environment=Docker --- Dockerfile | 3 ++- ProblemSource/TrainingApi/appsettings.Docker.json | 2 +- compose.yaml | 11 ++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index a2094c7..251ef8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ # Build the runtime image +# podman build -t trainingapi . -f Dockerfile FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base WORKDIR /app # Expose the port the app runs on @@ -33,7 +34,7 @@ WORKDIR /app COPY --from=publish /app/publish . ENV ASPNETCORE_HTTP_PORTS=80 # Define the startup command -ENTRYPOINT ["dotnet", "TrainingApi.dll"] +ENTRYPOINT ["dotnet", "TrainingApi.dll", "--environment=Docker"] # FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build diff --git a/ProblemSource/TrainingApi/appsettings.Docker.json b/ProblemSource/TrainingApi/appsettings.Docker.json index 195b289..b34c473 100644 --- a/ProblemSource/TrainingApi/appsettings.Docker.json +++ b/ProblemSource/TrainingApi/appsettings.Docker.json @@ -8,7 +8,7 @@ } } }, - "CorsOrigins": "http://localhost:5171,https://localhost:5171,http://localhost:8080,http://localhost:8081", + "CorsOrigins": "http://localhost:9091,http://localhost:5171,https://localhost:5171,http://localhost:8080,http://localhost:8081", "Cookies:SameSite": "", "AllowedHosts": "*" } \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 527e68f..63b06da 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,9 +7,14 @@ services: depends_on: - api environment: - PUBLIC_LOCAL_SERVER_PATH: http://api:9090/ + # none of these work :( - hard-coded in source for now... + PUBLIC_LOCAL_SERVER_PATH: http://api:9090 # https://localhost:80 VITE_HTTPS: false + PUBLIC_HTTPS: true + PUBLIC_VITE_TEST1: abc + VITE_TEST1: abce + VITE_PUBLIC_TEST1: cde api: image: "localhost/trainingapi" # image: "localhost/complimentgeneratorapi" @@ -26,8 +31,8 @@ services: mongo: image: "docker.io/mongodb/mongodb-community-server" # environment: - # - MONGO_INITDB_ROOT_USERNAME=user - # - MONGO_INITDB_ROOT_PASSWORD=pass + # - MONGODB_INITDB_ROOT_USERNAME_FILE=user + # - MONGODB_INITDB_ROOT_PASSWORD_FILE=pass volumes: - db-data:/etc/data # - type: bind From f809f100d46ec37333d201b1fa9be6c3e54cf7ed Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 18:09:55 +0100 Subject: [PATCH 33/38] better Dockerfile.web --- .dockerignore | 2 +- Dockerfile.web | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3b85058..3dc1769 100644 --- a/.dockerignore +++ b/.dockerignore @@ -334,4 +334,4 @@ **/sandbox # Vite -**/build/ \ No newline at end of file +# **/build/ huh, also ignored when building inside dockerfile, thought it was only during COPY... \ No newline at end of file diff --git a/Dockerfile.web b/Dockerfile.web index 0e9d8df..bebce75 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -8,9 +8,10 @@ WORKDIR /app # nothing copied? COPY ProblemSource/AdminApp/* / # works, but too much copied? -COPY ProblemSource/AdminApp/** / +# COPY ProblemSource/AdminApp/** / +# RUN rm -rf ./ProblemSource/AdminApp/build/_app/ -# RUN rm -rf ProblemSource/AdminApp/build/_app/ +COPY ProblemSource/AdminApp/package.json . # COPY ./ProblemSource/AdminApp/** ./ # COPY ./ProblemSource/AdminApp/package.json ./ @@ -19,13 +20,23 @@ COPY ProblemSource/AdminApp/** / # RUN find -maxdepth 2 -ls # no output.. +RUN echo "First copy result" RUN ls -ltR RUN npm install RUN npm install -g http-server # apk add curl -COPY . . +# RUN rm -rf ./ProblemSource/AdminApp/build/_app/ +# COPY . . +# COPY ProblemSource/AdminApp/ . +# COPY ProblemSource/AdminApp/ ProblemSource/AdminApp/ +# RUN rm -rf ./ProblemSource/AdminApp/build/_app/ + +COPY ProblemSource/AdminApp/ . + +RUN echo "Second copy result" +RUN ls -ltR EXPOSE 5171 ENV PORT=5171 @@ -36,8 +47,11 @@ ENV PORT=5171 RUN npm run build + # https://stackoverflow.com/questions/61106423/how-to-put-a-svelte-app-in-a-docker-container -CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] +CMD [ "http-server", "build", "--proxy", "http://localhost:5171?"] +# CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171?"] +# CMD [ "http-server", "ProblemSource/AdminApp/build", "--proxy", "http://localhost:5171/index.html?"] # CMD [ "http-server", "ProblemSource/AdminApp/build" ] # CMD ["npm", "run", "start"] From 6538109ad459fd59cefca35d510e4ddf103712b5 Mon Sep 17 00:00:00 2001 From: JWMB Date: Thu, 26 Feb 2026 21:08:12 +0100 Subject: [PATCH 34/38] initial users from env variables --- Common.Web/TypedConfiguration.cs | 128 ++++++++++++------ .../ProblemSourceModule.cs | 45 ++++-- .../Services/Storage/MongoDb/MongoTools.cs | 2 + ProblemSource/TrainingApi/AppSettings.cs | 11 +- .../Services/ICurrentUserProvider.cs | 6 +- .../TrainingApi/appsettings.Development.json | 11 ++ .../TrainingApi/appsettings.Docker.json | 2 +- ProblemSource/TrainingApi/appsettings.json | 3 +- compose.yaml | 3 + 9 files changed, 152 insertions(+), 59 deletions(-) diff --git a/Common.Web/TypedConfiguration.cs b/Common.Web/TypedConfiguration.cs index 96a932b..3954851 100644 --- a/Common.Web/TypedConfiguration.cs +++ b/Common.Web/TypedConfiguration.cs @@ -21,55 +21,101 @@ public class TypedConfiguration // https://referbruv.com/blog/posts/working-with-options-pattern-in-aspnet-core-the-complete-guide var appSettings = new T(); - config.GetSection(sectionKey).Bind(appSettings); - services.AddSingleton(appSettings.GetType(), appSettings!); + //config.GetSection(sectionKey).Bind(appSettings); + //services.AddSingleton(appSettings.GetType(), appSettings!); - RecurseBind(appSettings, services, config); + //RecurseBind(appSettings, services, config); + XBind(appSettings, services, config.GetSection(typeof(T).Name)); + // https://kaylumah.nl/2021/11/29/validated-strongly-typed-ioptions.html + // If we want to inject IOptions instead of just Type, this is needed: https://stackoverflow.com/a/61157181 services.ConfigureOptions(instance) + //services.Configure(config.GetSection("AceKnowledge")); - // https://kaylumah.nl/2021/11/29/validated-strongly-typed-ioptions.html - // If we want to inject IOptions instead of just Type, this is needed: https://stackoverflow.com/a/61157181 services.ConfigureOptions(instance) - //services.Configure(config.GetSection("AceKnowledge")); - - return appSettings; + return appSettings; } - private static void RecurseBind(object appSettings, IServiceCollection services, IConfiguration config) + private static void XBind(object setting, IServiceCollection services, IConfiguration config) + { + config.Bind(setting); + Rec(setting); + void Rec(object s) + { + services.AddSingleton(s.GetType(), s); + var props = s.GetType() + .GetProperties(System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public); + foreach (var item in props) + { + if (item.PropertyType == typeof(string) || item.PropertyType.IsPrimitive) + { } + else if (item.PropertyType.IsAssignableTo(typeof(System.Collections.IEnumerable)) + && item.PropertyType.IsGenericType) + { } + else + { + var v = item.GetValue(s); + if (v != null) + Rec(v); + } + } + } + } + + private static void RecurseBind(object appSettings, IServiceCollection services, IConfiguration config) { - var props = appSettings.GetType() - .GetProperties(System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public) - .Where(o => !o.PropertyType.IsSealed); // TODO: (low) better check than IsSealed (also unit test) + var props = appSettings.GetType() + .GetProperties(System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.Public) + ; //.Where(o => !o.PropertyType.IsSealed); // TODO: (low) better check than IsSealed (also unit test) - foreach (var prop in props) - { - var instance = prop.GetValue(appSettings); - //config.GetSection(prop.Name).Bind(instance); - services.AddSingleton(instance!.GetType(), instance!); + foreach (var prop in props) + { + var instance = prop.GetValue(appSettings); + if (instance == null) + { + var isNullable = Nullable.GetUnderlyingType(prop.PropertyType) != null; + if (isNullable) + continue; + if (prop.GetCustomAttributes(typeof(System.Runtime.CompilerServices.NullableAttribute), true).Any()) + continue; + throw new Exception($"Null value for non-nullable property '{prop.Name}'"); + } + //config.GetSection(prop.Name).Bind(instance); - RecurseBind(instance, services, config); + if (instance is System.Collections.IList lst) + { + foreach (var item in lst) + { + services.AddSingleton(item.GetType(), item); + //RecurseBind(item, services, config); + } + } + else + { + services.AddSingleton(instance!.GetType(), instance!); + RecurseBind(instance, services, config); + } - //var asOptions = Microsoft.Extensions.Options.Options.Create(instance); - //services.ConfigureOptions(instance); + //var asOptions = Microsoft.Extensions.Options.Options.Create(instance); + //services.ConfigureOptions(instance); - // Execute validation (if available) - var validatorType = instance.GetType().Assembly.GetTypes() - .Where(t => - { - var validatorInterface = t.GetInterfaces().SingleOrDefault(o => - o.IsGenericType && o.GetGenericTypeDefinition() == typeof(Microsoft.Extensions.Options.IValidateOptions<>)); - return validatorInterface != null && validatorInterface.GenericTypeArguments.Single() == instance.GetType(); - } - ).FirstOrDefault(); - if (validatorType != null) - { - var validator = Activator.CreateInstance(validatorType); - var m = validatorType.GetMethod("Validate"); - var result = (Microsoft.Extensions.Options.ValidateOptionsResult?)m?.Invoke(validator, new object[] { "", instance }); - if (result!.Failed) - { - throw new Exception($"{validatorType.Name}: {result.FailureMessage}"); - } - } - } - } + // Execute validation (if available) + var validatorType = instance.GetType().Assembly.GetTypes() + .Where(t => + { + var validatorInterface = t.GetInterfaces().SingleOrDefault(o => + o.IsGenericType && o.GetGenericTypeDefinition() == typeof(Microsoft.Extensions.Options.IValidateOptions<>)); + return validatorInterface != null && validatorInterface.GenericTypeArguments.Single() == instance.GetType(); + } + ).FirstOrDefault(); + if (validatorType != null) + { + var validator = Activator.CreateInstance(validatorType); + var m = validatorType.GetMethod("Validate"); + var result = (Microsoft.Extensions.Options.ValidateOptionsResult?)m?.Invoke(validator, new object[] { "", instance }); + if (result!.Failed) + { + throw new Exception($"{validatorType.Name}: {result.FailureMessage}"); + } + } + } + } } } diff --git a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs index 65210ea..f0be7c6 100644 --- a/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs +++ b/ProblemSource/ProblemSourceModule/ProblemSourceModule.cs @@ -100,15 +100,18 @@ public void ConfigureForAzureTables(IServiceCollection services, bool useCaching public void ConfigureForMongoDb(IServiceCollection services, IConfiguration config) { - var section = config.GetSection("AppSettings:Storage:MongoDB"); - - var connectionString = section["ConnectionString"]; // "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; - var database = section["Database"]; //"_Training"; + var dbConfig = services.Select(o => o.ImplementationInstance).OfType().FirstOrDefault(); + if (dbConfig == null) + { + var section = config.GetSection("AppSettings:Storage:MongoDB"); + dbConfig = new MongoTools.MongoConfig(section["ConnectionString"]!, section["Database"]!); + // "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500"; + } - Console.WriteLine($"connectionString={connectionString} database={database}"); + Console.WriteLine($"connectionString={dbConfig.ConnectionString} database={dbConfig.Database}"); - var client = new MongoClient(connectionString); - services.AddSingleton(sp => client.GetDatabase(database)); + var client = new MongoClient(dbConfig.ConnectionString); + services.AddSingleton(sp => client.GetDatabase(dbConfig.Database)); // DocumentBase MongoDocumentWrapper @@ -176,6 +179,32 @@ public void Configure(IApplicationBuilder app, bool initAzureStorage) var queueEventDispatcher = serviceProvider.GetService() as AzureQueueEventDispatcher; queueEventDispatcher?.Init().Wait(); } - } + + var storageConfig = serviceProvider.GetService(); + if (storageConfig?.Users?.Any() == true) + { + var userRepo = serviceProvider.GetService(); + if (userRepo != null) + { + var users = userRepo.GetAll().Result; + if (!users.Any()) + { + foreach (var item in storageConfig.Users) + { + item.PasswordForHashing = item.HashedPassword; + userRepo.Upsert(item).Wait(); + } + } + } + } + } } + + public class StorageConfig + { + public string Type { get; set; } = ""; + public List Users { get; set; } = []; + public MongoTools.MongoConfig MongoDB { get; set; } = new("", ""); + } + } diff --git a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs index 2b4ceb2..c3c6f39 100644 --- a/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs +++ b/ProblemSource/ProblemSourceModule/Services/Storage/MongoDb/MongoTools.cs @@ -23,5 +23,7 @@ public static string GetCollectionName(Type type) } return type.Name; } + + public record MongoConfig(string ConnectionString, string Database); } } diff --git a/ProblemSource/TrainingApi/AppSettings.cs b/ProblemSource/TrainingApi/AppSettings.cs index e261be5..dbc02c7 100644 --- a/ProblemSource/TrainingApi/AppSettings.cs +++ b/ProblemSource/TrainingApi/AppSettings.cs @@ -1,4 +1,7 @@ -using ProblemSource.Services.Storage.AzureTables; +using ProblemSource; +using ProblemSource.Services.Storage.AzureTables; +using ProblemSourceModule.Models; +using ProblemSourceModule.Services.Storage.MongoDb; using TrainingApi.RealTime; namespace TrainingApi @@ -8,7 +11,9 @@ public class AppSettings public AzureTableConfig AzureTable { get; set; } = new(); public RealTimeConfig RealTime { get; set; } = new(); public string? SyncUrls { get; set; } = ""; - //public InMemoryApiKeyRepository.Config ApiKeyConfig { get; set; } = new([]); - //public IEnumerable ApiKeyUsers { get; set; } = new List(); + + public StorageConfig Storage { get; set; } = new(); + //public InMemoryApiKeyRepository.Config ApiKeyConfig { get; set; } = new([]); + //public IEnumerable ApiKeyUsers { get; set; } = new List(); } } diff --git a/ProblemSource/TrainingApi/Services/ICurrentUserProvider.cs b/ProblemSource/TrainingApi/Services/ICurrentUserProvider.cs index ac30cc2..2cfbaa9 100644 --- a/ProblemSource/TrainingApi/Services/ICurrentUserProvider.cs +++ b/ProblemSource/TrainingApi/Services/ICurrentUserProvider.cs @@ -1,10 +1,6 @@ -using Azure.Core; -using Microsoft.AspNetCore.Authentication.Cookies; -using Microsoft.AspNetCore.Connections.Features; -using Microsoft.Extensions.Primitives; +using Microsoft.AspNetCore.Authentication.Cookies; using ProblemSourceModule.Models; using ProblemSourceModule.Services.Storage; -using System.Collections.Specialized; using System.Security.Claims; namespace TrainingApi.Services diff --git a/ProblemSource/TrainingApi/appsettings.Development.json b/ProblemSource/TrainingApi/appsettings.Development.json index 0c208ae..71a0fb2 100644 --- a/ProblemSource/TrainingApi/appsettings.Development.json +++ b/ProblemSource/TrainingApi/appsettings.Development.json @@ -4,5 +4,16 @@ "Default": "Information", "Microsoft.AspNetCore": "Warning" } + }, + "AppSettings": { + "Storage": { + "Users": [ + { + "Email": "admin", + "HashedPassword": "admin", + "Role": "Admin" + } + ] + } } } diff --git a/ProblemSource/TrainingApi/appsettings.Docker.json b/ProblemSource/TrainingApi/appsettings.Docker.json index b34c473..ecbecd3 100644 --- a/ProblemSource/TrainingApi/appsettings.Docker.json +++ b/ProblemSource/TrainingApi/appsettings.Docker.json @@ -11,4 +11,4 @@ "CorsOrigins": "http://localhost:9091,http://localhost:5171,https://localhost:5171,http://localhost:8080,http://localhost:8081", "Cookies:SameSite": "", "AllowedHosts": "*" -} \ No newline at end of file +} diff --git a/ProblemSource/TrainingApi/appsettings.json b/ProblemSource/TrainingApi/appsettings.json index cd298d7..4789223 100644 --- a/ProblemSource/TrainingApi/appsettings.json +++ b/ProblemSource/TrainingApi/appsettings.json @@ -28,7 +28,8 @@ "MongoDB": { "ConnectionString": "mongodb://localhost:27017/?maxPoolSize=500&waitQueueSize=2500", "Database": "_Training" - } + }, + "Users": [] }, "RealTime": { "Enabled": false, diff --git a/compose.yaml b/compose.yaml index 63b06da..9d8131d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -24,6 +24,9 @@ services: - mongo environment: ASPNETCORE_ENVIRONMENT: Docker.Development + AppSettings:Storage:Users:0:Email: admin + AppSettings:Storage:Users:0:HashedPassword: admin + AppSettings:Storage:Users:0:Role: Admin AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/?maxPoolSize=500&waitQueueSize=2500 # AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/?directConnection=true&serverSelectionTimeoutMS=2000 # AppSettings:Storage:MongoDB:ConnectionString: mongodb://mongo:27017/app_development From b0aa11d485d4ac3119c7b14cdd65141b66c98692 Mon Sep 17 00:00:00 2001 From: JWMB Date: Fri, 27 Feb 2026 07:38:06 +0100 Subject: [PATCH 35/38] some svelte+firefox bug --- ProblemSource/AdminApp/src/routes/teacher/+page.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ProblemSource/AdminApp/src/routes/teacher/+page.svelte b/ProblemSource/AdminApp/src/routes/teacher/+page.svelte index 7a75401..fe7c00d 100644 --- a/ProblemSource/AdminApp/src/routes/teacher/+page.svelte +++ b/ProblemSource/AdminApp/src/routes/teacher/+page.svelte @@ -70,7 +70,10 @@ groups = Object.entries(groupsData).map((o) => ({ group: o[0], summaries: o[1] })); } + let lastClick = 0; async function onSelectGroup(groupId: string) { + if (Date.now() - lastClick < 50) return; + lastClick = Date.now(); detailedTrainingsData = await apiFacade.trainings.getSummaries(groupId); // RealtimelineTools.testData(detailedTrainingsData.map(o => o.id)).forEach(o => rtlTools.append(o));; getRealtimeData(); @@ -159,7 +162,7 @@ tabs={groups.map((g) => { return { id: g.group }; })} - on:selected={(e) => onSelectGroup(e.detail)} + on:selected={(e) => { console.log(e); onSelectGroup(e.detail); }} >