From 9e191836d4c48f97143f7de9062d7f6b0ec1aff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 08:46:06 +0200 Subject: [PATCH 1/6] Thread-safe access to _annotations in DBMDataRef using lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 51 +++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index 19c01d79..e55eea2d 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -25,6 +25,7 @@ Imports System.DateTime Imports System.Double Imports System.Math Imports System.Runtime.InteropServices +Imports System.Threading Imports OSIsoft.AF.Asset Imports OSIsoft.AF.Asset.AFAttributeTrait Imports OSIsoft.AF.Data @@ -89,6 +90,7 @@ Namespace Vitens.DynamicBandwidthMonitor Const pValueMinMax As Double = 0.9999 ' CI for Minimum and Maximum + Private _lock As New Object ' Object for exclusive lock on critical section. Private _annotations As New Dictionary(Of AFTime, Object) Private Shared _dbm As New DBM(New DBMLoggerAFTrace) @@ -279,20 +281,27 @@ Namespace Vitens.DynamicBandwidthMonitor If value IsNot Nothing Then ' Key - _annotations.Remove(value.Timestamp) ' Remove existing - value.Annotated = False + Monitor.Enter(_lock) ' Block + Try - If annotation IsNot Nothing Then ' Value + _annotations.Remove(value.Timestamp) ' Remove existing + value.Annotated = False - DBM.Logger.LogDebug( - "value.Timestamp " & - value.Timestamp.LocalTime.ToString("s") & "; " & - "annotation " & DirectCast(annotation, String), Attribute.GetPath) + If annotation IsNot Nothing Then ' Value - _annotations.Add(value.Timestamp, annotation) ' Add - value.Annotated = True + DBM.Logger.LogDebug( + "value.Timestamp " & + value.Timestamp.LocalTime.ToString("s") & "; " & + "annotation " & DirectCast(annotation, String), Attribute.GetPath) - End If + _annotations.Add(value.Timestamp, annotation) ' Add + value.Annotated = True + + End If + + Finally + Monitor.Exit(_lock) ' Unblock + End Try End If @@ -308,12 +317,22 @@ Namespace Vitens.DynamicBandwidthMonitor ' attributes" GetAnnotation = Nothing - If value IsNot Nothing AndAlso - _annotations.TryGetValue(value.Timestamp, GetAnnotation) Then - _annotations.Remove(value.Timestamp) ' Remove after get - Return GetAnnotation - Else - Return String.Empty ' Default + If value IsNot Nothing Then ' Key + + Monitor.Enter(_lock) ' Block + Try + + If _annotations.TryGetValue(value.Timestamp, GetAnnotation) Then + _annotations.Remove(value.Timestamp) ' Remove after get + Return GetAnnotation + Else + Return String.Empty ' Default + End If + + Finally + Monitor.Exit(_lock) ' Unblock + End Try + End If End Function From d3acdf3590e58838481e95c565b9821c57518cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 09:19:50 +0200 Subject: [PATCH 2/6] Limit _annotations dictionary size to 10,000 by removing entries when exceeded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index e55eea2d..efe1983a 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -92,6 +92,7 @@ Namespace Vitens.DynamicBandwidthMonitor Private _lock As New Object ' Object for exclusive lock on critical section. Private _annotations As New Dictionary(Of AFTime, Object) + Private Shared _rand As New Random() Private Shared _dbm As New DBM(New DBMLoggerAFTrace) @@ -289,6 +290,10 @@ Namespace Vitens.DynamicBandwidthMonitor If annotation IsNot Nothing Then ' Value + While _annotations.Count >= 10000 ' Limit dictionary size + _annotations.Remove(_annotations.Keys.ElementAt(_rand.Next(_annotations.Count))) + End While + DBM.Logger.LogDebug( "value.Timestamp " & value.Timestamp.LocalTime.ToString("s") & "; " & From ba02335431d7fd61e63bd5ddac17f49c60a0a15a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 09:24:29 +0200 Subject: [PATCH 3/6] Refactor to use constant for maximum annotations size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index efe1983a..81ce2012 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -84,6 +84,7 @@ Namespace Vitens.DynamicBandwidthMonitor ' process output. + Const MaxAnnotationsSize As Integer = 10000 ' Maximum number of annotations Const StaleDataMinutes As Integer = 10 ' Minutes until snapshot is stale Const CategoryNoCorrelation As String = "NoCorrelation" Const pValueLoHi As Double = 0.95 ' Confidence interval for Lo and Hi @@ -290,7 +291,7 @@ Namespace Vitens.DynamicBandwidthMonitor If annotation IsNot Nothing Then ' Value - While _annotations.Count >= 10000 ' Limit dictionary size + While _annotations.Count >= MaxAnnotationsSize ' Limit dictionary size _annotations.Remove(_annotations.Keys.ElementAt(_rand.Next(_annotations.Count))) End While From 4be0a9c945ec2770c1c46f47b243e9bdfaab7e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 09:49:48 +0200 Subject: [PATCH 4/6] Refactor _annotations to use SortedDictionary for FIFO eviction instead of random removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index 81ce2012..2884d22d 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -92,8 +92,7 @@ Namespace Vitens.DynamicBandwidthMonitor Private _lock As New Object ' Object for exclusive lock on critical section. - Private _annotations As New Dictionary(Of AFTime, Object) - Private Shared _rand As New Random() + Private _annotations As New SortedDictionary(Of AFTime, Object) Private Shared _dbm As New DBM(New DBMLoggerAFTrace) @@ -291,8 +290,8 @@ Namespace Vitens.DynamicBandwidthMonitor If annotation IsNot Nothing Then ' Value - While _annotations.Count >= MaxAnnotationsSize ' Limit dictionary size - _annotations.Remove(_annotations.Keys.ElementAt(_rand.Next(_annotations.Count))) + While _annotations.Count >= MaxAnnotationsSize ' Limit dictionary size by removing oldest + _annotations.Remove(_annotations.Keys.First()) End While DBM.Logger.LogDebug( From 73f7643a885f40a665d49cf4e3f8a291c29c356c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 10:12:53 +0200 Subject: [PATCH 5/6] Limit line length correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- src/DBMDataRef/DBMDataRef.vb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index 2884d22d..9581ea52 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -290,8 +290,8 @@ Namespace Vitens.DynamicBandwidthMonitor If annotation IsNot Nothing Then ' Value - While _annotations.Count >= MaxAnnotationsSize ' Limit dictionary size by removing oldest - _annotations.Remove(_annotations.Keys.First()) + While _annotations.Count >= MaxAnnotationsSize ' Limit size + _annotations.Remove(_annotations.Keys.First()) ' Remove oldest End While DBM.Logger.LogDebug( From d4624523be5742a553968e76f55fd525b782f1c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Fiti=C3=A9?= Date: Wed, 27 Aug 2025 10:40:50 +0200 Subject: [PATCH 6/6] Update copyright year, change maximum number of annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johan Fitié --- README.md | 2 +- src/DBMDataRef/DBMDataRef.vb | 4 ++-- src/dbm/DBMManifest.vb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6effc52c..6099943b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # DBM Dynamic Bandwidth Monitor\ Leak detection method implemented in a real-time data historian\ -Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ## Description Water company Vitens has created a demonstration site called the Vitens Innovation Playground (VIP), in which new technologies and methodologies are developed, tested, and demonstrated. The projects conducted in the demonstration site can be categorized into one of four themes: energy optimization, real-time leak detection, online water quality monitoring, and customer interaction. In the real-time leak detection theme, a method for leak detection based on statistical demand forecasting was developed. diff --git a/src/DBMDataRef/DBMDataRef.vb b/src/DBMDataRef/DBMDataRef.vb index 9581ea52..d0751d91 100644 --- a/src/DBMDataRef/DBMDataRef.vb +++ b/src/DBMDataRef/DBMDataRef.vb @@ -1,6 +1,6 @@ ' Dynamic Bandwidth Monitor ' Leak detection method implemented in a real-time data historian -' Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +' Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ' ' This file is part of DBM. ' @@ -84,7 +84,7 @@ Namespace Vitens.DynamicBandwidthMonitor ' process output. - Const MaxAnnotationsSize As Integer = 10000 ' Maximum number of annotations + Const MaxAnnotationsSize As Integer = 1000 ' Maximum number of annotations Const StaleDataMinutes As Integer = 10 ' Minutes until snapshot is stale Const CategoryNoCorrelation As String = "NoCorrelation" Const pValueLoHi As Double = 0.95 ' Confidence interval for Lo and Hi diff --git a/src/dbm/DBMManifest.vb b/src/dbm/DBMManifest.vb index 25bb2939..e0223bae 100644 --- a/src/dbm/DBMManifest.vb +++ b/src/dbm/DBMManifest.vb @@ -1,6 +1,6 @@ ' Dynamic Bandwidth Monitor ' Leak detection method implemented in a real-time data historian -' Copyright (C) 2014-2024 J.H. Fitié, Vitens N.V. +' Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V. ' ' This file is part of DBM. ' @@ -30,6 +30,6 @@ "Leak detection method implemented in a real-time data historian")> + "Copyright (C) 2014-2025 J.H. Fitié, Vitens N.V.")>