From a436f91d5ece365617c0f1776a654c8581559a96 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Thu, 31 Jul 2025 12:05:52 +0200 Subject: [PATCH 01/33] Adding new tests in python for the new options added to the executable --- test_suite.py | 578 ++++++++++++++++++++++++++++++++++++++++ test_suite_functions.py | 558 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 1136 insertions(+) diff --git a/test_suite.py b/test_suite.py index 22afeafd..4ed094c5 100644 --- a/test_suite.py +++ b/test_suite.py @@ -187,6 +187,72 @@ 'the same order as sent\n' }, + 'Test_Reliability_5' : { + 'apps' : ['-P -t Square -r -k 0 -z 0 --num-instances 4', + '-S -t Square -r -k 0'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_reliability_no_losses_w_instances, + 'title' : 'Behavior of RELIABLE reliability with several instances', + 'description' : 'Verifies a RELIABLE publisher using 4 instances communicates with a ' + 'RELIABLE subscriber and samples are received ' + 'in order without any losses or duplicates\n\n' + ' * Configures the publisher and subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and subscriber with history KEEP_ALL\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' + 'receives subsequent samples per instance, without losses or duplicates, in ' + 'the same order as sent\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, + + 'Test_History_0' : { + 'apps' : ['-P -t Square -r -k 5 -z 0 --write-period 50', + '-S -t Square -r -k 5 --read-period 200'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_reliability_no_losses_w_instances, + 'title' : 'Behavior of KEEP_LAST history', + 'description' : 'Verifies a RELIABLE, KEEP_LAST 5 publisher using communicates with a ' + 'RELIABLE, KEEP_LAST 5 subscriber and samples are received in order without ' + 'any losses or duplicates. The subscriber reads data 4 time slower than' + 'the publisher publishes it, but the KEEP_LAST 5 history should be' + 'enough to avoid losses.\n\n' + ' * Configures the publisher and subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and subscriber with history KEEP_LAST 5\n' + ' * Configures the publisher with a writing period of 50ms\n' + ' * Configures the subscriber with a reading period of 200ms\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' + f'receives the next {tsf.MAX_SAMPLES_READ} subsequent samples, without losses or duplicates, in ' + 'the same order as sent.\n' + }, + + 'Test_History_1' : { + 'apps' : ['-P -t Square -r -k 5 -z 0 --write-period 50 --num-instances 4', + '-S -t Square -r -k 5 --read-period 200'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_reliability_no_losses_w_instances, + 'title' : 'Behavior of KEEP_LAST history with several instances', + 'description' : 'Verifies a RELIABLE, KEEP_LAST 5 publisher using 4 instances communicates with a ' + 'RELIABLE, KEEP_LAST 5 subscriber and samples are received in order without ' + 'any losses or duplicates. The subscriber reads data 4 time slower than' + 'the publisher publishes it, but the KEEP_LAST 5 history should be' + 'enough to avoid losses.\n\n' + ' * Configures the publisher and subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and subscriber with history KEEP_LAST 5\n' + ' * Configures the publisher with a writing period of 50ms\n' + ' * Configures the subscriber with a reading period of 200ms\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' + 'receives subsequent samples per instance, without losses or duplicates, in ' + 'the same order as sent\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, + # OWNERSHIP 'Test_Ownership_0' : { 'apps' : ['-P -t Square -s -1', '-S -t Square -s -1'], @@ -701,4 +767,516 @@ 'The test passes if the first sample the subscriber receives is the first sample ' 'that the publisher sent (by checking the "size" value is equal to 1).\n' }, + + 'Test_TimeBasedFilter_0' : { + 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100', '-S -t Square -r -k 0 --time-filter 1000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_reading_each_10_samples_w_instances, + 'title' : 'Test the behavior of the TIME_BASED_FILTER QoS', + 'description' : 'Verifies a subscriber with TIME_BASED_FILTER work as expected\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives a sample each 10 samples ' + 'the publisher application has sent\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/20}\n' + }, + + 'Test_TimeBasedFilter_1' : { + 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100 --num-instances 4', + '-S -t Square -r -k 0 --time-filter 1000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_reading_each_10_samples_w_instances, + 'title' : 'Test the behavior of the TIME_BASED_FILTER QoS with several instances', + 'description' : 'Verifies a subscriber with TIME_BASED_FILTER work as expected when the publisher ' + 'publishes several instances\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives a sample each 10 samples ' + 'the publisher application has sent per instance\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/20}\n' + }, + + 'Test_FinalInstanceState_0' : { + 'apps' : ['-P -t Square --num-iterations 200 --num-instances 4 --final-instance-state u', + '-S -t Square'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_unregistering_w_instances, + 'title' : 'Test the behavior of unregistering instances', + 'description' : 'Verifies a subscriber receives NOT_ALIVE_NO_WRITERS when the publisher unregister instances\n\n' + ' * Configures the publisher to unregister instances upon finalization\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher finishes after running 200 iterations\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives some samples and then ' + 'a NOT_ALIVE_NO_WRITERS per instance\n' + }, + + 'Test_FinalInstanceState_1' : { + 'apps' : ['-P -t Square --num-iterations 200 --num-instances 4 --final-instance-state d', + '-S -t Square'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_disposing_w_instances, + 'title' : 'Test the behavior of disposing instances', + 'description' : 'Verifies a subscriber receives NOT_ALIVE_DISPOSED when the publisher unregister instances\n\n' + ' * Configures the publisher to dispose instances upon finalization\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher finishes after running 200 iterations\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives some samples and then ' + 'a NOT_ALIVE_DISPOSED per instance\n' + }, + + 'Test_FinalInstanceState_2' : { + 'apps' : ['-P -t Square --num-iterations 200 --num-instances 4', + '-S -t Square'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_unregistering_w_instances, + 'title' : 'Test the instance state when finalizing the application', + 'description' : 'Verifies a subscriber receives NOT_ALIVE_NO_WRITERS when the publisher finishes ' + 'without unregistering or disposing instances\n\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher finishes after running 200 iterations\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives some samples and then ' + 'a NOT_ALIVE_NO_WRITERS per instance\n' + }, + + 'Test_LargeData_0' : { + 'apps' : ['-P -t Square --additional-payload-size 100000', + '-S -t Square'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_large_data, + 'title' : 'Test large data', + 'description' : 'This test covers the interoperability scenario with large data:\n\n' + ' * Configures the publisher to use 100000 additional payload size (to represent large data samples)\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The tests passes if the subscriber receives samples from the publisher and ' + 'the last byte sent in additional_payload_size is correctly set.\n' + }, + + 'Test_Lifespan_0' : { + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100 --lifespan 250', + '-S -t Square -r -k 0 --read-period 500'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_w_instances, + 'title' : 'Test the behavior of lifespan', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 250ms\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber with a reading period of 500ms\n' + ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_1' : { + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100 --lifespan 250 --num-instances 4', + '-S -t Square -r -k 0 --read-period 500'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_w_instances, + 'title' : 'Test the behavior of lifespan with several instances', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan and different instances\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 250ms\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber with a reading period of 500ms\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time per instance\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + + 'Test_OrderedAccess_0' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope i', + '-S -t Square -r -k 0 --ordered --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between INSTANCE_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher is compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_1' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope i', + '-S -t Square -r -k 0 --ordered --access-scope t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for ordered access between INSTANCE_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' + 'TOPIC_PRESENTATION subscriber when using ordered access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_OrderedAccess_2' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope i', + '-S -t Square -r -k 0 --ordered --access-scope g'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for ordered access between INSTANCE_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' + 'GROUP_PRESENTATION subscriber when using ordered access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_OrderedAccess_3' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t', + '-S -t Square -r -k 0 --ordered --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between TOPIC_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies a TOPIC_PRESENTATION publisher compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_4' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t', + '-S -t Square -r -k 0 --ordered --access-scope t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between TOPIC_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies a TOPIC_PRESENTATION publisher compatible with a ' + 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_5' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t', + '-S -t Square -r -k 0 --ordered --access-scope g'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for ordered access between TOPIC_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies an TOPIC_PRESENTATION publisher does not match with a ' + 'GROUP_PRESENTATION subscriber when using ordered access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_OrderedAccess_6' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope g', + '-S -t Square -r -k 0 --ordered --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_7' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope g', + '-S -t Square -r -k 0 --ordered --access-scope t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with a ' + 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_8' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope g', + '-S -t Square -r -k 0 --ordered --access-scope g'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with an ' + 'GROUP_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_OrderedAccess_9' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t --num-instances 4 --write-period 100', + '-S -t Square -r -k 0 --ordered --access-scope i --take-read --read-period 400', + '-S -t Square -r -k 0 --ordered --access-scope t --take-read --read-period 400'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.ORDERED_ACCESS_INSTANCE, ReturnCode.ORDERED_ACCESS_TOPIC], + 'check_function' : tsf.ordered_access_w_instances, + 'title' : 'Test the behavior of ordered access', + 'description' : 'Verifies subscribers receives data correctly depending on the ordered ' + 'access: TOPIC_PRESENTATION and INSTANCE_PRESENTATION.\n\n' + ' * Configures the publisher and all subscribers with a RELIABLE reliability\n' + ' * Configures the publisher and all subscribers with history KEEP_ALL\n' + ' * Configures the publisher and all subscribers with ordered access\n' + ' * Configures the publisher with access scope TOPIC_PRESENTATION\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the first subscriber with access scope INSTANCE_PRESENTATION\n' + ' * Configures the second subscriber with access scope TOPIC_PRESENTATION\n' + ' * Configures all subscribers with a reading period of 400ms\n' + ' * Configures all subscribers to use take() instead of take_next_instance()\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber applications are receiving the instances ' + 'with the specified ordered access / access scope. For INSTANCE_PRESENTATION, ' + 'all the samples for one instance is presented sequentially. For ' + 'TOPIC_PRESENTATION, the subscriber reads the samples in the same ' + 'order they have been published, independently of the instance they' + 'belong to (one sample of one instance each time)\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, + + 'Test_CoherentSets_0' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i', + '-S -t Square -r -k 0 --coherent --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between INSTANCE_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher is compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_CoherentSets_1' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i', + '-S -t Square -r -k 0 --coherent --access-scope t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for coherent access between INSTANCE_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' + 'TOPIC_PRESENTATION subscriber when using coherent access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_CoherentSets_2' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i', + '-S -t Square -r -k 0 --coherent --access-scope g'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for coherent access between INSTANCE_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' + 'GROUP_PRESENTATION subscriber when using coherent access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_CoherentSets_3' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t', + '-S -t Square -r -k 0 --coherent --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between TOPIC_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies an TOPIC_PRESENTATION publisher is compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_CoherentSets_4' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t', + '-S -t Square -r -k 0 --coherent --access-scope t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between TOPIC_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies an TOPIC_PRESENTATION publisher is compatible with an ' + 'TOPIC_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_CoherentSets_5' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t', + '-S -t Square -r -k 0 --coherent --access-scope g'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for coherent access between TOPIC_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies an TOPIC_PRESENTATION publisher does not match with a ' + 'GROUP_PRESENTATION subscriber when using coherent access and ' + 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + 'Test_CoherentSets_6' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g', + '-S -t Square -r -k 0 --coherent --access-scope i'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between GROUP_PRESENTATION publisher and ' + 'INSTANCE_PRESENTATION subscriber', + 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' + 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_CoherentSets_7' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g', + '-S -t Square -r -k 0 --coherent --access-scope t'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between GROUP_PRESENTATION publisher and ' + 'TOPIC_PRESENTATION subscriber', + 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' + 'TOPIC_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + 'Test_CoherentSets_8' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g', + '-S -t Square -r -k 0 --coherent --access-scope g'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'title' : 'Compatibility for coherent access between GROUP_PRESENTATION publisher and ' + 'GROUP_PRESENTATION subscriber', + 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' + 'GROUP_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' + }, + + 'Test_CoherentSets_9' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + '-S -t Square -r -k 0 --coherent --access-scope i --num-topics 3 --take-read --read-period 100'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.coherent_sets_w_instances, + 'title' : 'Test the behavior of coherent access with INSTANCE_PRESENTATION with several instances', + 'description' : 'Verifies subscribers receives data correctly when using coherent ' + 'access INSTANCE_PRESENTATION.\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher / subscriber with access scope INSTANCE_PRESENTATION\n' + ' * Configures the publisher / subscriber to use 3 topics\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher creates coherent sets of 3 consecutive samples each instance\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber to use take() instead of take_next_instance()\n' + ' * Configures the subscriber with a reading period of 100ms\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives all samples of a coherent ' + 'set at the same. This coherent set is created with 36 samples: 3 samples per ' + 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'per instance and topic is 3 each time we receive a new coherent set of samples\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, + 'Test_CoherentSets_10' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + '-S -t Square -r -k 0 --coherent --access-scope t --num-topics 3 --take-read --read-period 100'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.coherent_sets_w_instances, + 'title' : 'Test the behavior of coherent access with TOPIC_PRESENTATION with several instances', + 'description' : 'Verifies subscribers receives data correctly when using coherent ' + 'access TOPIC_PRESENTATION.\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher / subscriber with access scope TOPIC_PRESENTATION\n' + ' * Configures the publisher / subscriber to use 3 topics\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher creates coherent sets of 3 consecutive samples each instance\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber to use take() instead of take_next_instance()\n' + ' * Configures the subscriber with a reading period of 100ms\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives all samples of a coherent ' + 'set at the same. This coherent set is created with 36 samples: 3 samples per ' + 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'per instance and topic is 3 each time we receive a new coherent set of samples\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, + 'Test_CoherentSets_11' : { + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + '-S -t Square -r -k 0 --coherent --access-scope g --num-topics 3 --take-read --read-period 100'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.coherent_sets_w_instances, + 'title' : 'Test the behavior of coherent access with GROUP_PRESENTATION with several instances', + 'description' : 'Verifies subscribers receives data correctly when using coherent ' + 'access GROUP_PRESENTATION.\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with coherent access\n' + ' * Configures the publisher / subscriber with access scope GROUP_PRESENTATION\n' + ' * Configures the publisher / subscriber to use 3 topics\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher creates coherent sets of 3 consecutive samples each instance\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber to use take() instead of take_next_instance()\n' + ' * Configures the subscriber with a reading period of 100ms\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives all samples of a coherent ' + 'set at the same. This coherent set is created with 36 samples: 3 samples per ' + 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'per instance and topic is 3 each time we receive a new coherent set of samples\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + }, } diff --git a/test_suite_functions.py b/test_suite_functions.py index 81707d27..164348a8 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -353,6 +353,66 @@ def test_reliability_no_losses(child_sub, samples_sent, last_sample_saved, timeo print(f'Samples read: {samples_read}') return produced_code +def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests RELIABLE reliability, it checks whether the subscriber + receives the samples in order and with no losses (for several instances) + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern. + """ + + produced_code = ReturnCode.OK + + instance_color = [] + instance_seq_num = [] + first_iteration = [] + samples_read = 0 + + while samples_read < MAX_SAMPLES_READ: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + instance_seq_num.append(int(sub_string.group(2))) + first_iteration.append(True) + + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + if first_iteration[index]: + first_iteration[index] = False + else: + # check that the next sequence number is the next value + instance_seq_num[index] += 1 + if instance_seq_num[index] != int(sub_string.group(2)): + produced_code = ReturnCode.DATA_NOT_CORRECT + break + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + + # Get the next sample the subscriber is receiving + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + if index == 0: + samples_read += 1 + + print(f'Samples read: {samples_read}, instances: {instance_color}') + return produced_code + + def test_durability_volatile(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests the volatile durability, it checks that the sample the @@ -450,3 +510,501 @@ def test_deadline_missed(child_sub, samples_sent, last_sample_saved, timeout): return ReturnCode.OK else: return ReturnCode.DATA_NOT_RECEIVED + +def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests whether the subscriber application receives one sample + out of 10 (for each instance). For example, first sample received with size + 5, the next one should have size 15, then 25... + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + instance_color = [] + instance_seq_num = [] + first_iteration = [] + ignore_first_sample = [] + max_samples_received = MAX_SAMPLES_READ / 20 # 25 + samples_read = 0 + + while samples_read < max_samples_received: + + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + instance_seq_num.append(int(sub_string.group(2))) + first_iteration.append(True) + ignore_first_sample.append(True) + + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + if first_iteration[index]: + first_iteration[index] = False + else: + if ignore_first_sample[index]: + ignore_first_sample[index] = False + # check that the received sample has the sequence number + # previous + 10 + elif instance_seq_num[index] + 10 != int(sub_string.group(2)): + produced_code = ReturnCode.DATA_NOT_CORRECT + break + instance_seq_num[index] = int(sub_string.group(2)) + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + + # Get the next sample the subscriber is receiving + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + if index == 0: + samples_read += 1 + + print(f'Samples read: {samples_read}, instances: {instance_color}') + return produced_code + +def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests whether instances are correctly unregistered + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + instance_color = [] + unregistered_instance_color = [] + + while True: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + else: + # if no sample is received, it might be a UNREGISTER message + sub_string = re.search(r'\w+\s+(\w+)\s+NOT_ALIVE_NO_WRITERS_INSTANCE_STATE', + child_sub.before + child_sub.after) + if sub_string is not None: + unregistered_instance_color.append(sub_string.group(1)) + if len(instance_color) == len(unregistered_instance_color): + break + # Get the next sample the subscriber is receiving or unregister/dispose + index = child_sub.expect( + [ + r'\w+\s+\w+\s+.*?\n', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + + # compare that arrays contain the same elements + if set(instance_color) == set(unregistered_instance_color): + produced_code = ReturnCode.OK + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + + print(f'Unregistered {len(unregistered_instance_color)} elements: {unregistered_instance_color}') + + return produced_code + +def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests whether instances are correctly disposed + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + instance_color = [] + disposed_instance_color = [] + + while True: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + else: + # if no sample is received, it might be a DISPOSED message + sub_string = re.search(r'\w+\s+(\w+)\s+NOT_ALIVE_DISPOSED_INSTANCE_STATE', + child_sub.before + child_sub.after) + if sub_string is not None: + disposed_instance_color.append(sub_string.group(1)) + if len(instance_color) == len(disposed_instance_color): + break + # Get the next sample the subscriber is receiving or unregister/dispose + index = child_sub.expect( + [ + r'\w+\s+\w+\s+.*?\n', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + + # compare that arrays contain the same elements + if set(instance_color) == set(disposed_instance_color): + produced_code = ReturnCode.OK + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + + print(f'Disposed {len(disposed_instance_color)} elements: {disposed_instance_color}') + + return produced_code + +def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests whether large data is correctly received + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.DATA_NOT_CORRECT + # As the interoperability_test is just looking for the size [], + # this does not count the data after it, we need to read one more full + # sample + index = child_sub.expect( + [ + r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{[0-9]+\}', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + + if index == 0: + # Read the sample received, if it prints the additional_bytes == 255, + # it is sending large data correctly + sub_string = re.search(r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{([0-9]+)\}', + child_sub.before + child_sub.after) + # Check if the last element of the additional_bytes field element is + # received correctly + if sub_string is not None: + if int(sub_string.group(1)) == 255: + produced_code = ReturnCode.OK + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + + return produced_code + +def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests that lifespan works correctly. In the test situation, + only 2 or 3 consecutive samples should be received each time the reader + reads data. + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + # as the test is reading in a slower rate, reduce the number of samples read + max_samples_lifespan = MAX_SAMPLES_READ / 10 # 50 + + instance_color = [] + previous_seq_num = [] + first_iteration = [] + samples_read = 0 + consecutive_samples = [] + + while samples_read < max_samples_lifespan: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + previous_seq_num.append(int(sub_string.group(2))) + first_iteration.append(True) + consecutive_samples.append(1) # take into account the first sample + + # if the instance exists + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + # we should receive only 2 or 3 consecutive samples with the + # parameters defined by the test + if first_iteration[index]: + # do nothing for the first sample received + first_iteration[index] = False + else: + # if the sequence number is consecutive, increase the counter + if previous_seq_num[index] + 1 == int(sub_string.group(2)): + consecutive_samples[index] += 1 + else: + # if the sequence number is not consecutive, check that we + # receive only 3 or 2 samples + if consecutive_samples[index] == 3 or consecutive_samples[index] == 2: + # reset value to 1, as this test consider that the first + # sample is consecutive with itself + consecutive_samples[index] = 1 + else: + # if the amount of samples received is different than 3 or 2 + # this is an error + produced_code = ReturnCode.DATA_NOT_CORRECT + break + previous_seq_num[index] = int(sub_string.group(2)) + + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + + # Get the next sample the subscriber is receiving + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + if index == 0: + samples_read += 1 + + print(f'Samples read: {samples_read}, instances: {instance_color}') + return produced_code + +def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests that ordered access works correctly. This counts the + samples received in order and detects wether they are from the same instance + as the previously received sample or not. + If the number of consecutive samples from the same instance is greater than + the number of consecutive samples form different instances, this means that + the DW is using INSTANCE_PRESENTATION, if the case is the opposite, it is + using TOPIC_PRESENTATION. + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + instance_color = [] + samples_read = 0 + previous_sample_color = None + color_different_count = 0 + color_equal_count = 0 + samples_printed = False + + while samples_read < MAX_SAMPLES_READ: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', + child_sub.before + child_sub.after) + + # if a sample is read + if sub_string is not None: + # samples have been printed at least once + samples_printed = True + # add new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + + # the instance exists + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + current_color = sub_string.group(1) + # check the previous color and increase the different or equal + # counters + if previous_sample_color is not None: + if current_color != previous_sample_color: + color_different_count += 1 + else: + color_equal_count += 1 + previous_sample_color = current_color + # different message than a sample + else: + sub_string = re.search(r'Reading with ordered access', + child_sub.before + child_sub.after) + # if 'Reading with ordered access' message, it means that the DataReader + # is reading a new set of data (DataReader reads data slower that a + # DataWriter writes it) + if sub_string is not None: + # if samples have been already received by the DataReader and the + # counter addition (samples read) is greater than 5. It is 5 because + # there are 4 instances and we need to make sure that we receive + # at least 1 sample for every instance + # Note: color_equal_count + color_different_count will be the samples read + if samples_printed and color_equal_count + color_different_count > 5: + # if produced_code is not OK (this will happen in all iterations + # except for the first one). We check that the behavior is the same + # as in previous iterations. + if produced_code != ReturnCode.OK: + current_behavior = None + if color_equal_count > color_different_count: + current_behavior = ReturnCode.ORDERED_ACCESS_INSTANCE + elif color_equal_count < color_different_count: + current_behavior = ReturnCode.ORDERED_ACCESS_TOPIC + # in case of a behavior change, this will be an error + if produced_code != current_behavior: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + # this only happens on the first iteration and then this sets + # the initial ReturnCode + else: + if color_equal_count > color_different_count: + produced_code = ReturnCode.ORDERED_ACCESS_INSTANCE + elif color_equal_count < color_different_count: + produced_code = ReturnCode.ORDERED_ACCESS_TOPIC + # reset counters for the next set of samples read + color_equal_count = 0 + color_different_count = 0 + + # Get the next sample the subscriber is receiving or the next + # 'Reading with ordered access message' + index = child_sub.expect( + [ + r'(Reading with ordered access.*?\n|[\[[0-9]+\])', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + if index == 0: + samples_read += 1 + + print(f'Samples read: {samples_read}, instances: {instance_color}') + return produced_code + +def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests that coherent sets works correctly. This counts the + consecutive samples received from the same instance. The value should be 3 + as this is the coherent set count that the test is setting. + Note: when using GROUP_PRESENTATION, the first iteration may print more + samples (more coherent sets), the test check that the samples received per + instance is a multiple of 3, so the coherent sets are received complete. + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern + """ + + produced_code = ReturnCode.OK + + topics = {} + samples_read = 0 + previous_sample_color = None + new_coherent_set_read = False + first_time_reading = True + ignore_firsts_coherent_set = 2 + + while samples_read < MAX_SAMPLES_READ: + sub_string = re.search(r'(\w+)\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', + child_sub.before + child_sub.after) + + # if a sample is read + if sub_string is not None: + # DataReader has received a new coherent set + new_coherent_set_read = True + # add new instances to the corresponding topic + topic_name = sub_string.group(1) + instance_color = sub_string.group(2) + + if topic_name not in topics: + topics[topic_name] = {} + if instance_color not in topics[topic_name]: + topics[topic_name][instance_color] = 1 + # if the instance is already added + if instance_color in topics[topic_name]: + # the instance exists + current_color = instance_color + # check the previous color and increase consecutive samples + if previous_sample_color is not None: + if current_color == previous_sample_color: + topics[topic_name][instance_color] += 1 + previous_sample_color = current_color + # different message than a sample + else: + sub_string = re.search(r'Reading coherent sets', + child_sub.before + child_sub.after) + # if 'Reading coherent sets' message, it means that the DataReader + # is trying to read a new coherent set, it might not read any sample + if sub_string is not None: + # if DataReader has received samples + if ignore_firsts_coherent_set != 0 and new_coherent_set_read: + ignore_firsts_coherent_set -= 1 + for topic in topics: + for color in topics[topic]: + topics[topic][color] = 1 + elif new_coherent_set_read: + for topic in topics: + for color in topics[topic]: + if first_time_reading: + # with group presentation we may get several coherent + # sets at the beginning, just checking that the samples + # received are multiple of 3 + if topics[topic][color] % 3 != 0: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + else: + # there should be 3 consecutive samples per instance, + # as the test specifies this with the argument + # --coherent-sample-count 3 + if topics[topic][color] != 3: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + topics[topic][color] = 1 + if produced_code == ReturnCode.DATA_NOT_CORRECT: + break + first_time_reading = False + new_coherent_set_read = False + + # Get the next sample the subscriber is receiving or the next + # 'Reading with ordered access message' + index = child_sub.expect( + [ + r'(Reading coherent sets.*?\n|[\[[0-9]+\])', # index = 0 + pexpect.TIMEOUT # index = 1 + ], + timeout + ) + if index == 1: + # no more data to process + break + if index == 0: + samples_read += 1 + + print(f'Samples read: {samples_read}') + print("Instances:") + for topic in topics: + print(f"Topic {topic}: {', '.join(topics[topic].keys())}") + + return produced_code From 5fb6d4e6c816f7a782c818faa6b24cd97da0c368 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 13 Jan 2026 20:24:00 +0100 Subject: [PATCH 02/33] The basic check is that the sample has not 0 size --- rtps_test_utilities.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/rtps_test_utilities.py b/rtps_test_utilities.py index 7e9068ea..a6f219d6 100644 --- a/rtps_test_utilities.py +++ b/rtps_test_utilities.py @@ -55,4 +55,16 @@ def remove_ansi_colors(text): return cleaned_str def no_check(child_sub, samples_sent, last_sample_saved, timeout): + """ Only checks that the data is well formed and size is not zero.""" + sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is None: + return ReturnCode.DATA_NOT_RECEIVED + + sample_size = int(sub_string.group(1)) + + if sample_size == 0: + return ReturnCode.DATA_NOT_CORRECT + return ReturnCode.OK From e295d1a58492717edf02c67fd903a9849cca53b1 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 13 Jan 2026 20:25:16 +0100 Subject: [PATCH 03/33] Replaced test test_ownership_receivers that relies on samples_sent by test_size_receivers that relies on the size of the samples, less error prone --- test_suite.py | 26 +++--- test_suite_functions.py | 171 +++++++++------------------------------- 2 files changed, 52 insertions(+), 145 deletions(-) diff --git a/test_suite.py b/test_suite.py index 4ed094c5..90ae585b 100644 --- a/test_suite.py +++ b/test_suite.py @@ -296,17 +296,17 @@ # The DataReader should only receive samples from the DataWriter with higher # ownership. There may be the situation that the DataReader starts receiving # samples from one DataWriter until another DataWriter with higher ownership - # strength is created. This should be handled by test_ownership_receivers(). + # strength is created. This should be handled by test_size_receivers(). 'Test_Ownership_3': { 'apps' : [ - '-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s 4 -r -k 0 -c BLUE -w -z 30', + '-P -t Square -s 3 -r -k 0 -c BLUE -z 20', + '-P -t Square -s 4 -r -k 0 -c BLUE -z 30', '-S -t Square -s 1 -r -k 0'], 'expected_codes' :[ ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_ONE], - 'check_function' : tsf.test_ownership_receivers, + 'check_function' : tsf.test_size_receivers, 'title' : 'Behavior of EXCLUSIVE OWNERSHIP QoS with publishers of the same instance', 'description' : 'Verifies an exclusive ownership subscriber receives samples only from ' 'the highest ownership strength publisher of the same instance\n\n' @@ -332,14 +332,14 @@ # publish different instances and the ownership is applied per instance. 'Test_Ownership_4': { 'apps' :[ - '-P -t Square -s 3 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s 4 -r -k 0 -c RED -w -z 30', + '-P -t Square -s 3 -r -k 0 -c BLUE -z 20', + '-P -t Square -s 4 -r -k 0 -c RED -z 30', '-S -t Square -s 1 -r -k 0'], 'expected_codes' : [ ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_BOTH], - 'check_function' : tsf.test_ownership_receivers, + 'check_function' : tsf.test_size_receivers, 'title' : 'Behavior of EXCLUSIVE OWNERSHIP QoS with publishers with different instances', 'description' : 'Verifies an exclusive ownership subscriber receives samples from different ' 'publishers that publish different instances (ShapeType with different color)\n\n' @@ -361,14 +361,14 @@ # shared ownership. 'Test_Ownership_5': { 'apps' : [ - '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 30', + '-P -t Square -s -1 -r -k 0 -c BLUE -z 20', + '-P -t Square -s -1 -r -k 0 -c BLUE -z 30', '-S -t Square -s -1 -r -k 0'], 'expected_codes' :[ ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_BOTH], - 'check_function' : tsf.test_ownership_receivers, + 'check_function' : tsf.test_size_receivers, 'title' : 'Behavior of SHARED OWNERSHIP QoS with publishers with the same instance', 'description' : 'Verifies a shared ownership subscriber receives samples from all ' 'shared ownership publishers of the different instances\n\n' @@ -390,14 +390,14 @@ # shared ownership. 'Test_Ownership_6': { 'apps' : [ - '-P -t Square -s -1 -r -k 0 -c BLUE -w -z 20', - '-P -t Square -s -1 -r -k 0 -c RED -w -z 30', + '-P -t Square -s -1 -r -k 0 -c BLUE -z 20', + '-P -t Square -s -1 -r -k 0 -c RED -z 30', '-S -t Square -s -1 -r -k 0'], 'expected_codes' : [ ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_BOTH], - 'check_function' : tsf.test_ownership_receivers, + 'check_function' : tsf.test_size_receivers, 'title' : 'Behavior of SHARED OWNERSHIP QoS with different instances', 'description' : 'Verifies a shared ownership subscriber receives samples from all ' 'shared ownership publishers of different instances\n\n' diff --git a/test_suite_functions.py b/test_suite_functions.py index 164348a8..6df88995 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -18,166 +18,73 @@ # is received in order, or that OWNERSHIP works properly, etc... MAX_SAMPLES_READ = 500 -def test_ownership_receivers(child_sub, samples_sent, last_sample_saved, timeout): +def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): """ - This function is used by test cases that have two publishers and one subscriber. - This tests that the Ownership QoS works correctly. In order to do that the - function checks if the subscriber has received samples from one publisher or - from both. A subscriber has received from both publishers if there is a sample - from each publisher interleaved. This is done to make sure the subscriber did - not started receiving samples from the publisher that was run before, and then - change to the publisher with the greatest ownership. + This function is used by test cases that have two publishers and one + subscriber. This tests check how many samples are received by the + subscriber application with different sizes. child_sub: child program generated with pexpect - samples_sent: list of multiprocessing Queues with the samples - the publishers send. Element 1 of the list is for - publisher 1, etc. - last_sample_saved: list of multiprocessing Queues with the last - sample saved on samples_sent for each Publisher. Element 1 of - the list is for Publisher 1, etc. + samples_sent: not used + last_sample_saved: not used timeout: time pexpect waits until it matches a pattern. - - This functions assumes that the subscriber has already received samples - from, at least, one publisher. """ - first_sample_received_publisher = 0 - ignore_first_samples = True - first_received = False - second_received = False - list_data_received_second = [] - list_data_received_first = [] + sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + child_sub.before + child_sub.after) + print(f'First sample found: {sub_string}') + if sub_string is None: + return ReturnCode.DATA_NOT_RECEIVED + + first_sample_size = int(sub_string.group(1)) + max_samples_received = MAX_SAMPLES_READ samples_read = 0 - list_samples_processed = [] - last_first_sample = '' - last_second_sample = '' - - while(samples_read < max_samples_received): - # take the topic, color, position and size of the ShapeType. - # child_sub.before contains x and y, and child_sub.after contains - # [shapesize] - # Example: child_sub.before contains 'Square BLUE 191 152' - # child_sub.after contains '[30]' - sub_string = re.search('[0-9]+ [0-9]+ \[[0-9]+\]', - child_sub.before + child_sub.after) - # sub_string contains 'x y [shapesize]', example: '191 152 [30]' - - # takes samples written from both publishers stored in their queues - # ('samples_sent[i]') and save them in different lists. - # Try to get all available samples to avoid a race condition that - # happens when the samples are not in the list but the reader has - # already read them. - # waits until to stop the execution of the loop and - # returns the code "RECEIVING_FROM_ONE". - # list_data_received_[first|second] is a list with the samples sent from - # its corresponding publisher - try: - while True: - list_data_received_first.append(samples_sent[0].get( - block=False)) - except queue.Empty: - pass - - try: - while True: - list_data_received_second.append(samples_sent[1].get( - block=False)) - except queue.Empty: - pass - - # Take the last sample published by each publisher from their queues - # ('last_sample_saved[i]') and save them local variables. - try: - last_first_sample = last_sample_saved[0].get(block=False) - except queue.Empty: - pass + ignore_first_samples = True + retcode = ReturnCode.RECEIVING_FROM_ONE - try: - last_second_sample = last_sample_saved[1].get(block=False) - except queue.Empty: - pass - - # Determine to which publisher the current sample belong to - if sub_string.group(0) in list_data_received_second: - current_sample_from_publisher = 2 - elif sub_string.group(0) in list_data_received_first: - current_sample_from_publisher = 1 - else: - # If the sample is not in any queue, break the loop if the - # the last sample for any publisher has already been processed. - if last_first_sample in list_samples_processed: - break - if last_second_sample in list_samples_processed: + while sub_string is not None and samples_read < max_samples_received: + print(f'Samples read {samples_read}, current size: {sub_string.group(1)}') + current_sample_size = int(sub_string.group(1)) + + if current_sample_size != first_sample_size: + if ignore_first_samples: + # the first time we receive a different size, ignore it + # For example, if we receive samples of size 20, then only + # samples of size 30, we have to return RECEIVING_FROM_ONE. + # If after receiving samples of size 30, we receive samples of + # size 20 again, then we return RECEIVING_FROM_BOTH. + ignore_first_samples = False + first_sample_size = current_sample_size + else: + retcode = ReturnCode.RECEIVING_FROM_BOTH break - print(f'Last samples: {last_first_sample}, {last_second_sample}') - # Otherwise, wait a bit and continue - time.sleep(0.1) - continue - - # Keep all samples processed in a single list, so we can check whether - # the last sample published by any publisher has already been processed - list_samples_processed.append(sub_string.group(0)) - - # If the app hit this point, it is because the previous subscriber - # sample has been already read. Then, we can process the next sample - # read by the subscriber. - # Get the next samples the subscriber is receiving + index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT, # index = 1 + pexpect.TIMEOUT # index = 1 ], timeout ) + if index == 1: break samples_read += 1 - # A potential case is that the reader gets data from one writer and - # then start receiving from a different writer with a higher - # ownership. This avoids returning RECEIVING_FROM_BOTH if this is - # the case. - # This if is only run once we process the first sample received by the - # subscriber application - if first_sample_received_publisher == 0: - if current_sample_from_publisher == 1: - first_sample_received_publisher = 1 - elif current_sample_from_publisher == 2: - first_sample_received_publisher = 2 - - # Check if the app still needs to ignore samples - if ignore_first_samples == True: - if (first_sample_received_publisher == 1 \ - and current_sample_from_publisher == 2) \ - or (first_sample_received_publisher == 2 \ - and current_sample_from_publisher == 1): - # if receiving samples from a different publisher, then stop - # ignoring samples - ignore_first_samples = False - else: - # in case that the app only receives samples from one publisher - # this loop always continues and will return RECEIVING_FROM_ONE - continue - - if current_sample_from_publisher == 1: - first_received = True - else: - second_received = True - if second_received == True and first_received == True: - return ReturnCode.RECEIVING_FROM_BOTH + sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + child_sub.before + child_sub.after) print(f'Samples read: {samples_read}') - return ReturnCode.RECEIVING_FROM_ONE + return retcode def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): """ This function is used by test cases that have two publishers and one - subscriber. This tests that only one of the color is received by the - subscriber application because it contains a filter that only allows to - receive data from one color. + subscriber. This tests how many samples are received by the + subscriber application with different colors. child_sub: child program generated with pexpect samples_sent: not used From 2f9979b5c4ee27e02534c77dc15536dbf32cd783 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 13 Jan 2026 20:36:49 +0100 Subject: [PATCH 04/33] Removed debug message --- test_suite_functions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index 6df88995..e940e5b9 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -32,7 +32,6 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): """ sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) - print(f'First sample found: {sub_string}') if sub_string is None: return ReturnCode.DATA_NOT_RECEIVED @@ -44,7 +43,6 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): retcode = ReturnCode.RECEIVING_FROM_ONE while sub_string is not None and samples_read < max_samples_received: - print(f'Samples read {samples_read}, current size: {sub_string.group(1)}') current_sample_size = int(sub_string.group(1)) if current_sample_size != first_sample_size: From c09e840b8803d8a55b21eac2a9b8088e60b5699e Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 19 Jan 2026 14:39:37 +0100 Subject: [PATCH 05/33] Added basic check for all tests --- interoperability_report.py | 4 +- rtps_test_utilities.py | 3 ++ test_suite_functions.py | 77 +++++++++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/interoperability_report.py b/interoperability_report.py index 1f72c19d..f3cdc1c4 100644 --- a/interoperability_report.py +++ b/interoperability_report.py @@ -24,7 +24,7 @@ if __name__ == "__main__" and platform.system() == "Darwin": multiprocessing.set_start_method('fork') -from rtps_test_utilities import ReturnCode, log_message, no_check, remove_ansi_colors +from rtps_test_utilities import ReturnCode, log_message, basic_check, remove_ansi_colors # This parameter is used to save the samples the Publisher sends. # MAX_SAMPLES_SAVED is the maximum number of samples saved. @@ -764,7 +764,7 @@ def main(): raise RuntimeError('Cannot process function of ' f'test case: {test_case_name}') else: - check_function = no_check + check_function = basic_check assert(len(parameters) == len(expected_codes)) diff --git a/rtps_test_utilities.py b/rtps_test_utilities.py index a6f219d6..138b25be 100644 --- a/rtps_test_utilities.py +++ b/rtps_test_utilities.py @@ -55,6 +55,9 @@ def remove_ansi_colors(text): return cleaned_str def no_check(child_sub, samples_sent, last_sample_saved, timeout): + return ReturnCode.OK + +def basic_check(child_sub, samples_sent, last_sample_saved, timeout): """ Only checks that the data is well formed and size is not zero.""" sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) diff --git a/test_suite_functions.py b/test_suite_functions.py index e940e5b9..233a4b9e 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -7,7 +7,7 @@ # ################################################################# -from rtps_test_utilities import ReturnCode +from rtps_test_utilities import ReturnCode, basic_check import re import pexpect import queue @@ -30,6 +30,11 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): last_sample_saved: not used timeout: time pexpect waits until it matches a pattern. """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) if sub_string is None: @@ -89,6 +94,11 @@ def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): last_sample_saved: not used timeout: time pexpect waits until it matches a pattern. """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + sub_string = re.search('\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', child_sub.before + child_sub.after) first_sample_color = sub_string.group(1) @@ -133,6 +143,11 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): timeout: not used """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK # Read the first sample printed by the subscriber @@ -186,6 +201,11 @@ def test_reliability_no_losses(child_sub, samples_sent, last_sample_saved, timeo timeout: time pexpect waits until it matches a pattern. """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK processed_samples = 0 @@ -268,6 +288,11 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ timeout: time pexpect waits until it matches a pattern. """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK instance_color = [] @@ -335,6 +360,11 @@ def test_durability_volatile(child_sub, samples_sent, last_sample_saved, timeout timeout: not used """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + # Read the first sample, if it has the size > 5, it is using volatile # durability correctly sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', @@ -367,6 +397,11 @@ def test_durability_transient_local(child_sub, samples_sent, last_sample_saved, timeout: not used """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + # Read the first sample, if it has the size == 1, it is using transient # local durability correctly sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', @@ -396,6 +431,11 @@ def test_deadline_missed(child_sub, samples_sent, last_sample_saved, timeout): timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + # At this point, the subscriber app has already received one sample # Check deadline requested missed index = child_sub.expect([ @@ -427,6 +467,11 @@ def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sampl timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK instance_color = [] @@ -492,6 +537,11 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK instance_color = [] @@ -544,6 +594,11 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK instance_color = [] @@ -596,6 +651,11 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.DATA_NOT_CORRECT # As the interoperability_test is just looking for the size [], # this does not count the data after it, we need to read one more full @@ -634,6 +694,11 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK # as the test is reading in a slower rate, reduce the number of samples read @@ -719,6 +784,11 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK instance_color = [] @@ -822,6 +892,11 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou timeout: time pexpect waits until it matches a pattern """ + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + produced_code = ReturnCode.OK topics = {} From 89268a459e1fd2306236b9dd9e92ceca9e015771 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Thu, 22 Jan 2026 20:54:37 +0100 Subject: [PATCH 06/33] Replaced function of test_reliability_4 --- test_suite.py | 4 +- test_suite_functions.py | 91 ----------------------------------------- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/test_suite.py b/test_suite.py index 90ae585b..f2d57875 100644 --- a/test_suite.py +++ b/test_suite.py @@ -173,9 +173,9 @@ # This test checks that data is received in the right order 'Test_Reliability_4' : { - 'apps' : ['-P -t Square -r -k 0 -w', '-S -t Square -r -k 0'], + 'apps' : ['-P -t Square -r -k 0 -z 0', '-S -t Square -r -k 0'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_reliability_no_losses, + 'check_function' : tsf.test_reliability_no_losses_w_instances, 'title' : 'Behavior of RELIABLE reliability', 'description' : 'Verifies a RELIABLE publisher communicates with a RELIABLE subscriber and samples are received ' 'in order without any losses or duplicates\n\n' diff --git a/test_suite_functions.py b/test_suite_functions.py index 233a4b9e..c81250ab 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -187,97 +187,6 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): print(f'Samples read: {samples_read}') return produced_code - -def test_reliability_no_losses(child_sub, samples_sent, last_sample_saved, timeout): - """ - This function tests RELIABLE reliability, it checks whether the subscriber - receives the samples in order and with no losses. - - child_sub: child program generated with pexpect - samples_sent: list of multiprocessing Queues with the samples - the publishers send. Element 1 of the list is for - publisher 1, etc. - last_sample_saved: not used - timeout: time pexpect waits until it matches a pattern. - """ - - basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) - - if basic_check_retcode != ReturnCode.OK: - return basic_check_retcode - - produced_code = ReturnCode.OK - processed_samples = 0 - - # take the first sample received by the subscriber - sub_string = re.search('[0-9]+ [0-9]+ \[[0-9]+\]', - child_sub.before + child_sub.after) - - # This makes sure that at least one sample has been received - if sub_string.group(0) is None: - produced_code = ReturnCode.DATA_NOT_RECEIVED - - # Get the sample sent by the DataWriter that matches the first sample - # received - pub_sample = "" - try: - while pub_sample != sub_string.group(0): - pub_sample = samples_sent[0].get(block=True, timeout=timeout) - except: - # If we don't find a sample in the publisher app that matches the - # first sample received by the DataReader - produced_code = ReturnCode.DATA_NOT_CORRECT - - # The first execution we don't need to call samples_sent[0].get() because - # that takes the first element and remove it from the queue. As we have - # checked before that the samples are the same, we need to skip that part - # and get the next received sample - first_execution = True - - max_samples_received = MAX_SAMPLES_READ - samples_read = 0 - - while sub_string is not None and samples_read < max_samples_received: - # check that all the samples received by the DataReader are in order - # and matches the samples sent by the DataWriter - try: - if first_execution: - # do nothing because the first execution should already have - # a pub_sample so we don't need to get it from the queue - first_execution = False - else: - pub_sample = samples_sent[0].get(block=False) - - if pub_sample != sub_string.group(0): - produced_code = ReturnCode.DATA_NOT_CORRECT - break - processed_samples += 1 - - except: - # at least 2 samples should be received - if processed_samples <= 1: - produced_code = ReturnCode.DATA_NOT_CORRECT - break - - # Get the next sample the subscriber is receiving - index = child_sub.expect( - [ - '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 - ], - timeout - ) - if index == 1: - # no more data to process - break - samples_read += 1 - # search the next received sample by the subscriber app - sub_string = re.search('[0-9]+ [0-9]+ \[[0-9]+\]', - child_sub.before + child_sub.after) - - print(f'Samples read: {samples_read}') - return produced_code - def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests RELIABLE reliability, it checks whether the subscriber From b9de9c9f459bd619b231aef7bf85cf0b8e54bd5e Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 23 Jan 2026 18:56:03 +0100 Subject: [PATCH 07/33] Fixed lifespan tests in a corner case when the first sample is the only one from its group of samples --- test_suite_functions.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index c81250ab..55785850 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -618,6 +618,7 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou first_iteration = [] samples_read = 0 consecutive_samples = [] + ignore_first_sample = [] while samples_read < max_samples_lifespan: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', @@ -630,6 +631,7 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou previous_seq_num.append(int(sub_string.group(2))) first_iteration.append(True) consecutive_samples.append(1) # take into account the first sample + ignore_first_sample.append(True) # if the instance exists if sub_string.group(1) in instance_color: @@ -643,6 +645,8 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou # if the sequence number is consecutive, increase the counter if previous_seq_num[index] + 1 == int(sub_string.group(2)): consecutive_samples[index] += 1 + # if found consecutive samples, do not ignore the first sample + ignore_first_sample[index] = False else: # if the sequence number is not consecutive, check that we # receive only 3 or 2 samples @@ -651,10 +655,16 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou # sample is consecutive with itself consecutive_samples[index] = 1 else: - # if the amount of samples received is different than 3 or 2 - # this is an error - produced_code = ReturnCode.DATA_NOT_CORRECT - break + if ignore_first_sample[index]: + # there may be a case in which we receive a sample + # and the next one is not consecutive, if that is the + # case, ignore it + ignore_first_sample[index] = False + else: + # if the amount of samples received is different than 3 or 2 + # this is an error + produced_code = ReturnCode.DATA_NOT_CORRECT + break previous_seq_num[index] = int(sub_string.group(2)) else: From 0afac3c86da5ee299854afc2184b94f47fd11d2a Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 23 Jan 2026 19:29:48 +0100 Subject: [PATCH 08/33] Fixed a potential infinite loop in the FinalInstanceState tests --- test_suite_functions.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index 55785850..d97bccba 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -455,8 +455,10 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t instance_color = [] unregistered_instance_color = [] + max_samples_received = MAX_SAMPLES_READ + samples_read = 0 - while True: + while samples_read < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -483,9 +485,13 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t if index == 1: # no more data to process break + if index == 0: + samples_read += 1 - # compare that arrays contain the same elements - if set(instance_color) == set(unregistered_instance_color): + # compare that arrays contain the same elements and are not empty + if len(instance_color) == 0: + produced_code = ReturnCode.DATA_NOT_RECEIVED + elif set(instance_color) == set(unregistered_instance_color): produced_code = ReturnCode.OK else: produced_code = ReturnCode.DATA_NOT_CORRECT @@ -512,8 +518,9 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo instance_color = [] disposed_instance_color = [] + max_samples_received = MAX_SAMPLES_READ - while True: + while samples_read < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -540,9 +547,13 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo if index == 1: # no more data to process break + if index == 0: + samples_read += 1 - # compare that arrays contain the same elements - if set(instance_color) == set(disposed_instance_color): + # compare that arrays contain the same elements and are not empty + if len(instance_color) == 0: + produced_code = ReturnCode.DATA_NOT_RECEIVED + elif set(instance_color) == set(disposed_instance_color): produced_code = ReturnCode.OK else: produced_code = ReturnCode.DATA_NOT_CORRECT From fe02fc058ebfaf3f624a1ffe051628f1e1ddc52d Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 23 Jan 2026 21:22:12 +0100 Subject: [PATCH 09/33] Fixed disposing test --- test_suite_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_suite_functions.py b/test_suite_functions.py index d97bccba..7a477ff0 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -519,6 +519,7 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo instance_color = [] disposed_instance_color = [] max_samples_received = MAX_SAMPLES_READ + samples_read = 0 while samples_read < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', From c40e361f2f6efad7c2ef537e9643b26508154c96 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 27 Jan 2026 15:17:25 +0100 Subject: [PATCH 10/33] Adding EOF to all expect functions --- interoperability_report.py | 36 +++++++----- test_suite_functions.py | 113 +++++++++++++++++++++++++------------ 2 files changed, 97 insertions(+), 52 deletions(-) diff --git a/interoperability_report.py b/interoperability_report.py index f3cdc1c4..116559c1 100644 --- a/interoperability_report.py +++ b/interoperability_report.py @@ -121,15 +121,16 @@ def run_subscriber_shape_main( index = child_sub.expect( [ 'Create reader for topic:', # index = 0 - pexpect.TIMEOUT, # index = 1 - 'failed to create content filtered topic' # index = 2 + 'failed to create content filtered topic', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout ) - if index == 1: + if index == 2 or index == 3: produced_code[produced_code_index] = ReturnCode.READER_NOT_CREATED - elif index == 2: + elif index == 1: produced_code[produced_code_index] = ReturnCode.FILTER_NOT_CREATED elif index == 0: # Step 4: Read data or incompatible qos or deadline missed @@ -140,6 +141,7 @@ def run_subscriber_shape_main( 'on_requested_incompatible_qos()', # index = 1 'on_requested_deadline_missed()', # index = 2 pexpect.TIMEOUT, # index = 3 + pexpect.EOF # index = 4 ], timeout ) @@ -148,7 +150,7 @@ def run_subscriber_shape_main( produced_code[produced_code_index] = ReturnCode.INCOMPATIBLE_QOS elif index == 2: produced_code[produced_code_index] = ReturnCode.DEADLINE_MISSED - elif index == 3: + elif index == 3 or index == 4: produced_code[produced_code_index] = ReturnCode.DATA_NOT_RECEIVED elif index == 0: # Step 5: Receiving samples @@ -258,11 +260,12 @@ def run_publisher_shape_main( index = child_pub.expect( [ 'Create writer for topic', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: + if index == 1 or index == 2: produced_code[produced_code_index] = ReturnCode.WRITER_NOT_CREATED elif index == 0: # Step 4: Check if the writer matches the reader @@ -271,14 +274,15 @@ def run_publisher_shape_main( index = child_pub.expect( [ 'on_publication_matched()', # index = 0 - pexpect.TIMEOUT, # index = 1 - 'on_offered_incompatible_qos' # index = 2 + 'on_offered_incompatible_qos', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout ) - if index == 1: + if index == 2 or index == 3: produced_code[produced_code_index] = ReturnCode.READER_NOT_MATCHED - elif index == 2: + elif index == 1: produced_code[produced_code_index] = ReturnCode.INCOMPATIBLE_QOS elif index == 0: # In the case that the option -w is selected, the Publisher @@ -292,12 +296,13 @@ def run_publisher_shape_main( index = child_pub.expect([ '\[[0-9]+\]', # index = 0 'on_offered_deadline_missed()', # index = 1 - pexpect.TIMEOUT # index = 2 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout) if index == 1: produced_code[produced_code_index] = ReturnCode.DEADLINE_MISSED - elif index == 2: + elif index == 2 or index == 3: produced_code[produced_code_index] = ReturnCode.DATA_NOT_SENT elif index == 0: produced_code[produced_code_index] = ReturnCode.OK @@ -314,13 +319,14 @@ def run_publisher_shape_main( index = child_pub.expect([ '\[[0-9]+\]', # index = 0 'on_offered_deadline_missed()', # index = 1 - pexpect.TIMEOUT # index = 2 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout) if index == 1: produced_code[produced_code_index] = ReturnCode.DEADLINE_MISSED break - elif index == 2: + elif index == 2 or index == 3: produced_code[produced_code_index] = ReturnCode.DATA_NOT_SENT break last_sample_saved.put(last_sample) diff --git a/test_suite_functions.py b/test_suite_functions.py index 7a477ff0..c16ee348 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -66,13 +66,16 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) if index == 1: break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED samples_read += 1 @@ -116,13 +119,16 @@ def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) if index == 1: break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED samples_read += 1 @@ -170,13 +176,16 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) if index == 1: # no more data to process break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED samples_read += 1 @@ -238,15 +247,18 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 1: + # no more data to process + break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED print(f'Samples read: {samples_read}, instances: {instance_color}') return produced_code @@ -349,15 +361,19 @@ def test_deadline_missed(child_sub, samples_sent, last_sample_saved, timeout): # Check deadline requested missed index = child_sub.expect([ 'on_requested_deadline_missed()', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout) if index == 0: return ReturnCode.DEADLINE_MISSED + elif index == 1: + return ReturnCode.DATA_NOT_RECEIVED else: index = child_sub.expect([ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout) if index == 0: @@ -424,15 +440,18 @@ def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sampl index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 1: + # no more data to process + break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED print(f'Samples read: {samples_read}, instances: {instance_color}') return produced_code @@ -478,15 +497,18 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t index = child_sub.expect( [ r'\w+\s+\w+\s+.*?\n', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 1: + # no more data to process + break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED # compare that arrays contain the same elements and are not empty if len(instance_color) == 0: @@ -541,15 +563,18 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo index = child_sub.expect( [ r'\w+\s+\w+\s+.*?\n', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 1: + # no more data to process + break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED # compare that arrays contain the same elements and are not empty if len(instance_color) == 0: @@ -584,7 +609,8 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{[0-9]+\}', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) @@ -601,6 +627,8 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): produced_code = ReturnCode.OK else: produced_code = ReturnCode.DATA_NOT_CORRECT + elif index == 2: + produced_code = ReturnCode.DATA_NOT_RECEIVED return produced_code @@ -687,15 +715,18 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT # index = 1 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 1: + # no more data to process + break + elif index == 2: + return ReturnCode.DATA_NOT_RECEIVED print(f'Samples read: {samples_read}, instances: {instance_color}') return produced_code @@ -703,7 +734,7 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests that ordered access works correctly. This counts the - samples received in order and detects wether they are from the same instance + samples received in order and detects whether they are from the same instance as the previously received sample or not. If the number of consecutive samples from the same instance is greater than the number of consecutive samples form different instances, this means that @@ -795,16 +826,20 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo # 'Reading with ordered access message' index = child_sub.expect( [ - r'(Reading with ordered access.*?\n|[\[[0-9]+\])', # index = 0 - pexpect.TIMEOUT # index = 1 + '\[[0-9]+\]', # index = 0 + r'Reading with ordered access.*?\n', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 2: + # no more data to process + break + elif index == 3: + return ReturnCode.DATA_NOT_RECEIVED print(f'Samples read: {samples_read}, instances: {instance_color}') return produced_code @@ -902,16 +937,20 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # 'Reading with ordered access message' index = child_sub.expect( [ - r'(Reading coherent sets.*?\n|[\[[0-9]+\])', # index = 0 - pexpect.TIMEOUT # index = 1 + '\[[0-9]+\]', # index = 0 + r'Reading coherent sets.*?\n', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout ) - if index == 1: - # no more data to process - break if index == 0: samples_read += 1 + elif index == 2: + # no more data to process + break + elif index == 3: + return ReturnCode.DATA_NOT_RECEIVED print(f'Samples read: {samples_read}') print("Instances:") From e002b250036902c17368d296393bf1e8a957254f Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Wed, 28 Jan 2026 20:30:58 +0100 Subject: [PATCH 11/33] Fixed large data and coherent sets test that were returning false-positives --- test_suite.py | 10 +++---- test_suite_functions.py | 62 ++++++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/test_suite.py b/test_suite.py index f2d57875..fdb00a9d 100644 --- a/test_suite.py +++ b/test_suite.py @@ -851,8 +851,8 @@ }, 'Test_LargeData_0' : { - 'apps' : ['-P -t Square --additional-payload-size 100000', - '-S -t Square'], + 'apps' : ['-P -t Square -r -k 0 --additional-payload-size 100000', + '-S -t Square -r -k 0'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.test_large_data, 'title' : 'Test large data', @@ -1223,7 +1223,7 @@ ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' - 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' 'per instance and topic is 3 each time we receive a new coherent set of samples\n' f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' }, @@ -1249,7 +1249,7 @@ ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' - 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' 'per instance and topic is 3 each time we receive a new coherent set of samples\n' f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' }, @@ -1275,7 +1275,7 @@ ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' - 'instance, 4 instances and 3 topics. This test check that the consecutive samples received ' + 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' 'per instance and topic is 3 each time we receive a new coherent set of samples\n' f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' }, diff --git a/test_suite_functions.py b/test_suite_functions.py index c16ee348..bc4e4b68 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -603,32 +603,42 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): return basic_check_retcode produced_code = ReturnCode.DATA_NOT_CORRECT - # As the interoperability_test is just looking for the size [], - # this does not count the data after it, we need to read one more full - # sample - index = child_sub.expect( - [ - r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{[0-9]+\}', # index = 0 - pexpect.TIMEOUT, # index = 1 - pexpect.EOF # index = 2 - ], - timeout - ) + samples_read = 0 - if index == 0: - # Read the sample received, if it prints the additional_bytes == 255, - # it is sending large data correctly - sub_string = re.search(r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{([0-9]+)\}', - child_sub.before + child_sub.after) - # Check if the last element of the additional_bytes field element is - # received correctly - if sub_string is not None: - if int(sub_string.group(1)) == 255: - produced_code = ReturnCode.OK + while samples_read < MAX_SAMPLES_READ: + # As the interoperability_report is just looking for the size [], + # this does not count the data after it, we need to read a full sample + index = child_sub.expect( + [ + r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{[0-9]+\}', # index = 0 + pexpect.TIMEOUT, # index = 1 + pexpect.EOF # index = 2 + ], + timeout + ) + + if index == 0: + # Read the sample received, if it prints the additional_bytes == 255, + # it is sending large data correctly + sub_string = re.search(r'\w+\s+\w+\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]\s+\{([0-9]+)\}', + child_sub.before + child_sub.after) + # Check if the last element of the additional_bytes field element is + # received correctly + if sub_string is not None: + if int(sub_string.group(1)) == 255: + produced_code = ReturnCode.OK + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + samples_read += 1 else: produced_code = ReturnCode.DATA_NOT_CORRECT - elif index == 2: - produced_code = ReturnCode.DATA_NOT_RECEIVED + break + elif index == 1 or index == 2: + produced_code = ReturnCode.DATA_NOT_RECEIVED + break + + print(f'Samples read: {samples_read}') return produced_code @@ -850,7 +860,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou consecutive samples received from the same instance. The value should be 3 as this is the coherent set count that the test is setting. Note: when using GROUP_PRESENTATION, the first iteration may print more - samples (more coherent sets), the test check that the samples received per + samples (more coherent sets), the test checks that the samples received per instance is a multiple of 3, so the coherent sets are received complete. child_sub: child program generated with pexpect samples_sent: not used @@ -863,7 +873,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.OK + produced_code = ReturnCode.DATA_NOT_CORRECT topics = {} samples_read = 0 @@ -911,6 +921,8 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou for color in topics[topic]: topics[topic][color] = 1 elif new_coherent_set_read: + # the test is only ok if it has received coherent sets + produced_code = ReturnCode.OK for topic in topics: for color in topics[topic]: if first_time_reading: From e934129bca90e133b665818e1e2481b5f1ece8cd Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 30 Jan 2026 18:12:56 +0100 Subject: [PATCH 12/33] Added a way of stopping coherent_sets and ordered_access tests --- test_suite_functions.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index bc4e4b68..5b0f4340 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -769,6 +769,7 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo color_different_count = 0 color_equal_count = 0 samples_printed = False + ordered_access_group_count = 0 while samples_read < MAX_SAMPLES_READ: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', @@ -845,11 +846,21 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo ) if index == 0: samples_read += 1 + elif index == 1: + ordered_access_group_count += 1 elif index == 2: # no more data to process break elif index == 3: - return ReturnCode.DATA_NOT_RECEIVED + produced_code = ReturnCode.DATA_NOT_RECEIVED + break + + # Exit condition in case there are no samples being printed + if ordered_access_group_count > MAX_SAMPLES_READ: + # If we have not read enough samples, we consider it a failure + if samples_read <= MAX_SAMPLES_READ: + produced_code = ReturnCode.DATA_NOT_RECEIVED + break print(f'Samples read: {samples_read}, instances: {instance_color}') return produced_code @@ -881,6 +892,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou new_coherent_set_read = False first_time_reading = True ignore_firsts_coherent_set = 2 + coherent_sets_count = 0 while samples_read < MAX_SAMPLES_READ: sub_string = re.search(r'(\w+)\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', @@ -958,11 +970,21 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou ) if index == 0: samples_read += 1 + elif index == 1: + coherent_sets_count += 1 elif index == 2: # no more data to process break elif index == 3: - return ReturnCode.DATA_NOT_RECEIVED + produced_code = ReturnCode.DATA_NOT_RECEIVED + break + + # Exit condition in case there are no samples being printed + if coherent_sets_count > MAX_SAMPLES_READ: + # If we have not read enough samples, we consider it a failure + if samples_read <= MAX_SAMPLES_READ: + produced_code = ReturnCode.DATA_NOT_RECEIVED + break print(f'Samples read: {samples_read}') print("Instances:") From e6185762e7fb3c812c04a11cdc9b88e019241ce5 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 6 Feb 2026 22:55:36 +0100 Subject: [PATCH 13/33] Added more lifespan tests and improved test check functions --- test_suite.py | 162 +++++++++++++++++++++++++++++++++++++--- test_suite_functions.py | 128 +++++++++++++++++++------------ 2 files changed, 230 insertions(+), 60 deletions(-) diff --git a/test_suite.py b/test_suite.py index fdb00a9d..5f4fe796 100644 --- a/test_suite.py +++ b/test_suite.py @@ -769,9 +769,10 @@ }, 'Test_TimeBasedFilter_0' : { - 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100', '-S -t Square -r -k 0 --time-filter 1000'], + 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100', + '-S -t Square -r -k 0 --time-filter 1000'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_reading_each_10_samples_w_instances, + 'check_function' : tsf.test_reading_1_sample_every_10_samples_w_instances, 'title' : 'Test the behavior of the TIME_BASED_FILTER QoS', 'description' : 'Verifies a subscriber with TIME_BASED_FILTER work as expected\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' @@ -789,7 +790,7 @@ 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100 --num-instances 4', '-S -t Square -r -k 0 --time-filter 1000'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_reading_each_10_samples_w_instances, + 'check_function' : tsf.test_reading_1_sample_every_10_samples_w_instances, 'title' : 'Test the behavior of the TIME_BASED_FILTER QoS with several instances', 'description' : 'Verifies a subscriber with TIME_BASED_FILTER work as expected when the publisher ' 'publishes several instances\n\n' @@ -867,16 +868,15 @@ 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100 --lifespan 250', '-S -t Square -r -k 0 --read-period 500'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_lifespan_w_instances, - 'title' : 'Test the behavior of lifespan', + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan with fractions of seconds, reliable', 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' - ' lifespan\n\n' + ' lifespan with fractions of seconds\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' ' * Configures the subscriber with a reading period of 500ms\n' - ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' @@ -891,17 +891,157 @@ 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100 --lifespan 250 --num-instances 4', '-S -t Square -r -k 0 --read-period 500'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_lifespan_w_instances, - 'title' : 'Test the behavior of lifespan with several instances', + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan with fraction of seconds with several instances, reliable', 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' - ' lifespan and different instances\n\n' + ' lifespan with fraction of seconds and different instances\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' ' * Configures the subscriber with a reading period of 500ms\n' ' * The publisher publishes 4 different instances (using the same data value)\n' - ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time per instance\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_2' : { + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 400 --lifespan 1000', + '-S -t Square -r -k 0 --read-period 2000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan expressed in seconds, reliable', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan in seconds\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 1s\n' + ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the subscriber with a reading period of 2s\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_3' : { + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 400 --lifespan 1000 --num-instances 4', + '-S -t Square -r -k 0 --read-period 2000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan expressed in seconds with several instances, reliable', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan in seconds, and different instances\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 1s\n' + ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the subscriber with a reading period of 2s\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time per instance\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_4' : { + 'apps' : ['-P -t Square -b -z 0 --write-period 100 --lifespan 250', + '-S -t Square -b -k 0 --read-period 500'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan with fractions of seconds, best effort', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan with fractions of seconds\n\n' + ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' + ' * Configures the subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 250ms\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber with a reading period of 500ms\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_5' : { + 'apps' : ['-P -t Square -b -z 0 --write-period 100 --lifespan 250 --num-instances 4', + '-S -t Square -b -k 0 --read-period 500'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan with fraction of seconds with several instances, best effort', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan with fraction of seconds and different instances\n\n' + ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' + ' * Configures the subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 250ms\n' + ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the subscriber with a reading period of 500ms\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time per instance\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_6' : { + 'apps' : ['-P -t Square -b -z 0 --write-period 400 --lifespan 1000', + '-S -t Square -b -k 0 --read-period 2000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan expressed in seconds, best effort', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan in seconds\n\n' + ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' + ' * Configures the subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 1s\n' + ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the subscriber with a reading period of 2s\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' + 'each time it reads.\n' + 'As the publisher sets the lifespan, it expires in some samples before the ' + 'subscriber reads data. The amount of consecutive samples read by the ' + 'subscriber application is 2 or 3 each time\n' + f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + }, + + 'Test_Lifespan_7' : { + 'apps' : ['-P -t Square -b -z 0 --write-period 400 --lifespan 1000 --num-instances 4', + '-S -t Square -b -k 0 --read-period 2000'], + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_lifespan_2_3_consecutive_samples_w_instances, + 'title' : 'Test the behavior of lifespan expressed in seconds with several instances, best effort', + 'description' : 'Verifies a subscriber receives the expected samples when the publisher uses ' + ' lifespan in seconds, and different instances\n\n' + ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' + ' * Configures the subscriber with history KEEP_ALL\n' + ' * Configures the publisher with a lifespan of 1s\n' + ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the subscriber with a reading period of 2s\n' + ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber application receives 2 or 3 consecutive samples ' diff --git a/test_suite_functions.py b/test_suite_functions.py index 5b0f4340..62b90207 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -154,7 +154,7 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.OK + produced_code = ReturnCode.DATA_NOT_RECEIVED # Read the first sample printed by the subscriber sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', @@ -185,7 +185,8 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): # no more data to process break elif index == 2: - return ReturnCode.DATA_NOT_RECEIVED + produced_code = ReturnCode.DATA_NOT_RECEIVED + break samples_read += 1 @@ -193,6 +194,9 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) + if samples_read == max_samples_received: + produced_code = ReturnCode.OK + print(f'Samples read: {samples_read}') return produced_code @@ -211,14 +215,15 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.OK + produced_code = ReturnCode.DATA_NOT_RECEIVED instance_color = [] instance_seq_num = [] first_iteration = [] - samples_read = 0 + samples_read_per_instance = 0 + max_samples_received = MAX_SAMPLES_READ - while samples_read < MAX_SAMPLES_READ: + while samples_read_per_instance < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -253,14 +258,18 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(1) in instance_color: + samples_read_per_instance += 1 elif index == 1: # no more data to process break elif index == 2: return ReturnCode.DATA_NOT_RECEIVED - print(f'Samples read: {samples_read}, instances: {instance_color}') + if max_samples_received == samples_read_per_instance: + produced_code = ReturnCode.OK + + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code @@ -381,11 +390,11 @@ def test_deadline_missed(child_sub, samples_sent, last_sample_saved, timeout): else: return ReturnCode.DATA_NOT_RECEIVED -def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sample_saved, timeout): +def test_reading_1_sample_every_10_samples_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests whether the subscriber application receives one sample out of 10 (for each instance). For example, first sample received with size - 5, the next one should have size 15, then 25... + 5, the next one should have size [15,24], then [25,34], etc. child_sub: child program generated with pexpect samples_sent: not used last_sample_saved: not used @@ -397,16 +406,16 @@ def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sampl if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.OK + produced_code = ReturnCode.DATA_NOT_RECEIVED instance_color = [] instance_seq_num = [] first_iteration = [] ignore_first_sample = [] max_samples_received = MAX_SAMPLES_READ / 20 # 25 - samples_read = 0 + samples_read_per_instance = 0 - while samples_read < max_samples_received: + while samples_read_per_instance < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -424,14 +433,18 @@ def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sampl if first_iteration[index]: first_iteration[index] = False else: + current_seq_num = int(sub_string.group(2)) if ignore_first_sample[index]: ignore_first_sample[index] = False - # check that the received sample has the sequence number - # previous + 10 - elif instance_seq_num[index] + 10 != int(sub_string.group(2)): - produced_code = ReturnCode.DATA_NOT_CORRECT - break - instance_seq_num[index] = int(sub_string.group(2)) + else: + # check that the received sample reads only one sample in + # in the period of 10 samples. For example, if the previous + # sample received has size 5, the next one should be + # between [15-24], both included + if current_seq_num <= (instance_seq_num[index] + 9) or current_seq_num > instance_seq_num[index] + 19: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + instance_seq_num[index] = current_seq_num else: produced_code = ReturnCode.DATA_NOT_CORRECT break @@ -446,14 +459,19 @@ def test_reading_each_10_samples_w_instances(child_sub, samples_sent, last_sampl timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(1) == instance_color[0]: + # increase samples_read_per_instance only for the first instance + samples_read_per_instance += 1 elif index == 1: # no more data to process break elif index == 2: return ReturnCode.DATA_NOT_RECEIVED - print(f'Samples read: {samples_read}, instances: {instance_color}') + if max_samples_received == samples_read_per_instance: + produced_code = ReturnCode.OK + + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, timeout): @@ -475,9 +493,9 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t instance_color = [] unregistered_instance_color = [] max_samples_received = MAX_SAMPLES_READ - samples_read = 0 + samples_read_per_instance = 0 - while samples_read < max_samples_received: + while samples_read_per_instance < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -489,7 +507,7 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t # if no sample is received, it might be a UNREGISTER message sub_string = re.search(r'\w+\s+(\w+)\s+NOT_ALIVE_NO_WRITERS_INSTANCE_STATE', child_sub.before + child_sub.after) - if sub_string is not None: + if sub_string is not None and sub_string.group(1) not in unregistered_instance_color: unregistered_instance_color.append(sub_string.group(1)) if len(instance_color) == len(unregistered_instance_color): break @@ -503,7 +521,8 @@ def test_unregistering_w_instances(child_sub, samples_sent, last_sample_saved, t timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(1) == instance_color[0]: + samples_read_per_instance += 1 elif index == 1: # no more data to process break @@ -541,9 +560,9 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo instance_color = [] disposed_instance_color = [] max_samples_received = MAX_SAMPLES_READ - samples_read = 0 + samples_read_per_instance = 0 - while samples_read < max_samples_received: + while samples_read_per_instance < max_samples_received: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -555,7 +574,7 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo # if no sample is received, it might be a DISPOSED message sub_string = re.search(r'\w+\s+(\w+)\s+NOT_ALIVE_DISPOSED_INSTANCE_STATE', child_sub.before + child_sub.after) - if sub_string is not None: + if sub_string is not None and sub_string.group(1) not in disposed_instance_color: disposed_instance_color.append(sub_string.group(1)) if len(instance_color) == len(disposed_instance_color): break @@ -569,7 +588,8 @@ def test_disposing_w_instances(child_sub, samples_sent, last_sample_saved, timeo timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(1) == instance_color[0]: + samples_read_per_instance += 1 elif index == 1: # no more data to process break @@ -602,7 +622,7 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.DATA_NOT_CORRECT + produced_code = ReturnCode.DATA_NOT_RECEIVED samples_read = 0 while samples_read < MAX_SAMPLES_READ: @@ -625,9 +645,7 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): # Check if the last element of the additional_bytes field element is # received correctly if sub_string is not None: - if int(sub_string.group(1)) == 255: - produced_code = ReturnCode.OK - else: + if int(sub_string.group(1)) != 255: produced_code = ReturnCode.DATA_NOT_CORRECT break samples_read += 1 @@ -638,11 +656,14 @@ def test_large_data(child_sub, samples_sent, last_sample_saved, timeout): produced_code = ReturnCode.DATA_NOT_RECEIVED break + if samples_read == MAX_SAMPLES_READ: + produced_code = ReturnCode.OK + print(f'Samples read: {samples_read}') return produced_code -def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeout): +def test_lifespan_2_3_consecutive_samples_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests that lifespan works correctly. In the test situation, only 2 or 3 consecutive samples should be received each time the reader @@ -658,7 +679,7 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.OK + produced_code = ReturnCode.DATA_NOT_RECEIVED # as the test is reading in a slower rate, reduce the number of samples read max_samples_lifespan = MAX_SAMPLES_READ / 10 # 50 @@ -666,11 +687,11 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou instance_color = [] previous_seq_num = [] first_iteration = [] - samples_read = 0 + samples_read_per_instance = 0 consecutive_samples = [] ignore_first_sample = [] - while samples_read < max_samples_lifespan: + while samples_read_per_instance < max_samples_lifespan: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', child_sub.before + child_sub.after) @@ -704,6 +725,7 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou # reset value to 1, as this test consider that the first # sample is consecutive with itself consecutive_samples[index] = 1 + produced_code = ReturnCode.OK else: if ignore_first_sample[index]: # there may be a case in which we receive a sample @@ -731,14 +753,20 @@ def test_lifespan_w_instances(child_sub, samples_sent, last_sample_saved, timeou timeout ) if index == 0: - samples_read += 1 + if sub_string.group(1) == instance_color[0]: + # increase samples_read_per_instance only for the first instance + samples_read_per_instance += 1 + print(f'{child_sub.before + child_sub.after}') elif index == 1: # no more data to process break elif index == 2: return ReturnCode.DATA_NOT_RECEIVED - print(f'Samples read: {samples_read}, instances: {instance_color}') + if max_samples_lifespan == samples_read_per_instance: + produced_code = ReturnCode.OK + + print(f'Samples read: {samples_read_per_instance}, instances: {instance_color}') return produced_code def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeout): @@ -764,14 +792,14 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo produced_code = ReturnCode.OK instance_color = [] - samples_read = 0 + samples_read_per_instance = 0 previous_sample_color = None color_different_count = 0 color_equal_count = 0 samples_printed = False ordered_access_group_count = 0 - while samples_read < MAX_SAMPLES_READ: + while samples_read_per_instance < MAX_SAMPLES_READ: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', child_sub.before + child_sub.after) @@ -845,7 +873,8 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(1) in instance_color: + samples_read_per_instance += 1 elif index == 1: ordered_access_group_count += 1 elif index == 2: @@ -858,11 +887,11 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo # Exit condition in case there are no samples being printed if ordered_access_group_count > MAX_SAMPLES_READ: # If we have not read enough samples, we consider it a failure - if samples_read <= MAX_SAMPLES_READ: + if samples_read_per_instance <= MAX_SAMPLES_READ: produced_code = ReturnCode.DATA_NOT_RECEIVED break - print(f'Samples read: {samples_read}, instances: {instance_color}') + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeout): @@ -884,17 +913,17 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - produced_code = ReturnCode.DATA_NOT_CORRECT + produced_code = ReturnCode.DATA_NOT_RECEIVED topics = {} - samples_read = 0 + samples_read_per_instance = 0 previous_sample_color = None new_coherent_set_read = False first_time_reading = True ignore_firsts_coherent_set = 2 coherent_sets_count = 0 - while samples_read < MAX_SAMPLES_READ: + while samples_read_per_instance < MAX_SAMPLES_READ: sub_string = re.search(r'(\w+)\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', child_sub.before + child_sub.after) @@ -969,7 +998,8 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou timeout ) if index == 0: - samples_read += 1 + if sub_string is not None and sub_string.group(2) in topics[sub_string.group(1)]: + samples_read_per_instance += 1 elif index == 1: coherent_sets_count += 1 elif index == 2: @@ -982,11 +1012,11 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # Exit condition in case there are no samples being printed if coherent_sets_count > MAX_SAMPLES_READ: # If we have not read enough samples, we consider it a failure - if samples_read <= MAX_SAMPLES_READ: + if samples_read_per_instance <= MAX_SAMPLES_READ: produced_code = ReturnCode.DATA_NOT_RECEIVED break - print(f'Samples read: {samples_read}') + print(f'Samples read per instance: {samples_read_per_instance}') print("Instances:") for topic in topics: print(f"Topic {topic}: {', '.join(topics[topic].keys())}") From f8b68202ac02892567a86629e85320bc508307b2 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 6 Feb 2026 23:55:01 +0100 Subject: [PATCH 14/33] Removed debug message --- test_suite_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index 62b90207..45b131af 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -756,7 +756,6 @@ def test_lifespan_2_3_consecutive_samples_w_instances(child_sub, samples_sent, l if sub_string.group(1) == instance_color[0]: # increase samples_read_per_instance only for the first instance samples_read_per_instance += 1 - print(f'{child_sub.before + child_sub.after}') elif index == 1: # no more data to process break From a01b5577265ef737d6df8278265ea86fb7300c5c Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Sat, 7 Feb 2026 00:29:53 +0100 Subject: [PATCH 15/33] Fixed some issues --- test_suite_functions.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index 45b131af..b93ef98b 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -258,7 +258,7 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ timeout ) if index == 0: - if sub_string is not None and sub_string.group(1) in instance_color: + if sub_string is not None and sub_string.group(1) == instance_color[0]: samples_read_per_instance += 1 elif index == 1: # no more data to process @@ -440,8 +440,12 @@ def test_reading_1_sample_every_10_samples_w_instances(child_sub, samples_sent, # check that the received sample reads only one sample in # in the period of 10 samples. For example, if the previous # sample received has size 5, the next one should be - # between [15-24], both included - if current_seq_num <= (instance_seq_num[index] + 9) or current_seq_num > instance_seq_num[index] + 19: + # between [15-24], both included. + # As the write period does not take into account the + # execution overhead, the next valid sample may be + # between [14-24] if the filtering happens in the reader + # side. + if current_seq_num < (instance_seq_num[index] + 9) or current_seq_num > instance_seq_num[index] + 19: produced_code = ReturnCode.DATA_NOT_CORRECT break instance_seq_num[index] = current_seq_num @@ -753,7 +757,7 @@ def test_lifespan_2_3_consecutive_samples_w_instances(child_sub, samples_sent, l timeout ) if index == 0: - if sub_string.group(1) == instance_color[0]: + if sub_string is not None and sub_string.group(1) == instance_color[0]: # increase samples_read_per_instance only for the first instance samples_read_per_instance += 1 elif index == 1: @@ -801,19 +805,21 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo while samples_read_per_instance < MAX_SAMPLES_READ: sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', child_sub.before + child_sub.after) + current_color = None # if a sample is read if sub_string is not None: # samples have been printed at least once samples_printed = True + current_color = sub_string.group(1) + # add new instance to instance_color - if sub_string.group(1) not in instance_color: - instance_color.append(sub_string.group(1)) + if current_color not in instance_color: + instance_color.append(current_color) # the instance exists - if sub_string.group(1) in instance_color: - index = instance_color.index(sub_string.group(1)) - current_color = sub_string.group(1) + if current_color in instance_color: + index = instance_color.index(current_color) # check the previous color and increase the different or equal # counters if previous_sample_color is not None: @@ -872,7 +878,7 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo timeout ) if index == 0: - if sub_string is not None and sub_string.group(1) in instance_color: + if current_color is not None and current_color == instance_color[0]: samples_read_per_instance += 1 elif index == 1: ordered_access_group_count += 1 @@ -886,7 +892,7 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo # Exit condition in case there are no samples being printed if ordered_access_group_count > MAX_SAMPLES_READ: # If we have not read enough samples, we consider it a failure - if samples_read_per_instance <= MAX_SAMPLES_READ: + if samples_read_per_instance < MAX_SAMPLES_READ: produced_code = ReturnCode.DATA_NOT_RECEIVED break @@ -1011,7 +1017,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # Exit condition in case there are no samples being printed if coherent_sets_count > MAX_SAMPLES_READ: # If we have not read enough samples, we consider it a failure - if samples_read_per_instance <= MAX_SAMPLES_READ: + if samples_read_per_instance < MAX_SAMPLES_READ: produced_code = ReturnCode.DATA_NOT_RECEIVED break From 9634fbd81f9ffc38d8d585552cbd34cb3c1f5fb0 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Sat, 7 Feb 2026 04:24:09 +0100 Subject: [PATCH 16/33] Fixing coherent sets check function --- test_suite_functions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index b93ef98b..47a4f554 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -927,10 +927,13 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou first_time_reading = True ignore_firsts_coherent_set = 2 coherent_sets_count = 0 + coherent_sets_max_count = MAX_SAMPLES_READ / 5 # 100 - while samples_read_per_instance < MAX_SAMPLES_READ: + while samples_read_per_instance < coherent_sets_max_count: sub_string = re.search(r'(\w+)\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[[0-9]+\]', child_sub.before + child_sub.after) + topic_name = None + instance_color = None # if a sample is read if sub_string is not None: @@ -1003,7 +1006,10 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou timeout ) if index == 0: - if sub_string is not None and sub_string.group(2) in topics[sub_string.group(1)]: + # increase samples_read_per_instance only for the first topic and instance + if (topic_name is not None and instance_color is not None + and topic_name == list(topics.keys())[0] + and instance_color == list(topics[topic_name].keys())[0]): samples_read_per_instance += 1 elif index == 1: coherent_sets_count += 1 From 1b9213e5460939b09f96b889736c4b4f534153f6 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Sat, 7 Feb 2026 04:49:31 +0100 Subject: [PATCH 17/33] Added a way of removing bytes from publisher PTY that may block itself --- interoperability_report.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/interoperability_report.py b/interoperability_report.py index 116559c1..46b25c7c 100644 --- a/interoperability_report.py +++ b/interoperability_report.py @@ -335,8 +335,18 @@ def run_publisher_shape_main( log_message(f'Publisher {publisher_index}: Waiting for Subscribers to finish', verbosity) - for element in subscribers_finished: - element.wait() # wait for all subscribers to finish + # Check if all subscribers finished + while True: + if all(e.is_set() for e in subscribers_finished): + break + + # Drain publisher output + try: + child_pub.read_nonblocking(1024, timeout=0.1) + except pexpect.TIMEOUT: + pass + except pexpect.EOF: + break publisher_finished.set() # set publisher as finished # Send SIGINT to nicely close the application if child_pub.isalive(): From c59da27527f3e4bf9b400d72079266cdddee7fe7 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 16 Feb 2026 17:58:46 +0100 Subject: [PATCH 18/33] Fixing interoperability_report after merging master --- interoperability_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interoperability_report.py b/interoperability_report.py index 400937fb..24604b3d 100644 --- a/interoperability_report.py +++ b/interoperability_report.py @@ -380,7 +380,7 @@ def run_publisher_shape_main( r'\[[0-9]+\]', # index = 0 'on_offered_deadline_missed()', # index = 1 re.compile('not supported', re.IGNORECASE), # index = 2 - pexpect.TIMEOUT # index = 3 + pexpect.TIMEOUT, # index = 3 pexpect.EOF # index = 4 ], timeout) From b87ec792782aa79fe831105fa932620283a3f36e Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 16 Feb 2026 19:05:56 +0100 Subject: [PATCH 19/33] Increasing timeout to 120mins --- .github/workflows/1_run_interoperability_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/1_run_interoperability_tests.yml b/.github/workflows/1_run_interoperability_tests.yml index 720883cf..9097f0e7 100644 --- a/.github/workflows/1_run_interoperability_tests.yml +++ b/.github/workflows/1_run_interoperability_tests.yml @@ -50,7 +50,7 @@ jobs: - name: Install Python requirements run: pip install --requirement requirements.txt - name: Run Interoperability script - timeout-minutes: 60 + timeout-minutes: 120 run: | publisher_exe=executables/${{ matrix.publisher }}*shape_main_linux subscriber_exe=executables/${{ matrix.subscriber }}*shape_main_linux From 544f2d82a39a031e2a094fd4cbc17c6d606b424b Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 16 Feb 2026 20:42:13 +0100 Subject: [PATCH 20/33] Move the CFT test that uses color to -c param as it is different in each product --- test_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_suite.py b/test_suite.py index 1dfe4c13..32d87549 100644 --- a/test_suite.py +++ b/test_suite.py @@ -491,7 +491,7 @@ # Content Filtered Topic 'Test_Cft_0' : { - 'apps' : ['-P -t Square -r -k 0 -c BLUE', '-P -t Square -r -k 0 -c RED', '-S -t Square -r -k 0 --cft "color = \'RED\'"'], + 'apps' : ['-P -t Square -r -k 0 -c BLUE', '-P -t Square -r -k 0 -c RED', '-S -t Square -r -k 0 -c RED'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK, ReturnCode.RECEIVING_FROM_ONE], 'check_function' : tsf.test_color_receivers, 'title' : 'Use of Content filter to avoid receiving undesired data (key)', From f1d80616a8f097dbf8fa4107f3a7863531dc4786 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 16 Feb 2026 21:21:28 +0100 Subject: [PATCH 21/33] Adding a vendor coverage of tests now that we have a way of detecting unsupported tests --- generate_xlsx_report.py | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index af54edca..c6300d5d 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -759,6 +759,13 @@ def add_data_summary_worksheet(self, worksheet.write( current_row, current_column + 4, 'Supported Tests Passed', self.__formats['bold_w_border']) + worksheet.write( + current_row, current_column + 5, + 'Vendor Coverage', self.__formats['bold_w_border']) + worksheet.write( + current_row, current_column + 6, + 'Vendor Coverage Passed', self.__formats['bold_w_border']) + current_row += 1 @@ -795,8 +802,38 @@ def add_data_summary_worksheet(self, str(value.get_supported_tests()), self.get_format_color(value.get_passed_tests(), value.get_supported_tests())) + + # vendor coverage + + # for the vendor coverage, we need to find the right data in the + # product_summary_dict, which is the one with the same product as + # publisher and subscriber + for key, value in self.__data.product_summary_dict.items(): + if product_name == key[0] and product_name == key[1]: + vendor_coverage_supported_tests = value.get_supported_tests() + vendor_coverage_total_tests = value.get_total_tests() + vendor_coverage_passed_tests = value.get_passed_tests() + + # vendor coverage + worksheet.write( + current_row, current_column + 5, + str(vendor_coverage_supported_tests) + ' / ' + + str(vendor_coverage_total_tests), + self.get_format_color( + vendor_coverage_supported_tests, + vendor_coverage_total_tests)) + # vendor coverage passed + worksheet.write( + current_row, current_column + 6, + str(vendor_coverage_passed_tests) + ' / ' + + str(vendor_coverage_supported_tests), + self.get_format_color( + vendor_coverage_passed_tests, + vendor_coverage_supported_tests)) + current_row += 1 + # Add 2 rows of gap for the next table current_row += 2 worksheet.write( @@ -903,6 +940,19 @@ def add_static_data_summary_worksheet(self, worksheet.write(current_row, starting_column + 1,'Documentation') worksheet.write(current_row, starting_column + 2, self.REPO_DOC) + # Add number of tests + + # Find the total number of unique tests by looking for a + # (publisher, subscriber) pair where both are the same + test_count = 0 + for key, value in self.__data.product_summary_dict.items(): + if key[0] == key[1]: + test_count = value.get_total_tests() + break + current_row += 1 + worksheet.write(current_row, starting_column + 1,'Unique tests count') + worksheet.write(current_row, starting_column + 2, test_count) + def add_static_data_description_worksheet(self, worksheet: xlsxwriter.Workbook.worksheet_class, name: str = 'Test Descriptions', From aa7dfee387b524d5553cb1592627e83015e4443e Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 17 Feb 2026 20:57:44 +0100 Subject: [PATCH 22/33] Documentation updates --- test_suite.py | 221 +++++++++++++++++++++++++++++--------------------- 1 file changed, 129 insertions(+), 92 deletions(-) diff --git a/test_suite.py b/test_suite.py index 32d87549..eabab950 100644 --- a/test_suite.py +++ b/test_suite.py @@ -133,7 +133,7 @@ ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the subscriber application receives samples and the value of the "size" member is always increasing\n\n' 'The test passes if the value of the "size" is always increasing in ' - f'{tsf.MAX_SAMPLES_READ} samples, even if there are missed samples (since reliability ' + f'{tsf.MAX_SAMPLES_READ} samples read, even if there are missed samples (since reliability ' 'is BEST_EFFORT) as long as there are no out-of-order or duplicated samples\n' }, @@ -183,8 +183,8 @@ ' * Configures the publisher and subscriber with history KEEP_ALL\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' - f'receives the next {tsf.MAX_SAMPLES_READ} subsequent samples, without losses or duplicates, in ' - 'the same order as sent\n' + f'receives the next {tsf.MAX_SAMPLES_READ} subsequent samples read, without ' + 'losses or duplicates, in the same order as sent\n' }, 'Test_Reliability_5' : { @@ -201,10 +201,9 @@ ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' - 'receives subsequent samples per instance, without losses or duplicates, in ' - 'the same order as sent\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'The test passes if the subscriber, after receiving a (first) sample from the ' + f'publisher, it receives {tsf.MAX_SAMPLES_READ} subsequent samples per ' + 'instance, without losses or duplicates, in the same order as sent\n' }, 'Test_History_0' : { @@ -224,9 +223,9 @@ ' * Configures the subscriber with a reading period of 200ms\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' - f'receives the next {tsf.MAX_SAMPLES_READ} subsequent samples, without losses or duplicates, in ' - 'the same order as sent.\n' + 'The test passes if the subscriber, after receiving a (first) sample from the ' + f'publisher, it receives the next {tsf.MAX_SAMPLES_READ} subsequent samples, ' + 'without losses or duplicates, in the same order as sent.\n' }, 'Test_History_1' : { @@ -247,10 +246,9 @@ ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber, after receiving a (first) sample from the publisher, it ' - 'receives subsequent samples per instance, without losses or duplicates, in ' - 'the same order as sent\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'The test passes if the subscriber, after receiving a (first) sample from the ' + f'publisher, it receives {tsf.MAX_SAMPLES_READ} subsequent samples per ' + 'instance, without losses or duplicates, in the same order as sent\n' }, # OWNERSHIP @@ -521,8 +519,8 @@ ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Publisher sends samples with size cycling from 1 to 50 (using --size-modulo 50 and -z 0)\n' - ' * Subscriber uses --cft "shapesize <= 20"\n' - ' * The test passes if all received samples have size < 20\n' + ' * Subscriber uses --cft "shapesize <= 20"\n\n' + f'The test passes if the subscriber receives {tsf.MAX_SAMPLES_READ/2} samples with size < 20\n' }, # PARTITION @@ -560,7 +558,8 @@ ' * Configures a second publisher to use PARTITION "x1" and "color" equal to "RED"\n' ' * Verifies that only the first publisher (PARTITION "p1") discovers and matches subscriber\n' ' * Verifies that the second publisher (PARTITION "x1") does not match the subscriber\n\n' - f'The test passes if the subscriber receives {tsf.MAX_SAMPLES_READ} samples of one color (first publisher)\n' + f'The test passes if the subscriber receives {tsf.MAX_SAMPLES_READ} samples of one color ' + '(first publisher)\n' }, # DURABILITY @@ -786,7 +785,7 @@ }, 'Test_TimeBasedFilter_0' : { - 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100', + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100', '-S -t Square -r -k 0 --time-filter 1000'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.test_reading_1_sample_every_10_samples_w_instances, @@ -798,13 +797,13 @@ ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber application receives a sample each 10 samples ' - 'the publisher application has sent\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/20}\n' + 'The test passes if the subscriber application receives a sample every 10 samples ' + 'the publisher application has sent. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/20} samples\n' }, 'Test_TimeBasedFilter_1' : { - 'apps' : ['-P -t Square -z 0 -r -k 0 --write-period 100 --num-instances 4', + 'apps' : ['-P -t Square -r -k 0 -z 0 --write-period 100 --num-instances 4', '-S -t Square -r -k 0 --time-filter 1000'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.test_reading_1_sample_every_10_samples_w_instances, @@ -818,9 +817,9 @@ ' * Configures the subscriber TIME_BASED_FILTER to 1 second\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber application receives a sample each 10 samples ' - 'the publisher application has sent per instance\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/20}\n' + 'The test passes if the subscriber application receives a sample every 10 samples ' + 'the publisher application has sent per instance. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/20} samples per instance\n' }, 'Test_FinalInstanceState_0' : { @@ -860,7 +859,7 @@ 'check_function' : tsf.test_unregistering_w_instances, 'title' : 'Test the instance state when finalizing the application', 'description' : 'Verifies a subscriber receives NOT_ALIVE_NO_WRITERS when the publisher finishes ' - 'without unregistering or disposing instances\n\n' + 'without unregistering or disposing instances explicitly\n\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher finishes after running 200 iterations\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' @@ -875,6 +874,8 @@ 'check_function' : tsf.test_large_data, 'title' : 'Test large data', 'description' : 'This test covers the interoperability scenario with large data:\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher to use 100000 additional payload size (to represent large data samples)\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The tests passes if the subscriber receives samples from the publisher and ' @@ -891,8 +892,8 @@ ' lifespan with fractions of seconds\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the subscriber with a reading period of 500ms\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' @@ -900,8 +901,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/10} samples\n' }, 'Test_Lifespan_1' : { @@ -914,8 +915,8 @@ ' lifespan with fraction of seconds and different instances\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the subscriber with a reading period of 500ms\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' @@ -924,8 +925,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time per instance\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time per instance. The subscriber has ' + f'to read {tsf.MAX_SAMPLES_READ/10} samples per instance\n' }, 'Test_Lifespan_2' : { @@ -938,8 +939,8 @@ ' lifespan in seconds\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the subscriber with a reading period of 2s\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' @@ -947,8 +948,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/10} samples per instance\n' }, 'Test_Lifespan_3' : { @@ -961,8 +962,8 @@ ' lifespan in seconds, and different instances\n\n' ' * Configures the publisher / subscriber with a RELIABLE reliability\n' ' * Configures the publisher / subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the subscriber with a reading period of 2s\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' @@ -971,8 +972,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time per instance\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time per instance. The subscriber has ' + f'to read {tsf.MAX_SAMPLES_READ/10} samples per instance\n' }, 'Test_Lifespan_4' : { @@ -985,8 +986,8 @@ ' lifespan with fractions of seconds\n\n' ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' ' * Configures the subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the subscriber with a reading period of 500ms\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' @@ -994,8 +995,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/10} samples\n' }, 'Test_Lifespan_5' : { @@ -1008,8 +1009,8 @@ ' lifespan with fraction of seconds and different instances\n\n' ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' ' * Configures the subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the publisher with a writing period of 100ms\n' + ' * Configures the publisher with a lifespan of 250ms\n' ' * Configures the subscriber with a reading period of 500ms\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' @@ -1018,8 +1019,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time per instance\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time per instance. The subscriber has ' + f'to read {tsf.MAX_SAMPLES_READ/10} samples per instance\n' }, 'Test_Lifespan_6' : { @@ -1032,8 +1033,8 @@ ' lifespan in seconds\n\n' ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' ' * Configures the subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the subscriber with a reading period of 2s\n' ' * The publisher application sends samples with increasing value of the "size" member\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' @@ -1041,8 +1042,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time. The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ/10} samples\n' }, 'Test_Lifespan_7' : { @@ -1055,8 +1056,8 @@ ' lifespan in seconds, and different instances\n\n' ' * Configures the publisher / subscriber with a BEST_EFFORT reliability\n' ' * Configures the subscriber with history KEEP_ALL\n' - ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the publisher with a writing period of 400ms\n' + ' * Configures the publisher with a lifespan of 1s\n' ' * Configures the subscriber with a reading period of 2s\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher application sends samples with increasing value of the "size" member\n' @@ -1065,8 +1066,8 @@ 'each time it reads.\n' 'As the publisher sets the lifespan, it expires in some samples before the ' 'subscriber reads data. The amount of consecutive samples read by the ' - 'subscriber application is 2 or 3 each time per instance\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ/10}\n' + 'subscriber application is 2 or 3 each time per instance. The subscriber has ' + f'to read {tsf.MAX_SAMPLES_READ/10} samples per instance\n' }, @@ -1078,6 +1079,8 @@ 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies an INSTANCE_PRESENTATION publisher is compatible with an ' 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with ordered access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' @@ -1093,6 +1096,8 @@ 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' 'TOPIC_PRESENTATION subscriber when using ordered access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with ordered access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' @@ -1108,6 +1113,8 @@ 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' 'GROUP_PRESENTATION subscriber when using ordered access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with ordered access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' @@ -1121,12 +1128,14 @@ 'title' : 'Compatibility for ordered access between TOPIC_PRESENTATION publisher and ' 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies a TOPIC_PRESENTATION publisher compatible with an ' - 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' - ' * Configures the publisher / subscriber with ordered access\n' - ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' - ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' - ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber receives samples from the publisher\n' + 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' }, 'Test_OrderedAccess_4' : { 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t', @@ -1135,12 +1144,14 @@ 'title' : 'Compatibility for ordered access between TOPIC_PRESENTATION publisher and ' 'TOPIC_PRESENTATION subscriber', 'description' : 'Verifies a TOPIC_PRESENTATION publisher compatible with a ' - 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' - ' * Configures the publisher / subscriber with ordered access\n' - ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' - ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' - ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber receives samples from the publisher\n' + 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' }, 'Test_OrderedAccess_5' : { 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t', @@ -1151,6 +1162,8 @@ 'description' : 'Verifies an TOPIC_PRESENTATION publisher does not match with a ' 'GROUP_PRESENTATION subscriber when using ordered access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with ordered access\n' ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' @@ -1164,12 +1177,14 @@ 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with an ' - 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' - ' * Configures the publisher / subscriber with ordered access\n' - ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' - ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' - ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber receives samples from the publisher\n' + 'INSTANCE_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' }, 'Test_OrderedAccess_7' : { 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope g', @@ -1178,12 +1193,14 @@ 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' 'TOPIC_PRESENTATION subscriber', 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with a ' - 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' - ' * Configures the publisher / subscriber with ordered access\n' - ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' - ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' - ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber receives samples from the publisher\n' + 'TOPIC_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' }, 'Test_OrderedAccess_8' : { 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope g', @@ -1192,12 +1209,14 @@ 'title' : 'Compatibility for ordered access between GROUP_PRESENTATION publisher and ' 'GROUP_PRESENTATION subscriber', 'description' : 'Verifies a GROUP_PRESENTATION publisher compatible with an ' - 'GROUP_PRESENTATION subscriber when using ordered access\n\n' - ' * Configures the publisher / subscriber with ordered access\n' - ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' - ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' - ' * Verifies the publisher and subscriber discover and match each other\n\n' - 'The test passes if the subscriber receives samples from the publisher\n' + 'GROUP_PRESENTATION subscriber when using ordered access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' + ' * Configures the publisher / subscriber with ordered access\n' + ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' + ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' + ' * Verifies the publisher and subscriber discover and match each other\n\n' + 'The test passes if the subscriber receives samples from the publisher\n' }, 'Test_OrderedAccess_9' : { 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t --num-instances 4 --write-period 100', @@ -1216,16 +1235,16 @@ ' * Configures the publisher with a writing period of 100ms\n' ' * Configures the first subscriber with access scope INSTANCE_PRESENTATION\n' ' * Configures the second subscriber with access scope TOPIC_PRESENTATION\n' - ' * Configures all subscribers with a reading period of 400ms\n' ' * Configures all subscribers to use take() instead of take_next_instance()\n' + ' * Configures all subscribers with a reading period of 400ms\n' ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber applications are receiving the instances ' 'with the specified ordered access / access scope. For INSTANCE_PRESENTATION, ' 'all the samples for one instance is presented sequentially. For ' 'TOPIC_PRESENTATION, the subscriber reads the samples in the same ' 'order they have been published, independently of the instance they' - 'belong to (one sample of one instance each time)\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'belong to (one sample of one instance each time). The subscriber has to read ' + f'{tsf.MAX_SAMPLES_READ} samples per instance\n' }, 'Test_CoherentSets_0' : { @@ -1236,6 +1255,8 @@ 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies an INSTANCE_PRESENTATION publisher is compatible with an ' 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' @@ -1251,6 +1272,8 @@ 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' 'TOPIC_PRESENTATION subscriber when using coherent access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' @@ -1266,6 +1289,8 @@ 'description' : 'Verifies an INSTANCE_PRESENTATION publisher does not match with a ' 'GROUP_PRESENTATION subscriber when using coherent access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with INSTANCE_PRESENTATION access_scope\n' ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' @@ -1280,6 +1305,8 @@ 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies an TOPIC_PRESENTATION publisher is compatible with an ' 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' @@ -1294,6 +1321,8 @@ 'TOPIC_PRESENTATION subscriber', 'description' : 'Verifies an TOPIC_PRESENTATION publisher is compatible with an ' 'TOPIC_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' @@ -1309,6 +1338,8 @@ 'description' : 'Verifies an TOPIC_PRESENTATION publisher does not match with a ' 'GROUP_PRESENTATION subscriber when using coherent access and ' 'report an IncompatibleQos notification\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with TOPIC_PRESENTATION access_scope\n' ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' @@ -1323,6 +1354,8 @@ 'INSTANCE_PRESENTATION subscriber', 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' 'INSTANCE_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' ' * Configures the subscriber with INSTANCE_PRESENTATION access_scope\n' @@ -1337,6 +1370,8 @@ 'TOPIC_PRESENTATION subscriber', 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' 'TOPIC_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' ' * Configures the subscriber with TOPIC_PRESENTATION access_scope\n' @@ -1351,6 +1386,8 @@ 'GROUP_PRESENTATION subscriber', 'description' : 'Verifies an GROUP_PRESENTATION publisher is compatible with an ' 'GROUP_PRESENTATION subscriber when using coherent access\n\n' + ' * Configures the publisher / subscriber with a RELIABLE reliability\n' + ' * Configures the publisher / subscriber with history KEEP_ALL\n' ' * Configures the publisher / subscriber with coherent access\n' ' * Configures the publisher with GROUP_PRESENTATION access_scope\n' ' * Configures the subscriber with GROUP_PRESENTATION access_scope\n' @@ -1359,7 +1396,7 @@ }, 'Test_CoherentSets_9' : { - 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope i --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.coherent_sets_w_instances, @@ -1381,11 +1418,11 @@ 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' - 'per instance and topic is 3 each time we receive a new coherent set of samples\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'per instance and topic is 3 each time we receive a new coherent set of samples. ' + f'The subscriber has to read {tsf.MAX_SAMPLES_READ} samples per instance\n' }, 'Test_CoherentSets_10' : { - 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope t --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.coherent_sets_w_instances, @@ -1407,11 +1444,11 @@ 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' - 'per instance and topic is 3 each time we receive a new coherent set of samples\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'per instance and topic is 3 each time we receive a new coherent set of samples. ' + f'The subscriber has to read {tsf.MAX_SAMPLES_READ} samples per instance\n' }, 'Test_CoherentSets_11' : { - 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g --num-instances 4 --num-topics 3 --coherent-sample-count 3 --write-period 100 -z 0', + 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope g --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.coherent_sets_w_instances, @@ -1433,7 +1470,7 @@ 'The test passes if the subscriber application receives all samples of a coherent ' 'set at the same. This coherent set is created with 36 samples: 3 samples per ' 'instance, 4 instances and 3 topics. This test checks that the consecutive samples received ' - 'per instance and topic is 3 each time we receive a new coherent set of samples\n' - f'Max amount of samples read is {tsf.MAX_SAMPLES_READ}\n' + 'per instance and topic is 3 each time we receive a new coherent set of samples. ' + f'The subscriber has to read {tsf.MAX_SAMPLES_READ} samples per instance\n' }, } From 9c2c9e8bb0c56e452aad86170f3e3015d684ffc1 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Wed, 18 Feb 2026 17:25:59 +0100 Subject: [PATCH 23/33] Added tests for compatibility between ordered and coherent access. Also changed ordered access check function --- test_suite.py | 42 ++++++++++++++++++++--- test_suite_functions.py | 76 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/test_suite.py b/test_suite.py index eabab950..15f79ba2 100644 --- a/test_suite.py +++ b/test_suite.py @@ -1218,12 +1218,28 @@ ' * Verifies the publisher and subscriber discover and match each other\n\n' 'The test passes if the subscriber receives samples from the publisher\n' }, + 'Test_OrderedAccess_9' : { - 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t --num-instances 4 --write-period 100', + 'apps' : ['-P -t Square -r -k 0 --access-scope t', + '-S -t Square -r -k 0 --ordered --access-scope t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for no ordered access publisher and ordered access subscriber', + 'description' : 'Verifies that publisher without ordered access does not match with a ' + 'subscriber with ordered access\n\n' + ' * Configures the publisher and the subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and the subscriber with history KEEP_ALL\n' + ' * Configures the publisher and the subscriber with access scope TOPIC_PRESENTATION\n' + ' * Configures the subscriber with ordered access\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_OrderedAccess_10' : { + 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t -z 0 --num-instances 4 --write-period 100', '-S -t Square -r -k 0 --ordered --access-scope i --take-read --read-period 400', '-S -t Square -r -k 0 --ordered --access-scope t --take-read --read-period 400'], - 'expected_codes' : [ReturnCode.OK, ReturnCode.ORDERED_ACCESS_INSTANCE, ReturnCode.ORDERED_ACCESS_TOPIC], - 'check_function' : tsf.ordered_access_w_instances, + 'expected_codes' : [ReturnCode.OK, ReturnCode.OK, ReturnCode.OK], + 'check_function' : tsf.test_order_w_instances, 'title' : 'Test the behavior of ordered access', 'description' : 'Verifies subscribers receives data correctly depending on the ordered ' 'access: TOPIC_PRESENTATION and INSTANCE_PRESENTATION.\n\n' @@ -1231,6 +1247,7 @@ ' * Configures the publisher and all subscribers with history KEEP_ALL\n' ' * Configures the publisher and all subscribers with ordered access\n' ' * Configures the publisher with access scope TOPIC_PRESENTATION\n' + ' * The publisher application sends samples with increasing value of the "size" member\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * Configures the publisher with a writing period of 100ms\n' ' * Configures the first subscriber with access scope INSTANCE_PRESENTATION\n' @@ -1396,6 +1413,21 @@ }, 'Test_CoherentSets_9' : { + 'apps' : ['-P -t Square -r -k 0 --access-scope t', + '-S -t Square -r -k 0 --coherent --access-scope t'], + 'expected_codes' : [ReturnCode.INCOMPATIBLE_QOS, ReturnCode.INCOMPATIBLE_QOS], + 'title' : 'No compatibility for no coherent access publisher and coherent access subscriber', + 'description' : 'Verifies that publisher without coherent access does not match with a ' + 'subscriber with coherent access\n\n' + ' * Configures the publisher and the subscriber with a RELIABLE reliability\n' + ' * Configures the publisher and the subscriber with history KEEP_ALL\n' + ' * Configures the publisher and the subscriber with access scope TOPIC_PRESENTATION\n' + ' * Configures the subscriber with coherent access\n' + 'The test passes if the listeners trigger the IncompatibleQos notification in the publisher ' + 'and the subscriber\n' + }, + + 'Test_CoherentSets_10' : { 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope i --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope i --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], @@ -1421,7 +1453,7 @@ 'per instance and topic is 3 each time we receive a new coherent set of samples. ' f'The subscriber has to read {tsf.MAX_SAMPLES_READ} samples per instance\n' }, - 'Test_CoherentSets_10' : { + 'Test_CoherentSets_11' : { 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope t --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope t --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], @@ -1447,7 +1479,7 @@ 'per instance and topic is 3 each time we receive a new coherent set of samples. ' f'The subscriber has to read {tsf.MAX_SAMPLES_READ} samples per instance\n' }, - 'Test_CoherentSets_11' : { + 'Test_CoherentSets_12' : { 'apps' : ['-P -t Square -r -k 0 --coherent --access-scope g --num-topics 3 --num-instances 4 --coherent-sample-count 3 --write-period 100 -z 0', '-S -t Square -r -k 0 --coherent --access-scope g --num-topics 3 --take-read --read-period 100'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], diff --git a/test_suite_functions.py b/test_suite_functions.py index f873503b..4dab23b4 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -937,6 +937,82 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code +def test_order_w_instances(child_sub, samples_sent, last_sample_saved, timeout): + """ + This function tests that the subscriber receives the samples in order + (for several instances) + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern. + """ + + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode + + produced_code = ReturnCode.DATA_NOT_RECEIVED + + instance_color = [] + instance_seq_num = [] + first_iteration = [] + samples_read_per_instance = 0 + max_samples_received = MAX_SAMPLES_READ + + while samples_read_per_instance < max_samples_received: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + instance_seq_num.append(int(sub_string.group(2))) + first_iteration.append(True) + + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + if first_iteration[index]: + first_iteration[index] = False + else: + # check that the next sequence number is the next value + current_size = int(sub_string.group(2)) + if (current_size > instance_seq_num[index]): + instance_seq_num[index] = current_size + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + + # Get the next sample the subscriber is receiving + index = child_sub.expect( + [ + '\[[0-9]+\]', # index = 0 + r'Reading with ordered access.*?\n', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 + ], + timeout + ) + if index == 0: + if sub_string is not None and sub_string.group(1) == instance_color[0]: + samples_read_per_instance += 1 + elif index == 2: + # no more data to process + break + elif index == 3: + return ReturnCode.DATA_NOT_RECEIVED + + if max_samples_received == samples_read_per_instance: + produced_code = ReturnCode.OK + + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') + return produced_code + + def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests that coherent sets works correctly. This counts the From 6808dccd828a476df20d8f97ff4c5ee679865851 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Tue, 24 Feb 2026 19:16:29 +0100 Subject: [PATCH 24/33] Fixed ordered access test and coherent set tests. Also modified the test_order to accept instances --- test_suite.py | 6 +- test_suite_functions.py | 184 +++++++++++++++------------------------- 2 files changed, 71 insertions(+), 119 deletions(-) diff --git a/test_suite.py b/test_suite.py index 15f79ba2..c4b5d386 100644 --- a/test_suite.py +++ b/test_suite.py @@ -124,7 +124,7 @@ 'Test_Reliability_0' : { 'apps' : ['-P -t Square -b -z 0', '-S -t Square -b'], 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_reliability_order, + 'check_function' : tsf.test_order_w_instances, 'title' : 'Communication between BEST_EFFORT publisher and subscriber', 'description' : 'Verifies a best effort publisher communicates with a best effort subscriber with no out-of-order ' 'or duplicate samples\n\n' @@ -1238,8 +1238,8 @@ 'apps' : ['-P -t Square -r -k 0 --ordered --access-scope t -z 0 --num-instances 4 --write-period 100', '-S -t Square -r -k 0 --ordered --access-scope i --take-read --read-period 400', '-S -t Square -r -k 0 --ordered --access-scope t --take-read --read-period 400'], - 'expected_codes' : [ReturnCode.OK, ReturnCode.OK, ReturnCode.OK], - 'check_function' : tsf.test_order_w_instances, + 'expected_codes' : [ReturnCode.OK, ReturnCode.ORDERED_ACCESS_INSTANCE, ReturnCode.ORDERED_ACCESS_TOPIC], + 'check_function' : tsf.ordered_access_w_instances, 'title' : 'Test the behavior of ordered access', 'description' : 'Verifies subscribers receives data correctly depending on the ordered ' 'access: TOPIC_PRESENTATION and INSTANCE_PRESENTATION.\n\n' diff --git a/test_suite_functions.py b/test_suite_functions.py index 4dab23b4..fdd52c20 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -175,16 +175,14 @@ def test_size_less_than_20(child_sub, samples_sent, last_sample_saved, timeout): print(f'Samples read: {samples_read}') return return_code - -def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): +def test_order_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ - This function tests reliability, it checks whether the subscriber receives - the samples in order. - + This function tests that the subscriber receives the samples in order + (for several instances) child_sub: child program generated with pexpect samples_sent: not used last_sample_saved: not used - timeout: not used + timeout: time pexpect waits until it matches a pattern. """ basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) @@ -194,48 +192,62 @@ def test_reliability_order(child_sub, samples_sent, last_sample_saved, timeout): produced_code = ReturnCode.DATA_NOT_RECEIVED - # Read the first sample printed by the subscriber - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', - child_sub.before + child_sub.after) - last_size = 0 - + instance_color = [] + instance_seq_num = [] + first_iteration = [] + samples_read_per_instance = 0 max_samples_received = MAX_SAMPLES_READ - samples_read = 0 - while sub_string is not None and samples_read < max_samples_received: - current_size = int(sub_string.group(1)) - if (current_size > last_size): - last_size = current_size - else: - produced_code = ReturnCode.DATA_NOT_CORRECT - break + while samples_read_per_instance < max_samples_received: + sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', + child_sub.before + child_sub.after) + + if sub_string is not None: + # add a new instance to instance_color + if sub_string.group(1) not in instance_color: + instance_color.append(sub_string.group(1)) + instance_seq_num.append(int(sub_string.group(2))) + first_iteration.append(True) + + if sub_string.group(1) in instance_color: + index = instance_color.index(sub_string.group(1)) + if first_iteration[index]: + first_iteration[index] = False + else: + # check that the next sequence number is the next value + current_size = int(sub_string.group(2)) + if (current_size > instance_seq_num[index]): + instance_seq_num[index] = current_size + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break + else: + produced_code = ReturnCode.DATA_NOT_CORRECT + break # Get the next sample the subscriber is receiving index = child_sub.expect( [ '\[[0-9]+\]', # index = 0 - pexpect.TIMEOUT, # index = 1 - pexpect.EOF # index = 2 + r'Reading with ordered access.*?\n', # index = 1 + pexpect.TIMEOUT, # index = 2 + pexpect.EOF # index = 3 ], timeout ) - if index == 1: - # no more data to process - break + if index == 0: + if sub_string is not None and sub_string.group(1) == instance_color[0]: + samples_read_per_instance += 1 elif index == 2: - produced_code = ReturnCode.DATA_NOT_RECEIVED + # no more data to process break + elif index == 3: + return ReturnCode.DATA_NOT_RECEIVED - samples_read += 1 - - # search the next received sample by the subscriber app - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', - child_sub.before + child_sub.after) - - if samples_read == max_samples_received: + if max_samples_received == samples_read_per_instance: produced_code = ReturnCode.OK - print(f'Samples read: {samples_read}') + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_saved, timeout): @@ -807,7 +819,7 @@ def test_lifespan_2_3_consecutive_samples_w_instances(child_sub, samples_sent, l if max_samples_lifespan == samples_read_per_instance: produced_code = ReturnCode.OK - print(f'Samples read: {samples_read_per_instance}, instances: {instance_color}') + print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeout): @@ -937,82 +949,6 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') return produced_code -def test_order_w_instances(child_sub, samples_sent, last_sample_saved, timeout): - """ - This function tests that the subscriber receives the samples in order - (for several instances) - child_sub: child program generated with pexpect - samples_sent: not used - last_sample_saved: not used - timeout: time pexpect waits until it matches a pattern. - """ - - basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) - - if basic_check_retcode != ReturnCode.OK: - return basic_check_retcode - - produced_code = ReturnCode.DATA_NOT_RECEIVED - - instance_color = [] - instance_seq_num = [] - first_iteration = [] - samples_read_per_instance = 0 - max_samples_received = MAX_SAMPLES_READ - - while samples_read_per_instance < max_samples_received: - sub_string = re.search(r'\w+\s+(\w+)\s+[0-9]+\s+[0-9]+\s+\[([0-9]+)\]', - child_sub.before + child_sub.after) - - if sub_string is not None: - # add a new instance to instance_color - if sub_string.group(1) not in instance_color: - instance_color.append(sub_string.group(1)) - instance_seq_num.append(int(sub_string.group(2))) - first_iteration.append(True) - - if sub_string.group(1) in instance_color: - index = instance_color.index(sub_string.group(1)) - if first_iteration[index]: - first_iteration[index] = False - else: - # check that the next sequence number is the next value - current_size = int(sub_string.group(2)) - if (current_size > instance_seq_num[index]): - instance_seq_num[index] = current_size - else: - produced_code = ReturnCode.DATA_NOT_CORRECT - break - else: - produced_code = ReturnCode.DATA_NOT_CORRECT - break - - # Get the next sample the subscriber is receiving - index = child_sub.expect( - [ - '\[[0-9]+\]', # index = 0 - r'Reading with ordered access.*?\n', # index = 1 - pexpect.TIMEOUT, # index = 2 - pexpect.EOF # index = 3 - ], - timeout - ) - if index == 0: - if sub_string is not None and sub_string.group(1) == instance_color[0]: - samples_read_per_instance += 1 - elif index == 2: - # no more data to process - break - elif index == 3: - return ReturnCode.DATA_NOT_RECEIVED - - if max_samples_received == samples_read_per_instance: - produced_code = ReturnCode.OK - - print(f'Samples read per instance: {samples_read_per_instance}, instances: {instance_color}') - return produced_code - - def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeout): """ This function tests that coherent sets works correctly. This counts the @@ -1041,6 +977,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou first_time_reading = True ignore_firsts_coherent_set = 2 coherent_sets_count = 0 + coherent_set_sample_count = 0 coherent_sets_max_count = MAX_SAMPLES_READ / 5 # 100 while samples_read_per_instance < coherent_sets_max_count: @@ -1060,7 +997,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou if topic_name not in topics: topics[topic_name] = {} if instance_color not in topics[topic_name]: - topics[topic_name][instance_color] = 1 + topics[topic_name][instance_color] = None # if the instance is already added if instance_color in topics[topic_name]: # the instance exists @@ -1068,6 +1005,8 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # check the previous color and increase consecutive samples if previous_sample_color is not None: if current_color == previous_sample_color: + if topics[topic_name][instance_color] is None: + topics[topic_name][instance_color] = 1 topics[topic_name][instance_color] += 1 previous_sample_color = current_color # different message than a sample @@ -1080,29 +1019,41 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # if DataReader has received samples if ignore_firsts_coherent_set != 0 and new_coherent_set_read: ignore_firsts_coherent_set -= 1 + coherent_set_sample_count = 0 for topic in topics: for color in topics[topic]: - topics[topic][color] = 1 + topics[topic][color] = None elif new_coherent_set_read: # the test is only ok if it has received coherent sets produced_code = ReturnCode.OK + # each set has 4 instances, 3 samples per instance, 3 topics + # therefore, each set contains 4*3*3 = 36 samples, if the count + # is different than 36, it means that there is an error + if coherent_set_sample_count != 36: + print(f'Coherent set sample count is {coherent_set_sample_count} instead of 36') + produced_code = ReturnCode.DATA_NOT_CORRECT + break + else: + coherent_set_sample_count = 0 for topic in topics: for color in topics[topic]: if first_time_reading: # with group presentation we may get several coherent # sets at the beginning, just checking that the samples - # received are multiple of 3 - if topics[topic][color] % 3 != 0: + # received are multiple of 3 (coherent set count) + if topics[topic][color] is not None and topics[topic][color] % 3 != 0: + print(f'Coherent set count for topic {topic} and instance {color} is {topics[topic][color]} instead of 3') produced_code = ReturnCode.DATA_NOT_CORRECT break else: # there should be 3 consecutive samples per instance, # as the test specifies this with the argument # --coherent-sample-count 3 - if topics[topic][color] != 3: + if topics[topic][color] is not None and topics[topic][color] != 3: + print(f'Coherent set count for topic {topic} and instance {color} is {topics[topic][color]} instead of 3') produced_code = ReturnCode.DATA_NOT_CORRECT break - topics[topic][color] = 1 + topics[topic][color] = None if produced_code == ReturnCode.DATA_NOT_CORRECT: break first_time_reading = False @@ -1125,6 +1076,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou and topic_name == list(topics.keys())[0] and instance_color == list(topics[topic_name].keys())[0]): samples_read_per_instance += 1 + coherent_set_sample_count += 1 elif index == 1: coherent_sets_count += 1 elif index == 2: From 4694d816ddef57dfb7251699a30ea042bb1b45bd Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Thu, 26 Feb 2026 17:38:12 +0100 Subject: [PATCH 25/33] Updated the generated xlsx to show the summary in two columns --- generate_xlsx_report.py | 106 ++++++++++++++++------------------------ test_suite.py | 4 +- 2 files changed, 45 insertions(+), 65 deletions(-) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index c6300d5d..47d47c32 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -404,7 +404,7 @@ def create_summary_worksheet(self, name: str = 'Summary'): # rows. # The tables leave the first column (value 0) as gap self.add_data_summary_worksheet( - starting_row=9, + starting_row=11, starting_column=1, worksheet=summary_worksheet) # After having all data that may have an unknown length, we call @@ -752,20 +752,10 @@ def add_data_summary_worksheet(self, 'Product', self.__formats['bold_w_border']) worksheet.write( current_row, current_column + 2, - 'Tests Passed', self.__formats['bold_w_border']) + 'Product Coverage', self.__formats['bold_w_border']) worksheet.write( current_row, current_column + 3, - 'Supported Tests', self.__formats['bold_w_border']) - worksheet.write( - current_row, current_column + 4, - 'Supported Tests Passed', self.__formats['bold_w_border']) - worksheet.write( - current_row, current_column + 5, - 'Vendor Coverage', self.__formats['bold_w_border']) - worksheet.write( - current_row, current_column + 6, - 'Vendor Coverage Passed', self.__formats['bold_w_border']) - + 'Total Tests', self.__formats['bold_w_border']) current_row += 1 @@ -781,65 +771,42 @@ def add_data_summary_worksheet(self, current_row, current_column + 1, product_name, self.__formats['bold_w_border']) - # test passed - worksheet.write( - current_row, current_column + 2, - str(value.get_passed_tests()) + ' / ' + - str(value.get_total_tests()), - self.get_format_color(value.get_passed_tests(), - value.get_total_tests())) - # supported tests - worksheet.write( - current_row, current_column + 3, - str(value.get_supported_tests()) + ' / ' + - str(value.get_total_tests()), - self.__formats['result_yellow'] if value.get_unsupported_tests() > 0 - else self.__formats['result_green']) - # supported tests passed - worksheet.write( - current_row, current_column + 4, - str(value.get_passed_tests()) + ' / ' + - str(value.get_supported_tests()), - self.get_format_color(value.get_passed_tests(), - value.get_supported_tests())) # vendor coverage # for the vendor coverage, we need to find the right data in the # product_summary_dict, which is the one with the same product as # publisher and subscriber - for key, value in self.__data.product_summary_dict.items(): - if product_name == key[0] and product_name == key[1]: - vendor_coverage_supported_tests = value.get_supported_tests() - vendor_coverage_total_tests = value.get_total_tests() - vendor_coverage_passed_tests = value.get_passed_tests() + for product_key, product_value in self.__data.product_summary_dict.items(): + if product_name == product_key[0] and product_name == product_key[1]: + product_coverage_supported_tests = product_value.get_supported_tests() + product_coverage_total_tests = product_value.get_total_tests() + product_coverage_passed_tests = product_value.get_passed_tests() # vendor coverage worksheet.write( - current_row, current_column + 5, - str(vendor_coverage_supported_tests) + ' / ' + - str(vendor_coverage_total_tests), - self.get_format_color( - vendor_coverage_supported_tests, - vendor_coverage_total_tests)) - # vendor coverage passed - worksheet.write( - current_row, current_column + 6, - str(vendor_coverage_passed_tests) + ' / ' + - str(vendor_coverage_supported_tests), + current_row, current_column + 2, + str(product_coverage_passed_tests) + ' / ' + + str(product_coverage_supported_tests) + ' / ' + + str(product_coverage_total_tests), self.get_format_color( - vendor_coverage_passed_tests, - vendor_coverage_supported_tests)) + product_coverage_supported_tests, + product_coverage_total_tests)) + + # total tests + worksheet.write( + current_row, current_column + 3, + str(value.get_passed_tests()) + ' / ' + + str(value.get_supported_tests()) + ' / ' + + str(value.get_total_tests()), + self.get_format_color( + value.get_passed_tests(), value.get_supported_tests())) current_row += 1 # Add 2 rows of gap for the next table current_row += 2 - worksheet.write( - current_row, current_column, - 'Test Result: passed / supported / total', self.__formats['bold_w_border']) - current_row += 1 worksheet.write( current_row, current_column, 'Publisher (row)/Subscriber (column)', self.__formats['bold_w_border']) @@ -856,7 +823,7 @@ def add_data_summary_worksheet(self, # Add the table passed_tests/total_tests with all combinations of product # as publishers and as subscribers - for (publisher_name, subscriber_name), value in self.__data.product_summary_dict.items(): + for (publisher_name, subscriber_name), product_value in self.__data.product_summary_dict.items(): # if the publisher hasn't been already processed yet, determine # what is the process_row by selecting the next free row # (current_row+1) @@ -886,10 +853,10 @@ def add_data_summary_worksheet(self, process_column = column_dict[subscriber_name] worksheet.write(process_row, process_column, - str(value.get_passed_tests()) + ' / ' + - str(value.get_supported_tests()) + ' / ' + - str(value.get_total_tests()), - self.get_format_color(value.get_passed_tests(), value.get_supported_tests())) + str(product_value.get_passed_tests()) + ' / ' + + str(product_value.get_supported_tests()) + ' / ' + + str(product_value.get_total_tests()), + self.get_format_color(product_value.get_passed_tests(), product_value.get_supported_tests())) def add_static_data_summary_worksheet(self, worksheet: xlsxwriter.Workbook.worksheet_class, @@ -922,10 +889,9 @@ def add_static_data_summary_worksheet(self, worksheet.insert_image( row=current_row, col=starting_column, filename=dds_logo_path, - options={'x_scale': 0.4, 'y_scale': 0.4, 'decorative': True, 'object_position': 2}) + options={'x_scale': 0.6, 'y_scale': 0.6, 'decorative': True, 'object_position': 2}) # Add date - current_row += 1 worksheet.write(current_row, starting_column + 1, 'Date') date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') worksheet.write(current_row, starting_column + 2, date_time) @@ -953,6 +919,20 @@ def add_static_data_summary_worksheet(self, worksheet.write(current_row, starting_column + 1,'Unique tests count') worksheet.write(current_row, starting_column + 2, test_count) + # add legend + current_row += 1 + worksheet.write(current_row, starting_column + 1, + 'Test results expressed in: test passed / supported / total', + self.__formats['bold']) + current_row += 1 + worksheet.write(current_row, starting_column + 1, + '"Product Coverage" contains information about unique tests supported by product', + self.__formats['bold']) + current_row += 1 + worksheet.write(current_row, starting_column + 1, + '"Total Tests" contains all test performed between products as publisher and subscriber', + self.__formats['bold']) + def add_static_data_description_worksheet(self, worksheet: xlsxwriter.Workbook.worksheet_class, name: str = 'Test Descriptions', diff --git a/test_suite.py b/test_suite.py index c4b5d386..8eed85bf 100644 --- a/test_suite.py +++ b/test_suite.py @@ -828,7 +828,7 @@ 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.test_unregistering_w_instances, 'title' : 'Test the behavior of unregistering instances', - 'description' : 'Verifies a subscriber receives NOT_ALIVE_NO_WRITERS when the publisher unregister instances\n\n' + 'description' : 'Verifies a subscriber receives NOT_ALIVE_NO_WRITERS when the publisher unregisters instances\n\n' ' * Configures the publisher to unregister instances upon finalization\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher finishes after running 200 iterations\n' @@ -843,7 +843,7 @@ 'expected_codes' : [ReturnCode.OK, ReturnCode.OK], 'check_function' : tsf.test_disposing_w_instances, 'title' : 'Test the behavior of disposing instances', - 'description' : 'Verifies a subscriber receives NOT_ALIVE_DISPOSED when the publisher unregister instances\n\n' + 'description' : 'Verifies a subscriber receives NOT_ALIVE_DISPOSED when the publisher disposes instances\n\n' ' * Configures the publisher to dispose instances upon finalization\n' ' * The publisher publishes 4 different instances (using the same data value)\n' ' * The publisher finishes after running 200 iterations\n' From 81881e4e2c2201548726be2cc37b5faf86e76c6b Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 27 Feb 2026 14:41:01 +0100 Subject: [PATCH 26/33] Use current repo executables for testing and and added an error if a executable does not exist --- .github/workflows/1_run_interoperability_tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/1_run_interoperability_tests.yml b/.github/workflows/1_run_interoperability_tests.yml index 9097f0e7..b8c75f7b 100644 --- a/.github/workflows/1_run_interoperability_tests.yml +++ b/.github/workflows/1_run_interoperability_tests.yml @@ -41,7 +41,6 @@ jobs: - name: Download shape_main executables uses: robinraju/release-downloader@v1.10 with: - repository: omg-dds/dds-rtps latest: true fileName: "*" out-file-path: zipped_executables @@ -56,6 +55,16 @@ jobs: subscriber_exe=executables/${{ matrix.subscriber }}*shape_main_linux output_file=junit_report-${{ matrix.publisher }}-${{ matrix.subscriber }}.xml extra_args="" + # Check if publisher executable exists + if ! ls $publisher_exe 1> /dev/null 2>&1; then + echo "ERROR: Publisher executable not found: $publisher_exe" + exit 1 + fi + # Check if subscriber executable exists + if ! ls $subscriber_exe 1> /dev/null 2>&1; then + echo "ERROR: Subscriber executable not found: $subscriber_exe" + exit 1 + fi if [[ "${subscriber_exe,,}" == *opendds* && "${publisher_exe,,}" == *connext_dds* ]]; then extra_args="--periodic-announcement 5000" fi From 9e93e8fe6d68ee44b37018301b6ca6c24774f2d5 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Fri, 27 Feb 2026 15:12:10 +0100 Subject: [PATCH 27/33] Added continue-on-error in case a executable is not found --- .github/workflows/1_run_interoperability_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/1_run_interoperability_tests.yml b/.github/workflows/1_run_interoperability_tests.yml index b8c75f7b..57fb7914 100644 --- a/.github/workflows/1_run_interoperability_tests.yml +++ b/.github/workflows/1_run_interoperability_tests.yml @@ -50,6 +50,7 @@ jobs: run: pip install --requirement requirements.txt - name: Run Interoperability script timeout-minutes: 120 + continue-on-error: true run: | publisher_exe=executables/${{ matrix.publisher }}*shape_main_linux subscriber_exe=executables/${{ matrix.subscriber }}*shape_main_linux From 4c38488f2ec967e38c08d9b3b15a84139570f97e Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Thu, 5 Mar 2026 21:47:11 +0100 Subject: [PATCH 28/33] Modified legend of the xlsx report --- generate_xlsx_report.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index 47d47c32..5ba18dd2 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -404,7 +404,7 @@ def create_summary_worksheet(self, name: str = 'Summary'): # rows. # The tables leave the first column (value 0) as gap self.add_data_summary_worksheet( - starting_row=11, + starting_row=13, starting_column=1, worksheet=summary_worksheet) # After having all data that may have an unknown length, we call @@ -752,10 +752,10 @@ def add_data_summary_worksheet(self, 'Product', self.__formats['bold_w_border']) worksheet.write( current_row, current_column + 2, - 'Product Coverage', self.__formats['bold_w_border']) + 'Single-Product Tests', self.__formats['bold_w_border']) worksheet.write( current_row, current_column + 3, - 'Total Tests', self.__formats['bold_w_border']) + 'Cross Product Tests', self.__formats['bold_w_border']) current_row += 1 @@ -920,18 +920,24 @@ def add_static_data_summary_worksheet(self, worksheet.write(current_row, starting_column + 2, test_count) # add legend - current_row += 1 + current_row += 2 worksheet.write(current_row, starting_column + 1, - 'Test results expressed in: test passed / supported / total', + 'Single-Product Tests', self.__formats['bold']) + worksheet.write(current_row, starting_column + 2, + 'Results where each product is tested only with itself') current_row += 1 worksheet.write(current_row, starting_column + 1, - '"Product Coverage" contains information about unique tests supported by product', + 'Cross-Product Tests', self.__formats['bold']) + worksheet.write(current_row, starting_column + 2, + 'Results where each product is tested against all others (including itself)') current_row += 1 worksheet.write(current_row, starting_column + 1, - '"Total Tests" contains all test performed between products as publisher and subscriber', + 'Format X / Y / Z', self.__formats['bold']) + worksheet.write(current_row, starting_column + 2, + 'Represents test passed / supported / total') def add_static_data_description_worksheet(self, worksheet: xlsxwriter.Workbook.worksheet_class, From 1eebefea5eabffacc79bdc3fb284086e1df1a5d5 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 9 Mar 2026 17:33:07 +0100 Subject: [PATCH 29/33] Fixing warnings with python strings --- test_suite_functions.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test_suite_functions.py b/test_suite_functions.py index fdd52c20..7b4e3f95 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -34,7 +34,7 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + sub_string = re.search(r'\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) if sub_string is None: return ReturnCode.DATA_NOT_RECEIVED @@ -64,7 +64,7 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -78,7 +78,7 @@ def test_size_receivers(child_sub, samples_sent, last_sample_saved, timeout): samples_read += 1 - sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + sub_string = re.search(r'\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) print(f'Samples read: {samples_read}') @@ -101,7 +101,7 @@ def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): if basic_check_retcode != ReturnCode.OK: return basic_check_retcode - sub_string = re.search('\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', + sub_string = re.search(r'\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', child_sub.before + child_sub.after) first_sample_color = sub_string.group(1) @@ -117,7 +117,7 @@ def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -131,7 +131,7 @@ def test_color_receivers(child_sub, samples_sent, last_sample_saved, timeout): samples_read += 1 - sub_string = re.search('\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', + sub_string = re.search(r'\w\s+(\w+)\s+[0-9]+ [0-9]+ \[[0-9]+\]', child_sub.before + child_sub.after) print(f'Samples read: {samples_read}') @@ -149,7 +149,7 @@ def test_size_less_than_20(child_sub, samples_sent, last_sample_saved, timeout): samples_read = 0 return_code = ReturnCode.OK - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) + sub_string = re.search(r'[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) while sub_string is not None and samples_read < max_samples_received: size = int(sub_string.group(1)) @@ -159,7 +159,7 @@ def test_size_less_than_20(child_sub, samples_sent, last_sample_saved, timeout): index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -170,7 +170,7 @@ def test_size_less_than_20(child_sub, samples_sent, last_sample_saved, timeout): break samples_read += 1 - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) + sub_string = re.search(r'[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) print(f'Samples read: {samples_read}') return return_code @@ -228,7 +228,7 @@ def test_order_w_instances(child_sub, samples_sent, last_sample_saved, timeout): # Get the next sample the subscriber is receiving index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 r'Reading with ordered access.*?\n', # index = 1 pexpect.TIMEOUT, # index = 2 pexpect.EOF # index = 3 @@ -301,7 +301,7 @@ def test_reliability_no_losses_w_instances(child_sub, samples_sent, last_sample_ # Get the next sample the subscriber is receiving index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -347,7 +347,7 @@ def test_durability_volatile(child_sub, samples_sent, last_sample_saved, timeout # Read the first sample, if it has the size > 5, it is using volatile # durability correctly - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', + sub_string = re.search(r'[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) # Check if the element received is not the first 5 samples (aka size >= 5) @@ -384,7 +384,7 @@ def test_durability_transient_local(child_sub, samples_sent, last_sample_saved, # Read the first sample, if it has the size == 1, it is using transient # local durability correctly - sub_string = re.search('[0-9]+ [0-9]+ \[([0-9]+)\]', + sub_string = re.search(r'[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) # Check if the element is the first one sent (aka size == 1), which should @@ -430,7 +430,7 @@ def test_deadline_missed(child_sub, samples_sent, last_sample_saved, timeout): return ReturnCode.DATA_NOT_RECEIVED else: index = child_sub.expect([ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -506,7 +506,7 @@ def test_reading_1_sample_every_10_samples_w_instances(child_sub, samples_sent, # Get the next sample the subscriber is receiving index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -800,7 +800,7 @@ def test_lifespan_2_3_consecutive_samples_w_instances(child_sub, samples_sent, l # Get the next sample the subscriber is receiving index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 pexpect.TIMEOUT, # index = 1 pexpect.EOF # index = 2 ], @@ -920,7 +920,7 @@ def ordered_access_w_instances(child_sub, samples_sent, last_sample_saved, timeo # 'Reading with ordered access message' index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 r'Reading with ordered access.*?\n', # index = 1 pexpect.TIMEOUT, # index = 2 pexpect.EOF # index = 3 @@ -1063,7 +1063,7 @@ def coherent_sets_w_instances(child_sub, samples_sent, last_sample_saved, timeou # 'Reading with ordered access message' index = child_sub.expect( [ - '\[[0-9]+\]', # index = 0 + r'\[[0-9]+\]', # index = 0 r'Reading coherent sets.*?\n', # index = 1 pexpect.TIMEOUT, # index = 2 pexpect.EOF # index = 3 From 6f33373671dbfa0d420fff36d764a0a295e13228 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 23 Mar 2026 17:14:24 +0100 Subject: [PATCH 30/33] Added basic check to test_size_less_than_20 --- rtps_test_utilities.py | 2 +- test_suite_functions.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rtps_test_utilities.py b/rtps_test_utilities.py index 4b2e746b..483d0bf5 100644 --- a/rtps_test_utilities.py +++ b/rtps_test_utilities.py @@ -63,7 +63,7 @@ def no_check(child_sub, samples_sent, last_sample_saved, timeout): def basic_check(child_sub, samples_sent, last_sample_saved, timeout): """ Only checks that the data is well formed and size is not zero.""" - sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', + sub_string = re.search(r'\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]', child_sub.before + child_sub.after) if sub_string is None: diff --git a/test_suite_functions.py b/test_suite_functions.py index 7b4e3f95..68b2ff82 100644 --- a/test_suite_functions.py +++ b/test_suite_functions.py @@ -141,9 +141,16 @@ def test_size_less_than_20(child_sub, samples_sent, last_sample_saved, timeout): """ Checks that all received samples have size between 1 and 20 (inclusive). Returns ReturnCode.OK if all samples are in range, otherwise ReturnCode.DATA_NOT_CORRECT. + child_sub: child program generated with pexpect + samples_sent: not used + last_sample_saved: not used + timeout: time pexpect waits until it matches a pattern. """ - import re - from rtps_test_utilities import ReturnCode + + basic_check_retcode = basic_check(child_sub, samples_sent, last_sample_saved, timeout) + + if basic_check_retcode != ReturnCode.OK: + return basic_check_retcode max_samples_received = MAX_SAMPLES_READ / 2 samples_read = 0 From 972fa15fade4b1701e72ab1bab66b557bf53f0d3 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 23 Mar 2026 17:30:00 +0100 Subject: [PATCH 31/33] Changed how to calculate the color of the single-product test, now it uses the passed tests instead of supported tests --- generate_xlsx_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index 5ba18dd2..2ad2cfbe 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -800,7 +800,7 @@ def add_data_summary_worksheet(self, str(value.get_supported_tests()) + ' / ' + str(value.get_total_tests()), self.get_format_color( - value.get_passed_tests(), value.get_supported_tests())) + value.get_passed_tests(), value.get_passed_tests())) current_row += 1 From de7a9b9969ec68792d849fc521240ba9bc8a86e7 Mon Sep 17 00:00:00 2001 From: Angel Martinez Date: Mon, 23 Mar 2026 17:32:38 +0100 Subject: [PATCH 32/33] Fixing color total tests and vendor coverage --- generate_xlsx_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index 2ad2cfbe..1dc4dee0 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -790,7 +790,7 @@ def add_data_summary_worksheet(self, str(product_coverage_supported_tests) + ' / ' + str(product_coverage_total_tests), self.get_format_color( - product_coverage_supported_tests, + product_coverage_passed_tests, product_coverage_total_tests)) # total tests @@ -800,7 +800,7 @@ def add_data_summary_worksheet(self, str(value.get_supported_tests()) + ' / ' + str(value.get_total_tests()), self.get_format_color( - value.get_passed_tests(), value.get_passed_tests())) + value.get_passed_tests(), value.get_supported_tests())) current_row += 1 From fe821a6a5a35dc90ab58a700f02c620e9ad7c3ee Mon Sep 17 00:00:00 2001 From: Clark Tucker Date: Fri, 3 Apr 2026 13:20:01 -0600 Subject: [PATCH 33/33] Update generate_xlsx_report.py Add HDDS to generate_xlsx_report.py --- generate_xlsx_report.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/generate_xlsx_report.py b/generate_xlsx_report.py index 1dc4dee0..240588d3 100644 --- a/generate_xlsx_report.py +++ b/generate_xlsx_report.py @@ -79,6 +79,8 @@ def get_company_name(product:str) -> str: return 'eProsima' elif 'dust' in product.lower(): return 'S2E Software Systems' + elif 'hdds' in product.lower(): + return 'Naskel' else: raise RuntimeError('Impossible to get company name: ' + product) @@ -100,6 +102,8 @@ def get_product_name(product:str) -> str: return 'FastDDS ' + re.search(r'([\d.]+)', product).group(1) elif 'dust_dds' in product.lower(): return 'Dust DDS ' + re.search(r'([\d.]+)', product).group(1) + elif 'hdds' in product.lower(): + return 'HDDS ' + re.search(r'([\d.]+)', product).group(1) else: raise RuntimeError('Impossible to get product name: ' + product)