Add new tests - python

.github/workflows/1_run_interoperability_tests.yml

@@ -41,7 +41,6 @@ jobs:
     - name: Download shape_main executables
       uses: robinraju/release-downloader@v1.10
       with:
-        repository: omg-dds/dds-rtps
         latest: true
         fileName: "*"
         out-file-path: zipped_executables
@@ -50,12 +49,23 @@ jobs:
     - name: Install Python requirements
       run: pip install --requirement requirements.txt
     - name: Run Interoperability script
-      timeout-minutes: 60
+      timeout-minutes: 120
+      continue-on-error: true
       run: |
         publisher_exe=executables/${{ matrix.publisher }}*shape_main_linux
         subscriber_exe=executables/${{ matrix.subscriber }}*shape_main_linux
         output_file=junit_report-${{ matrix.publisher }}-${{ matrix.subscriber }}.xml
         extra_args=""
+        # Check if publisher executable exists
+        if ! ls $publisher_exe 1> /dev/null 2>&1; then
+          echo "ERROR: Publisher executable not found: $publisher_exe"
+          exit 1
+        fi
+        # Check if subscriber executable exists
+        if ! ls $subscriber_exe 1> /dev/null 2>&1; then
+          echo "ERROR: Subscriber executable not found: $subscriber_exe"
+          exit 1
+        fi
         if [[ "${subscriber_exe,,}" == *opendds* && "${publisher_exe,,}" == *connext_dds* ]]; then
           extra_args="--periodic-announcement 5000"
         fi

generate_xlsx_report.py

@@ -79,6 +79,8 @@ def get_company_name(product:str) -> str:
             return 'eProsima'
         elif 'dust' in product.lower():
             return 'S2E Software Systems'
+        elif 'hdds' in product.lower():
+            return 'Naskel'
         else:
             raise RuntimeError('Impossible to get company name: ' + product)
 
@@ -100,6 +102,8 @@ def get_product_name(product:str) -> str:
             return 'FastDDS ' + re.search(r'([\d.]+)', product).group(1)
         elif 'dust_dds' in product.lower():
             return 'Dust DDS ' + re.search(r'([\d.]+)', product).group(1)
+        elif 'hdds' in product.lower():
+            return 'HDDS ' + re.search(r'([\d.]+)', product).group(1)
         else:
             raise RuntimeError('Impossible to get product name: ' + product)
 
@@ -404,7 +408,7 @@ def create_summary_worksheet(self, name: str = 'Summary'):
         # rows.
         # The tables leave the first column (value 0) as gap
         self.add_data_summary_worksheet(
-            starting_row=9,
+            starting_row=13,
             starting_column=1,
             worksheet=summary_worksheet)
         # After having all data that may have an unknown length, we call
@@ -752,13 +756,10 @@ def add_data_summary_worksheet(self,
             'Product', self.__formats['bold_w_border'])
         worksheet.write(
             current_row, current_column + 2,
-            'Tests Passed', self.__formats['bold_w_border'])
+            'Single-Product Tests', self.__formats['bold_w_border'])
         worksheet.write(
             current_row, current_column + 3,
-            'Supported Tests', self.__formats['bold_w_border'])
-        worksheet.write(
-            current_row, current_column + 4,
-            'Supported Tests Passed', self.__formats['bold_w_border'])
+            'Cross Product Tests', self.__formats['bold_w_border'])
 
         current_row += 1
 
@@ -774,35 +775,42 @@ def add_data_summary_worksheet(self,
                 current_row, current_column + 1,
                 product_name,
                 self.__formats['bold_w_border'])
-            # test passed
-            worksheet.write(
-                current_row, current_column + 2,
-                str(value.get_passed_tests()) + ' / ' +
-                    str(value.get_total_tests()),
-                self.get_format_color(value.get_passed_tests(),
-                                      value.get_total_tests()))
-            # supported tests
+
+            # vendor coverage
+
+            # for the vendor coverage, we need to find the right data in the
+            # product_summary_dict, which is the one with the same product as
+            # publisher and subscriber
+            for product_key, product_value in self.__data.product_summary_dict.items():
+                if product_name == product_key[0] and product_name == product_key[1]:
+                    product_coverage_supported_tests = product_value.get_supported_tests()
+                    product_coverage_total_tests = product_value.get_total_tests()
+                    product_coverage_passed_tests = product_value.get_passed_tests()
+
+                    # vendor coverage
+                    worksheet.write(
+                        current_row, current_column + 2,
+                        str(product_coverage_passed_tests) + ' / ' +
+                                str(product_coverage_supported_tests) + ' / ' +
+                                str(product_coverage_total_tests),
+                        self.get_format_color(
+                                product_coverage_passed_tests,
+                                product_coverage_total_tests))
+
+            # total tests
             worksheet.write(
                 current_row, current_column + 3,
-                str(value.get_supported_tests()) + ' / ' +
-                    str(value.get_total_tests()),
-                self.__formats['result_yellow'] if value.get_unsupported_tests() > 0
-                    else self.__formats['result_green'])
-            # supported tests passed
-            worksheet.write(
-                current_row, current_column + 4,
                 str(value.get_passed_tests()) + ' / ' +
-                    str(value.get_supported_tests()),
-                self.get_format_color(value.get_passed_tests(),
-                                      value.get_supported_tests()))
+                    str(value.get_supported_tests()) + ' / ' +
+                    str(value.get_total_tests()),
+                self.get_format_color(
+                        value.get_passed_tests(), value.get_supported_tests()))
+
             current_row += 1
 
+
         # Add 2 rows of gap for the next table
         current_row += 2
-        worksheet.write(
-            current_row, current_column,
-            'Test Result: passed / supported / total', self.__formats['bold_w_border'])
-        current_row += 1
         worksheet.write(
             current_row, current_column,
             'Publisher (row)/Subscriber (column)', self.__formats['bold_w_border'])
@@ -819,7 +827,7 @@ def add_data_summary_worksheet(self,
 
         # Add the table passed_tests/total_tests with all combinations of product
         # as publishers and as subscribers
-        for (publisher_name, subscriber_name), value in self.__data.product_summary_dict.items():
+        for (publisher_name, subscriber_name), product_value in self.__data.product_summary_dict.items():
             # if the publisher hasn't been already processed yet, determine
             # what is the process_row by selecting the next free row
             # (current_row+1)
@@ -849,10 +857,10 @@ def add_data_summary_worksheet(self,
                 process_column = column_dict[subscriber_name]
 
             worksheet.write(process_row, process_column,
-                    str(value.get_passed_tests()) + ' / ' +
-                        str(value.get_supported_tests()) + ' / ' +
-                        str(value.get_total_tests()),
-                    self.get_format_color(value.get_passed_tests(), value.get_supported_tests()))
+                    str(product_value.get_passed_tests()) + ' / ' +
+                        str(product_value.get_supported_tests()) + ' / ' +
+                        str(product_value.get_total_tests()),
+                    self.get_format_color(product_value.get_passed_tests(), product_value.get_supported_tests()))
 
     def add_static_data_summary_worksheet(self,
             worksheet: xlsxwriter.Workbook.worksheet_class,
@@ -885,10 +893,9 @@ def add_static_data_summary_worksheet(self,
         worksheet.insert_image(
             row=current_row, col=starting_column,
             filename=dds_logo_path,
-            options={'x_scale': 0.4, 'y_scale': 0.4, 'decorative': True, 'object_position': 2})
+            options={'x_scale': 0.6, 'y_scale': 0.6, 'decorative': True, 'object_position': 2})
 
         # Add date
-        current_row += 1
         worksheet.write(current_row, starting_column + 1, 'Date')
         date_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         worksheet.write(current_row, starting_column + 2, date_time)
@@ -903,6 +910,39 @@ def add_static_data_summary_worksheet(self,
         worksheet.write(current_row, starting_column + 1,'Documentation')
         worksheet.write(current_row, starting_column + 2, self.REPO_DOC)
 
+        # Add number of tests
+
+        # Find the total number of unique tests by looking for a
+        # (publisher, subscriber) pair where both are the same
+        test_count = 0
+        for key, value in self.__data.product_summary_dict.items():
+            if key[0] == key[1]:
+                test_count = value.get_total_tests()
+                break
+        current_row += 1
+        worksheet.write(current_row, starting_column + 1,'Unique tests count')
+        worksheet.write(current_row, starting_column + 2, test_count)
+
+        # add legend
+        current_row += 2
+        worksheet.write(current_row, starting_column + 1,
+                'Single-Product Tests',
+                self.__formats['bold'])
+        worksheet.write(current_row, starting_column + 2,
+                'Results where each product is tested only with itself')
+        current_row += 1
+        worksheet.write(current_row, starting_column + 1,
+                'Cross-Product Tests',
+                self.__formats['bold'])
+        worksheet.write(current_row, starting_column + 2,
+                'Results where each product is tested against all others (including itself)')
+        current_row += 1
+        worksheet.write(current_row, starting_column + 1,
+                'Format X / Y / Z',
+                self.__formats['bold'])
+        worksheet.write(current_row, starting_column + 2,
+                'Represents test passed / supported / total')
+
     def add_static_data_description_worksheet(self,
             worksheet: xlsxwriter.Workbook.worksheet_class,
             name: str = 'Test Descriptions',

interoperability_report.py

@@ -380,7 +380,8 @@ def run_publisher_shape_main(
                                     r'\[[0-9]+\]', # index = 0
                                     'on_offered_deadline_missed()', # index = 1
                                     re.compile('not supported', re.IGNORECASE), # index = 2
-                                    pexpect.TIMEOUT # index = 3
+                                    pexpect.TIMEOUT, # index = 3
+                                    pexpect.EOF # index = 4
                                 ],
                                 timeout)
                             if index == 1:
@@ -389,7 +390,7 @@ def run_publisher_shape_main(
                             elif index == 2:
                                 produced_code[produced_code_index] = ReturnCode.PUB_UNSUPPORTED_FEATURE
                                 break
-                            elif index == 3:
+                            elif index == 3 or index == 4:
                                 produced_code[produced_code_index] = ReturnCode.DATA_NOT_SENT
                                 break
                         last_sample_saved.put(last_sample)
@@ -398,8 +399,18 @@ def run_publisher_shape_main(
 
     log_message(f'Publisher {publisher_index}: Waiting for Subscribers to finish',
             verbosity)
-    for element in subscribers_finished:
-        element.wait() # wait for all subscribers to finish
+    # Check if all subscribers finished
+    while True:
+        if all(e.is_set() for e in subscribers_finished):
+            break
+
+        # Drain publisher output
+        try:
+            child_pub.read_nonblocking(1024, timeout=0.1)
+        except pexpect.TIMEOUT:
+            pass
+        except pexpect.EOF:
+            break
     publisher_finished.set()   # set publisher as finished
     # Stop process
     if not stop_process(child_pub):

rtps_test_utilities.py

@@ -63,7 +63,7 @@ def no_check(child_sub, samples_sent, last_sample_saved, timeout):
 
 def basic_check(child_sub, samples_sent, last_sample_saved, timeout):
     """ Only checks that the data is well formed and size is not zero."""
-    sub_string = re.search('\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]',
+    sub_string = re.search(r'\w\s+\w+\s+[0-9]+ [0-9]+ \[([0-9]+)\]',
         child_sub.before + child_sub.after)
 
     if sub_string is None: