Added cov-report to CI in review_reco

.github/workflows/build_and_test_app.yml

@@ -21,10 +21,13 @@ jobs:
       - name: install quality deps
         run: |
           pip install pylint mccabe
-      - name: code quality
+      - name: Run Pylint
+        shell: bash -l {0}
+        run: |
+          pylint --recursive=y --disable=all --fail-under=10 --enable=too-many-statements,too-many-nested-blocks .
+      - name: Run McCabe Complexity Check
+        shell: bash -l {0}
         run: |
-          pylint --disable=all --fail-under=10 --enable=too-many-statements --max-statements=100 .
-          pylint --disable=all --fail-under=10 --enable=too-many-nested-blocks .
           ./continuous_integration/scripts/check_mccabe_complexity.sh 25 .
   test:
     runs-on: ubuntu-22.04
@@ -57,5 +60,13 @@ jobs:
         export OTB_APPLICATION_PATH=/opt/otb/lib/otb/applications
         export OTB_INSTALL_DIR=/opt/otb
         export LC_NUMERIC=C
-
-        pytest -s --cov-fail-under=65
+
+        pytest --cov=. --cov-report=xml:.ci-reports-alcd/coverage_alcd.xml --cov-report html:cov_html_alcd --cov-report=term --junitxml=pytest-results-alcd.xml
+    - name: Upload Coverage Report
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report-alcd
+        path: |
+          .ci-reports-alcd/
+          cov_html_alcd/
+          pytest-results-alcd.xml

L1C_band_composition.py

@@ -419,49 +419,21 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur
 
     # Create new indices if needed
     new_indices = global_parameters["features"]["special_indices"]
-    for indice in new_indices:
-        out_tif = op.join(out_dir_bands, (indice + '.tif'))
-        create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution)
-        additional_bands.append(str(out_tif))
+    create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution)
 
     # Create the ratios
     ratios = create_ratio_bands(global_parameters, bands_dir, out_dir_bands, resolution=60)
     additional_bands.extend(ratios)
 
-    use_DTM = str2bool(global_parameters["features"]["DTM"])
-    if use_DTM:
-        # Append the DTM model
-        out_dtm = op.join(out_dir_bands, ('DTM.tif'))
-        # try to append it. If an error occurs, the DTM probably does not exist
-        # and we will therefore skip this band
-        try:
-            dtm_addition(location, out_dtm, resolution=resolution)
-            additional_bands.append(str(out_dtm))
-        except:
-            print('ERROR : THE DTM DOES NOT EXIST !!!')
+    use_dtm(additional_bands, global_parameters, location, out_dir_bands, resolution)
 
     create_textures = str2bool(global_parameters["features"]["textures"])
     # Create the texture features
     if create_textures:
-        band_used_for_contours = 2
-        in_tif = glob.glob(
-            op.join(bands_dir, '*{}*{:02}.jp2'.format(band_prefix, band_used_for_contours)))[0]
-        in_channel = 1
-
-        out_tif = op.join(out_dir_bands, 'density_contours.tif')
-        create_contours_density(in_tif, in_channel, out_tif, radius=3)
-        additional_bands.append(str(out_tif))
-        out_tif = op.join(out_dir_bands, 'variation_coeff.tif')
-        create_variation_coeff(in_tif, in_channel, out_tif, radius=3)
-        additional_bands.append(str(out_tif))
+        create_texture_feat(additional_bands, band_prefix, bands_dir, out_dir_bands)
 
     # Create time difference features
-    bands_num = [int(band) for band in global_parameters["features"]["time_difference_bands"]]
-    out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate')
-    for band_num in bands_num:
-        out_tif = op.join(out_dir_bands, ('time_' + str(band_num) + '.tif'))
-        create_time_difference_band(global_parameters, paths_parameters, band_num, out_tif, resolution=resolution)
-        additional_bands.append(str(out_tif))
+    time_diff_feat(additional_bands, global_parameters, paths_parameters, resolution)
 
     # --- Create the main TIF with low resolution
     # create intermediate resolution files
@@ -501,10 +473,7 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur
         new_indices = ['NDVI', 'NDWI']
         out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate')
         additional_bands = []
-        for indice in new_indices:
-            out_tif = op.join(out_dir_bands, (indice + '.tif'))
-            create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution)
-            additional_bands.append(str(out_tif))
+        create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution)
 
         # create intermediate resolution files
         intermediate_bands_dir = op.join(
@@ -513,15 +482,8 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur
         pixelresY = resolution
 
         # The bands to put into the heavy file
-        bands_num = [2, 3, 4, 10]
-
-        intermediate_sizes_paths = []
-        for band in bands_num:
-            in_band = str(op.join(bands_dir, band_prefix) + '{:02d}'.format(band)+'.jp2')
-            out_band = op.join(intermediate_bands_dir, op.basename(in_band)[0:-4]+'.tif')
-
-            resize_band(in_band, out_band, pixelresX, pixelresY)
-            intermediate_sizes_paths.append(out_band)
+        intermediate_sizes_paths = put_band_heavy_tif(band_prefix, bands_dir, intermediate_bands_dir,
+                                                      intermediate_sizes_paths, pixelresX, pixelresY)
 
         out_heavy_tif = op.join(global_parameters["user_choices"]["main_dir"], 'In_data',
                                 'Image', global_parameters["user_choices"]["raw_img"])[0:-4]+'_H.tif'
@@ -531,7 +493,6 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur
         compose_bands_heavy(intermediate_sizes_paths, str(out_heavy_tif))
 
     if "user_function" in list(global_parameters["user_choices"].keys()) and global_parameters["user_choices"]["user_function"] != None:
-        print('ENTERED')
         user_process(raw_img = out_all_bands_tif,
                  main_dir = global_parameters["user_choices"]["main_dir"],
                  module_path = global_parameters["user_choices"]["user_module"],
@@ -541,6 +502,61 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur
     return
 
 
+def put_band_heavy_tif(band_prefix, bands_dir, intermediate_bands_dir, intermediate_sizes_paths, pixelresX, pixelresY):
+    bands_num = [2, 3, 4, 10]
+    intermediate_sizes_paths = []
+    for band in bands_num:
+        in_band = str(op.join(bands_dir, band_prefix) + '{:02d}'.format(band) + '.jp2')
+        out_band = op.join(intermediate_bands_dir, op.basename(in_band)[0:-4] + '.tif')
+
+        resize_band(in_band, out_band, pixelresX, pixelresY)
+        intermediate_sizes_paths.append(out_band)
+    return intermediate_sizes_paths
+
+
+def create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution):
+    for indice in new_indices:
+        out_tif = op.join(out_dir_bands, (indice + '.tif'))
+        create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution)
+        additional_bands.append(str(out_tif))
+
+
+def time_diff_feat(additional_bands, global_parameters, paths_parameters, resolution):
+    bands_num = [int(band) for band in global_parameters["features"]["time_difference_bands"]]
+    out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate')
+    for band_num in bands_num:
+        out_tif = op.join(out_dir_bands, ('time_' + str(band_num) + '.tif'))
+        create_time_difference_band(global_parameters, paths_parameters, band_num, out_tif, resolution=resolution)
+        additional_bands.append(str(out_tif))
+
+
+def use_dtm(additional_bands, global_parameters, location, out_dir_bands, resolution):
+    use_DTM = str2bool(global_parameters["features"]["DTM"])
+    if use_DTM:
+        # Append the DTM model
+        out_dtm = op.join(out_dir_bands, ('DTM.tif'))
+        # try to append it. If an error occurs, the DTM probably does not exist
+        # and we will therefore skip this band
+        try:
+            dtm_addition(location, out_dtm, resolution=resolution)
+            additional_bands.append(str(out_dtm))
+        except:
+            print('ERROR : THE DTM DOES NOT EXIST !!!')
+
+
+def create_texture_feat(additional_bands, band_prefix, bands_dir, out_dir_bands):
+    band_used_for_contours = 2
+    in_tif = glob.glob(
+        op.join(bands_dir, '*{}*{:02}.jp2'.format(band_prefix, band_used_for_contours)))[0]
+    in_channel = 1
+    out_tif = op.join(out_dir_bands, 'density_contours.tif')
+    create_contours_density(in_tif, in_channel, out_tif, radius=3)
+    additional_bands.append(str(out_tif))
+    out_tif = op.join(out_dir_bands, 'variation_coeff.tif')
+    create_variation_coeff(in_tif, in_channel, out_tif, radius=3)
+    additional_bands.append(str(out_tif))
+
+
 def create_no_data_tif(global_parameters, paths_parameters, out_tif, dilation_radius=10):
     '''
     Create the no_data TIF using both the clear and cloudy date.

all_run_alcd.py

@@ -124,25 +124,7 @@ def run_all(part, global_parameters, paths_parameters, model_parameters, first_i
                 main_dir, global_parameters, paths_parameters, raw_img_name, location, current_date, clear_date)
 
             if first_iteration == True:
-                # Create the directories
-                OTB_workflow.create_directories(global_parameters)
-
-                # Copy the global_parameters files to save it
-                src = global_parameters["json_file"]
-                dst = op.join(global_parameters["user_choices"]
-                              ["main_dir"], 'In_data', 'used_global_parameters.json')
-                shutil.copyfile(src, dst)
-
-                # Create the images .tif and .jp2, i.e. the features
-                L1C_band_composition.create_image_compositions(
-                    global_parameters, location, paths_parameters, current_date, heavy=True, force=force)
-
-                # Create the empty layers
-                layers_creation.create_all_classes_empty_layers(global_parameters, force=force)
-
-                # Fill automatically the no_data layer from the L1C missing
-                # pixels
-                layers_creation.create_no_data_shp(global_parameters,paths_parameters, force=force)
+                first_it_worklfow(current_date, force, global_parameters, location, paths_parameters)
     # ----------------------------------------------
     # WAIT FOR USER MODIFICATION OF THE LAYERS IN LOCAL
     # ----------------------------------------------
@@ -197,6 +179,24 @@ def run_all(part, global_parameters, paths_parameters, model_parameters, first_i
         OTB_wf.create_contour_from_labeled(global_parameters, proceed=True)
 
 
+def first_it_worklfow(current_date, force, global_parameters, location, paths_parameters):
+    # Create the directories
+    OTB_workflow.create_directories(global_parameters)
+    # Copy the global_parameters files to save it
+    src = global_parameters["json_file"]
+    dst = op.join(global_parameters["user_choices"]
+                  ["main_dir"], 'In_data', 'used_global_parameters.json')
+    shutil.copyfile(src, dst)
+    # Create the images .tif and .jp2, i.e. the features
+    L1C_band_composition.create_image_compositions(
+        global_parameters, location, paths_parameters, current_date, heavy=True, force=force)
+    # Create the empty layers
+    layers_creation.create_all_classes_empty_layers(global_parameters, force=force)
+    # Fill automatically the no_data layer from the L1C missing
+    # pixels
+    layers_creation.create_no_data_shp(global_parameters, paths_parameters, force=force)
+
+
 def str2bool(v):
     '''
     Use it to change multiple pseudo-boolean values to a real boolean
@@ -211,7 +211,7 @@ def str2bool(v):
         raise argparse.ArgumentTypeError('Boolean value expected.')
 
 
-def main():
+def getarguments():
     # parsing from the shell
     parser = argparse.ArgumentParser()
 
@@ -238,35 +238,40 @@ def main():
     parser.add_argument('-model_parameters', dest='model_parameters_file',
                         help='str, path to json file which contain classifier parameters', required=True)
     results = parser.parse_args()
-    location = results.location
-    global_parameters = read_global_parameters(results.global_parameters_file)
-    paths_parameters = read_paths_parameters(results.paths_parameters_file)
-    model_parameters = read_models_parameters(results.model_parameters_file)
+    return vars(results)
+
 
-    global_parameters["json_file"] = results.global_parameters_file
-    get_dates = str2bool(results.get_dates)
+def all_run_alcd(global_parameters_file, paths_parameters_file, model_parameters_file, location=None,
+                 wanted_date=None, clear_date=None, first_iteration=None, user_input=None, get_dates='false',
+                 force='false', kfold='false'):
+    global_parameters = read_global_parameters(global_parameters_file)
+    paths_parameters = read_paths_parameters(paths_parameters_file)
+    model_parameters = read_models_parameters(model_parameters_file)
+
+    global_parameters["json_file"] = global_parameters_file
+    get_dates = str2bool(get_dates)
     if get_dates:
         available_dates = find_directory_names.get_all_dates(location, paths_parameters)
         print('\nAvailable dates:\n')
         print([str(d) for d in available_dates])
         return
 
-    if results.first_iteration == None:
+    if first_iteration == None:
         print('Please enter a boolean for the first iteration')
         return
     else:
-        first_iteration = str2bool(results.first_iteration)
+        first_iteration = str2bool(first_iteration)
 
-    if results.user_input == None:
+    if user_input == None:
         print('Please enter an integer for the step')
         return
     else:
-        user_input = results.user_input
+        user_input = user_input
 
-    wanted_date = results.wanted_date
-    clear_date = results.clear_date
-    force = str2bool(results.force)
-    kfold = str2bool(results.kfold)
+    wanted_date = wanted_date
+    clear_date = clear_date
+    force = str2bool(force)
+    kfold = str2bool(kfold)
 
     if kfold:
         tmp_name = next(tempfile._get_candidate_names())
@@ -319,6 +324,12 @@ def main():
     else:
         print('Please enter a valid step value [0 or 1]')
 
+def main():
+    """
+    It parses the command line arguments and calls the all_run_alcd function.
+    """
+    args = getarguments()
+    all_run_alcd(**args)
 
 if __name__ == '__main__':
-    main()
+    main()

metrics_exploitation.py

@@ -370,45 +370,43 @@ def retrieve_Kfold_data(global_parameters, metrics_plotting=False, location='',
         recalls = [m[2] for m in all_metrics]
         precisions = [m[3] for m in all_metrics]
 
-        plt.errorbar(indices, means[0:4], stds[0:4], linestyle='',
-                     marker='o', color='b')
-        met_nb = 0
-
-        for metric in [accuracies, f1scores, recalls, precisions]:
-            rnd = [(indices[met_nb] - 0.1 + 0.2*(float(j)/len(accuracies)))
-                   for j in range(len(accuracies))]
-            plt.scatter(rnd, metric, color='k', marker='.', alpha=0.2)
-            met_nb += 1
-
-        plt.ylim(0.5, 1)
-        metrics_names = ['Accuracy\n{:.1f}%'.format(means[0]*100),
-                         'F1-score\n{:.1f}%'.format(means[1]*100),
-                         'Recall\n{:.1f}%'.format(means[2]*100),
-                         'Precision\n{:.1f}%'.format(means[3]*100)]
-        plt.xticks(indices, metrics_names)
-
-        nb_dates = float(len(accuracies))/11
-        plt.title(
-            'Metrics of a {}-fold random cross-validation\n{}, {}'.format(len(accuracies), location, date))
-        plt.xlabel('Score type')
-        plt.ylabel('Scores')
-
-        # Custom legend
-        custom_lines = []
-        custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='k', marker='.', alpha=0.2))
-        custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='b', marker='o', alpha=1))
-        legend_labels = ['Single fold point', 'Mean and std of all folds']
-        plt.legend(custom_lines, legend_labels, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
-
-        #~ plt.show()
-
-        out_fig = op.join(statistics_dir, 'kfold_metrics.png')
-        plt.savefig(out_fig, bbox_inches='tight')
+        plot_metrics(accuracies, date, f1scores, indices, location, means, precisions, recalls, statistics_dir, stds)
         #~ plt.close()
 
     return means, stds, all_metrics
 
 
+def plot_metrics(accuracies, date, f1scores, indices, location, means, precisions, recalls, statistics_dir, stds):
+    plt.errorbar(indices, means[0:4], stds[0:4], linestyle='',
+                 marker='o', color='b')
+    met_nb = 0
+    for metric in [accuracies, f1scores, recalls, precisions]:
+        rnd = [(indices[met_nb] - 0.1 + 0.2 * (float(j) / len(accuracies)))
+               for j in range(len(accuracies))]
+        plt.scatter(rnd, metric, color='k', marker='.', alpha=0.2)
+        met_nb += 1
+    plt.ylim(0.5, 1)
+    metrics_names = ['Accuracy\n{:.1f}%'.format(means[0] * 100),
+                     'F1-score\n{:.1f}%'.format(means[1] * 100),
+                     'Recall\n{:.1f}%'.format(means[2] * 100),
+                     'Precision\n{:.1f}%'.format(means[3] * 100)]
+    plt.xticks(indices, metrics_names)
+    nb_dates = float(len(accuracies)) / 11
+    plt.title(
+        'Metrics of a {}-fold random cross-validation\n{}, {}'.format(len(accuracies), location, date))
+    plt.xlabel('Score type')
+    plt.ylabel('Scores')
+    # Custom legend
+    custom_lines = []
+    custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='k', marker='.', alpha=0.2))
+    custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='b', marker='o', alpha=1))
+    legend_labels = ['Single fold point', 'Mean and std of all folds']
+    plt.legend(custom_lines, legend_labels, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
+    # ~ plt.show()
+    out_fig = op.join(statistics_dir, 'kfold_metrics.png')
+    plt.savefig(out_fig, bbox_inches='tight')
+
+
 def load_previous_global_parameters(location, date):
     paths_configuration = read_paths_parameters(op.join('parameters_files', 'paths_configuration.json'))
     Data_ALCD_dir = paths_configuration["data_paths"]["data_alcd"]