From a070a11ccbdc3ab4b3c228379db34ece2a26299b Mon Sep 17 00:00:00 2001 From: cadauxe Date: Fri, 22 Aug 2025 16:58:31 +0200 Subject: [PATCH 1/5] feat: refactored for cov-report in github ci --- .github/workflows/build_and_test_app.yml | 21 +++-- all_run_alcd.py | 43 ++++++---- pytest.ini | 4 + quicklook_generator.py | 26 ++++-- tests/test_run_alcd.py | 102 +++++++++++++++-------- 5 files changed, 134 insertions(+), 62 deletions(-) create mode 100644 pytest.ini diff --git a/.github/workflows/build_and_test_app.yml b/.github/workflows/build_and_test_app.yml index c17082d..d7b1404 100644 --- a/.github/workflows/build_and_test_app.yml +++ b/.github/workflows/build_and_test_app.yml @@ -21,10 +21,13 @@ jobs: - name: install quality deps run: | pip install pylint mccabe - - name: code quality + - name: Run Pylint + shell: bash -l {0} + run: | + pylint --recursive=y --disable=all --fail-under=10 --enable=too-many-statements,too-many-nested-blocks . + - name: Run McCabe Complexity Check + shell: bash -l {0} run: | - pylint --disable=all --fail-under=10 --enable=too-many-statements --max-statements=100 . - pylint --disable=all --fail-under=10 --enable=too-many-nested-blocks . ./continuous_integration/scripts/check_mccabe_complexity.sh 25 . test: runs-on: ubuntu-22.04 @@ -57,5 +60,13 @@ jobs: export OTB_APPLICATION_PATH=/opt/otb/lib/otb/applications export OTB_INSTALL_DIR=/opt/otb export LC_NUMERIC=C - - pytest -s --cov-fail-under=65 + + pytest --cov=. --cov-report=xml:.ci-reports-alcd/coverage_alcd.xml --cov-report html:cov_html_alcd --cov-report=term --junitxml=pytest-results-alcd.xml + - name: Upload Coverage Report + uses: actions/upload-artifact@v4 + with: + name: coverage-report-alcd + path: | + .ci-reports-alcd/ + cov_html_alcd/ + pytest-results-alcd.xml \ No newline at end of file diff --git a/all_run_alcd.py b/all_run_alcd.py index d6002db..8f61a84 100644 --- a/all_run_alcd.py +++ b/all_run_alcd.py @@ -211,7 +211,7 @@ def str2bool(v): raise argparse.ArgumentTypeError('Boolean value expected.') -def main(): +def getarguments(): # parsing from the shell parser = argparse.ArgumentParser() @@ -238,35 +238,40 @@ def main(): parser.add_argument('-model_parameters', dest='model_parameters_file', help='str, path to json file which contain classifier parameters', required=True) results = parser.parse_args() - location = results.location - global_parameters = read_global_parameters(results.global_parameters_file) - paths_parameters = read_paths_parameters(results.paths_parameters_file) - model_parameters = read_models_parameters(results.model_parameters_file) + return vars(results) + - global_parameters["json_file"] = results.global_parameters_file - get_dates = str2bool(results.get_dates) +def all_run_alcd(global_parameters_file, paths_parameters_file, model_parameters_file, location=None, + wanted_date=None, clear_date=None, first_iteration=None, user_input=None, get_dates='false', + force='false', kfold='false'): + global_parameters = read_global_parameters(global_parameters_file) + paths_parameters = read_paths_parameters(paths_parameters_file) + model_parameters = read_models_parameters(model_parameters_file) + + global_parameters["json_file"] = global_parameters_file + get_dates = str2bool(get_dates) if get_dates: available_dates = find_directory_names.get_all_dates(location, paths_parameters) print('\nAvailable dates:\n') print([str(d) for d in available_dates]) return - if results.first_iteration == None: + if first_iteration == None: print('Please enter a boolean for the first iteration') return else: - first_iteration = str2bool(results.first_iteration) + first_iteration = str2bool(first_iteration) - if results.user_input == None: + if user_input == None: print('Please enter an integer for the step') return else: - user_input = results.user_input + user_input = user_input - wanted_date = results.wanted_date - clear_date = results.clear_date - force = str2bool(results.force) - kfold = str2bool(results.kfold) + wanted_date = wanted_date + clear_date = clear_date + force = str2bool(force) + kfold = str2bool(kfold) if kfold: tmp_name = next(tempfile._get_candidate_names()) @@ -319,6 +324,12 @@ def main(): else: print('Please enter a valid step value [0 or 1]') +def main(): + """ + It parses the command line arguments and calls the all_run_alcd function. + """ + args = getarguments() + all_run_alcd(**args) if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..00930af --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = -ra +testpaths = tests +pythonpath=. diff --git a/quicklook_generator.py b/quicklook_generator.py index e8decdc..eaedd71 100644 --- a/quicklook_generator.py +++ b/quicklook_generator.py @@ -84,7 +84,7 @@ def create_all_quicklook(location, out_dir, paths_parameters): k += 1 -def main(): +def getarguments(): parser = argparse.ArgumentParser() parser.add_argument('-l', action='store', default=None, dest='locations', @@ -93,21 +93,31 @@ def main(): help='str, path to json file which contain useful path for ALCD', required=True) results = parser.parse_args() - locations_to_parse = results.locations - paths_parameters = read_paths_parameters(results.paths_parameters_file) + return vars(results) + +def quicklook_generator(locations=None, paths_parameters_file=None): + locations_to_parse = locations + paths_parameters = read_paths_parameters(paths_parameters_file) if locations_to_parse != None: - locations = locations_to_parse.split(',') + loc = locations_to_parse.split(',') else: - locations = ['Mongu', 'Gobabeb', 'RailroadValley', 'Arles', 'Marrakech'] - locations = ['Alta_Floresta_Brazil'] + loc = ['Mongu', 'Gobabeb', 'RailroadValley', 'Arles', 'Marrakech'] + loc = ['Alta_Floresta_Brazil'] - for location in locations: + for location in loc: out_dir = op.join('tmp', location) create_all_quicklook(location, out_dir, paths_parameters) return +def main(): + """ + It parses the command line arguments and calls the all_run_alcd function. + """ + args = getarguments() + quicklook_generator(**args) + if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/tests/test_run_alcd.py b/tests/test_run_alcd.py index 1f86231..1fcfeb1 100644 --- a/tests/test_run_alcd.py +++ b/tests/test_run_alcd.py @@ -33,6 +33,8 @@ from conftest import ALCDTestsData from sklearn.base import BaseEstimator +from all_run_alcd import all_run_alcd +from quicklook_generator import quicklook_generator def check_expected_quicklook_results( @@ -102,7 +104,7 @@ def check_expected_features_content_alcd( # Extract the number of bands in the user's data band_path = op.join(feat_dir, "In_data", "Image", "Toulouse_bands_bands.txt") training_samples_extracted = op.join(feat_dir, "Samples", "training_samples_extracted_user_prim.sqlite") - + tif_path = op.join(feat_dir, "In_data", "Image", "Toulouse_bands.tif") with rasterio.open(tif_path) as src: exp_nband = src.count @@ -189,12 +191,19 @@ def test_run_alcd(alcd_paths: ALCDTestsData) -> None: output_dir = alcd_paths.data_dir / "test_run_alcd" / "Toulouse_31TCJ_20240305" global_param_file, paths_param_file = prepare_test_dir(alcd_paths, output_dir, "rf_otb") - cmd = f"python {alcd_paths.project_dir}/all_run_alcd.py -f True -s 1 -l Toulouse -d 20240305 -c 20240120 -dates False -kfold False -force False -global_parameters {global_param_file} -paths_parameters {paths_param_file} -model_parameters {alcd_paths.cfg}/model_parameters.json" - proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + all_run_alcd( + global_parameters_file=global_param_file, + paths_parameters_file=paths_param_file, + model_parameters_file=f"{alcd_paths.cfg}/model_parameters.json", + location="Toulouse", + wanted_date="20240305", + clear_date="20240120", + first_iteration="1", + user_input="True", # corresponds to -f True + get_dates="False", + force="False", + kfold="False" ) - out, _ = proc.communicate() - assert proc.returncode == 0, out.decode('utf-8') alcd_results, details = check_expected_alcd_results( alcd_paths.data_dir / "test_run_alcd" / "Toulouse_31TCJ_20240305" / "Out") assert alcd_results, f"some output files are missing: {', '.join(file_name for file_name, exists in details.items() if not exists)}" @@ -218,12 +227,19 @@ def test_scikit_alcd(alcd_paths: ALCDTestsData) -> None: output_dir = alcd_paths.data_dir / "test_scikit_alcd" / "Toulouse_31TCJ_20240305" global_param_file, paths_param_file = prepare_test_dir(alcd_paths, output_dir, "rf_scikit") - cmd = f"python {alcd_paths.project_dir}/all_run_alcd.py -f True -s 1 -l Toulouse -d 20240305 -c 20240120 -dates False -kfold False -force False -global_parameters {global_param_file} -paths_parameters {paths_param_file} -model_parameters {alcd_paths.cfg}/model_parameters.json" - proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + all_run_alcd( + global_parameters_file=global_param_file, + paths_parameters_file=paths_param_file, + model_parameters_file=f"{alcd_paths.cfg}/model_parameters.json", + location="Toulouse", + wanted_date="20240305", + clear_date="20240120", + first_iteration="1", + user_input="True", + get_dates="False", + force="False", + kfold="False" ) - out, _ = proc.communicate() - assert proc.returncode == 0, out.decode('utf-8') alcd_results, details = check_expected_alcd_results( alcd_paths.data_dir / "test_scikit_alcd" / "Toulouse_31TCJ_20240305" / "Out") assert alcd_results, f"some output files are missing: {', '.join(file_name for file_name, exists in details.items() if not exists)}" @@ -255,17 +271,34 @@ def test_user_prim_alcd(alcd_paths: ALCDTestsData) -> None: output_dir = alcd_paths.data_dir / "test_user_prim_alcd" / "Toulouse_31TCJ_20240305" global_param_file, paths_param_file = prepare_test_dir(alcd_paths, output_dir, "rf_scikit","global_parameters_user_prim.json") - cmd1 = f"python {alcd_paths.project_dir}/all_run_alcd.py -force True -f True -s 0 -l Toulouse -d 20240305 -c 20240120 -dates False -kfold False -force False -global_parameters {global_param_file} -paths_parameters {paths_param_file} -model_parameters {alcd_paths.cfg}/model_parameters.json" - proc1 = subprocess.Popen(cmd1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out, _ = proc1.communicate() - assert proc1.returncode == 0, out.decode('utf-8') - + all_run_alcd( + global_parameters_file=global_param_file, + paths_parameters_file=paths_param_file, + model_parameters_file=f"{alcd_paths.cfg}/model_parameters.json", + location="Toulouse", + wanted_date="20240305", + clear_date="20240120", + first_iteration="0", + user_input="True", + get_dates="False", + force="False", + kfold="False" + ) shutil.copytree(alcd_paths.s2_data / "Toulouse_31TCJ_20240305" / "In_data" / "Image", output_dir / "In_data" / "Image", dirs_exist_ok=True) - cmd2 = f"python {alcd_paths.project_dir}/all_run_alcd.py -f True -s 1 -l Toulouse -d 20240305 -c 20240120 -dates False -kfold False -force False -global_parameters {global_param_file} -paths_parameters {paths_param_file} -model_parameters {alcd_paths.cfg}/model_parameters.json" - proc2 = subprocess.Popen(cmd2, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, _ = proc2.communicate() - assert proc2.returncode == 0, out.decode('utf-8') + all_run_alcd( + global_parameters_file=global_param_file, + paths_parameters_file=paths_param_file, + model_parameters_file=f"{alcd_paths.cfg}/model_parameters.json", + location="Toulouse", + wanted_date="20240305", + clear_date="20240120", + first_iteration="1", + user_input="True", + get_dates="False", + force="False", + kfold="False" + ) alcd_results, details = check_expected_alcd_results( alcd_paths.data_dir / "test_user_prim_alcd" / "Toulouse_31TCJ_20240305" / "Out") @@ -292,13 +325,19 @@ def test_run_alcd_gen_features(alcd_paths: ALCDTestsData) -> None: output_dir = alcd_paths.data_dir / "test_gen_features" / "Toulouse_31TCJ_20240305" global_param_file, paths_param_file = prepare_test_dir(alcd_paths, output_dir, "rf_otb") - cmd = f"python {alcd_paths.project_dir}/all_run_alcd.py -force True -f 1 -s 0 -l Toulouse -d 20240305 -c 20240120 -dates False -kfold False -global_parameters {global_param_file} -paths_parameters {paths_param_file} -model_parameters {alcd_paths.cfg}/model_parameters.json" - - proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + all_run_alcd( + global_parameters_file=global_param_file, + paths_parameters_file=paths_param_file, + model_parameters_file=f"{alcd_paths.cfg}/model_parameters.json", + location="Toulouse", + wanted_date="20240305", + clear_date="20240120", + first_iteration="0", + user_input="1", + get_dates="False", + force="True", + kfold="False" ) - out, _ = proc.communicate() - assert proc.returncode == 0, out.decode('utf-8') alcd_results, details = check_expected_features_alcd( alcd_paths.data_dir / "test_gen_features" / "Toulouse_31TCJ_20240305" / "In_data" / "Image") assert alcd_results, f"some output files are missing: {', '.join(file_name for file_name, exists in details.items() if not exists)}" @@ -321,12 +360,9 @@ def test_quicklook(alcd_paths: ALCDTestsData) -> None: output_dir = alcd_paths.data_dir / "test_quicklooks" / "Toulouse_31TCJ_20240305" global_param_file, paths_param_file = prepare_test_dir(alcd_paths, output_dir, "rf_otb") - cmd = f"python {alcd_paths.project_dir}/quicklook_generator.py -l Toulouse -paths_parameters {paths_param_file}" - - proc = subprocess.Popen( - cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + quicklook_generator( + locations="Toulouse", + paths_parameters_file=paths_param_file ) - out, _ = proc.communicate() - assert proc.returncode == 0, out.decode('utf-8') quicklook_results, details = check_expected_quicklook_results(Path("tmp") / "Toulouse") - assert quicklook_results, f"some quicklook files are missing: {', '.join(file_name for file_name, exists in details.items() if not exists)}" + assert quicklook_results, f"some quicklook files are missing: {', '.join(file_name for file_name, exists in details.items() if not exists)}" \ No newline at end of file From c9be41412fc858dfde13e7b92be72ce9ecb9c487 Mon Sep 17 00:00:00 2001 From: cadauxe Date: Tue, 26 Aug 2025 14:25:02 +0200 Subject: [PATCH 2/5] feat: refactored for cov-report in github ci --- .github/workflows/build_and_test_app.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test_app.yml b/.github/workflows/build_and_test_app.yml index d7b1404..51a03bb 100644 --- a/.github/workflows/build_and_test_app.yml +++ b/.github/workflows/build_and_test_app.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - "master" + - "review_reco" permissions: contents: read From 233357b8c1e04bbcc5451d461b9cac646bbc86df Mon Sep 17 00:00:00 2001 From: cadauxe Date: Tue, 26 Aug 2025 14:48:15 +0200 Subject: [PATCH 3/5] feat: code quality refactoring --- L1C_band_composition.py | 108 +++++++++++++++++++++++----------------- all_run_alcd.py | 38 +++++++------- metrics_exploitation.py | 66 ++++++++++++------------ split_samples.py | 95 ++++++++++++++++++----------------- 4 files changed, 162 insertions(+), 145 deletions(-) diff --git a/L1C_band_composition.py b/L1C_band_composition.py index 96ede4b..27d006a 100644 --- a/L1C_band_composition.py +++ b/L1C_band_composition.py @@ -419,49 +419,21 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur # Create new indices if needed new_indices = global_parameters["features"]["special_indices"] - for indice in new_indices: - out_tif = op.join(out_dir_bands, (indice + '.tif')) - create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution) - additional_bands.append(str(out_tif)) + create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution) # Create the ratios ratios = create_ratio_bands(global_parameters, bands_dir, out_dir_bands, resolution=60) additional_bands.extend(ratios) - use_DTM = str2bool(global_parameters["features"]["DTM"]) - if use_DTM: - # Append the DTM model - out_dtm = op.join(out_dir_bands, ('DTM.tif')) - # try to append it. If an error occurs, the DTM probably does not exist - # and we will therefore skip this band - try: - dtm_addition(location, out_dtm, resolution=resolution) - additional_bands.append(str(out_dtm)) - except: - print('ERROR : THE DTM DOES NOT EXIST !!!') + use_dtm(additional_bands, global_parameters, location, out_dir_bands, resolution) create_textures = str2bool(global_parameters["features"]["textures"]) # Create the texture features if create_textures: - band_used_for_contours = 2 - in_tif = glob.glob( - op.join(bands_dir, '*{}*{:02}.jp2'.format(band_prefix, band_used_for_contours)))[0] - in_channel = 1 - - out_tif = op.join(out_dir_bands, 'density_contours.tif') - create_contours_density(in_tif, in_channel, out_tif, radius=3) - additional_bands.append(str(out_tif)) - out_tif = op.join(out_dir_bands, 'variation_coeff.tif') - create_variation_coeff(in_tif, in_channel, out_tif, radius=3) - additional_bands.append(str(out_tif)) + create_texture_feat(additional_bands, band_prefix, bands_dir, out_dir_bands) # Create time difference features - bands_num = [int(band) for band in global_parameters["features"]["time_difference_bands"]] - out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate') - for band_num in bands_num: - out_tif = op.join(out_dir_bands, ('time_' + str(band_num) + '.tif')) - create_time_difference_band(global_parameters, paths_parameters, band_num, out_tif, resolution=resolution) - additional_bands.append(str(out_tif)) + time_diff_feat(additional_bands, global_parameters, paths_parameters, resolution) # --- Create the main TIF with low resolution # create intermediate resolution files @@ -501,10 +473,7 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur new_indices = ['NDVI', 'NDWI'] out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate') additional_bands = [] - for indice in new_indices: - out_tif = op.join(out_dir_bands, (indice + '.tif')) - create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution) - additional_bands.append(str(out_tif)) + create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution) # create intermediate resolution files intermediate_bands_dir = op.join( @@ -513,15 +482,8 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur pixelresY = resolution # The bands to put into the heavy file - bands_num = [2, 3, 4, 10] - - intermediate_sizes_paths = [] - for band in bands_num: - in_band = str(op.join(bands_dir, band_prefix) + '{:02d}'.format(band)+'.jp2') - out_band = op.join(intermediate_bands_dir, op.basename(in_band)[0:-4]+'.tif') - - resize_band(in_band, out_band, pixelresX, pixelresY) - intermediate_sizes_paths.append(out_band) + intermediate_sizes_paths = put_band_heavy_tif(band_prefix, bands_dir, intermediate_bands_dir, + intermediate_sizes_paths, pixelresX, pixelresY) out_heavy_tif = op.join(global_parameters["user_choices"]["main_dir"], 'In_data', 'Image', global_parameters["user_choices"]["raw_img"])[0:-4]+'_H.tif' @@ -531,7 +493,6 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur compose_bands_heavy(intermediate_sizes_paths, str(out_heavy_tif)) if "user_function" in list(global_parameters["user_choices"].keys()) and global_parameters["user_choices"]["user_function"] != None: - print('ENTERED') user_process(raw_img = out_all_bands_tif, main_dir = global_parameters["user_choices"]["main_dir"], module_path = global_parameters["user_choices"]["user_module"], @@ -541,6 +502,61 @@ def create_image_compositions(global_parameters, location, paths_parameters, cur return +def put_band_heavy_tif(band_prefix, bands_dir, intermediate_bands_dir, intermediate_sizes_paths, pixelresX, pixelresY): + bands_num = [2, 3, 4, 10] + intermediate_sizes_paths = [] + for band in bands_num: + in_band = str(op.join(bands_dir, band_prefix) + '{:02d}'.format(band) + '.jp2') + out_band = op.join(intermediate_bands_dir, op.basename(in_band)[0:-4] + '.tif') + + resize_band(in_band, out_band, pixelresX, pixelresY) + intermediate_sizes_paths.append(out_band) + return intermediate_sizes_paths + + +def create_new_indices(additional_bands, bands_dir, new_indices, out_dir_bands, resolution): + for indice in new_indices: + out_tif = op.join(out_dir_bands, (indice + '.tif')) + create_specific_indices(bands_dir, out_tif, indice_name=indice, resolution=resolution) + additional_bands.append(str(out_tif)) + + +def time_diff_feat(additional_bands, global_parameters, paths_parameters, resolution): + bands_num = [int(band) for band in global_parameters["features"]["time_difference_bands"]] + out_dir_bands = op.join(global_parameters["user_choices"]["main_dir"], 'Intermediate') + for band_num in bands_num: + out_tif = op.join(out_dir_bands, ('time_' + str(band_num) + '.tif')) + create_time_difference_band(global_parameters, paths_parameters, band_num, out_tif, resolution=resolution) + additional_bands.append(str(out_tif)) + + +def use_dtm(additional_bands, global_parameters, location, out_dir_bands, resolution): + use_DTM = str2bool(global_parameters["features"]["DTM"]) + if use_DTM: + # Append the DTM model + out_dtm = op.join(out_dir_bands, ('DTM.tif')) + # try to append it. If an error occurs, the DTM probably does not exist + # and we will therefore skip this band + try: + dtm_addition(location, out_dtm, resolution=resolution) + additional_bands.append(str(out_dtm)) + except: + print('ERROR : THE DTM DOES NOT EXIST !!!') + + +def create_texture_feat(additional_bands, band_prefix, bands_dir, out_dir_bands): + band_used_for_contours = 2 + in_tif = glob.glob( + op.join(bands_dir, '*{}*{:02}.jp2'.format(band_prefix, band_used_for_contours)))[0] + in_channel = 1 + out_tif = op.join(out_dir_bands, 'density_contours.tif') + create_contours_density(in_tif, in_channel, out_tif, radius=3) + additional_bands.append(str(out_tif)) + out_tif = op.join(out_dir_bands, 'variation_coeff.tif') + create_variation_coeff(in_tif, in_channel, out_tif, radius=3) + additional_bands.append(str(out_tif)) + + def create_no_data_tif(global_parameters, paths_parameters, out_tif, dilation_radius=10): ''' Create the no_data TIF using both the clear and cloudy date. diff --git a/all_run_alcd.py b/all_run_alcd.py index 8f61a84..ce478c1 100644 --- a/all_run_alcd.py +++ b/all_run_alcd.py @@ -124,25 +124,7 @@ def run_all(part, global_parameters, paths_parameters, model_parameters, first_i main_dir, global_parameters, paths_parameters, raw_img_name, location, current_date, clear_date) if first_iteration == True: - # Create the directories - OTB_workflow.create_directories(global_parameters) - - # Copy the global_parameters files to save it - src = global_parameters["json_file"] - dst = op.join(global_parameters["user_choices"] - ["main_dir"], 'In_data', 'used_global_parameters.json') - shutil.copyfile(src, dst) - - # Create the images .tif and .jp2, i.e. the features - L1C_band_composition.create_image_compositions( - global_parameters, location, paths_parameters, current_date, heavy=True, force=force) - - # Create the empty layers - layers_creation.create_all_classes_empty_layers(global_parameters, force=force) - - # Fill automatically the no_data layer from the L1C missing - # pixels - layers_creation.create_no_data_shp(global_parameters,paths_parameters, force=force) + first_it_worklfow(current_date, force, global_parameters, location, paths_parameters) # ---------------------------------------------- # WAIT FOR USER MODIFICATION OF THE LAYERS IN LOCAL # ---------------------------------------------- @@ -197,6 +179,24 @@ def run_all(part, global_parameters, paths_parameters, model_parameters, first_i OTB_wf.create_contour_from_labeled(global_parameters, proceed=True) +def first_it_worklfow(current_date, force, global_parameters, location, paths_parameters): + # Create the directories + OTB_workflow.create_directories(global_parameters) + # Copy the global_parameters files to save it + src = global_parameters["json_file"] + dst = op.join(global_parameters["user_choices"] + ["main_dir"], 'In_data', 'used_global_parameters.json') + shutil.copyfile(src, dst) + # Create the images .tif and .jp2, i.e. the features + L1C_band_composition.create_image_compositions( + global_parameters, location, paths_parameters, current_date, heavy=True, force=force) + # Create the empty layers + layers_creation.create_all_classes_empty_layers(global_parameters, force=force) + # Fill automatically the no_data layer from the L1C missing + # pixels + layers_creation.create_no_data_shp(global_parameters, paths_parameters, force=force) + + def str2bool(v): ''' Use it to change multiple pseudo-boolean values to a real boolean diff --git a/metrics_exploitation.py b/metrics_exploitation.py index 6b7efbb..df5e052 100644 --- a/metrics_exploitation.py +++ b/metrics_exploitation.py @@ -370,45 +370,43 @@ def retrieve_Kfold_data(global_parameters, metrics_plotting=False, location='', recalls = [m[2] for m in all_metrics] precisions = [m[3] for m in all_metrics] - plt.errorbar(indices, means[0:4], stds[0:4], linestyle='', - marker='o', color='b') - met_nb = 0 - - for metric in [accuracies, f1scores, recalls, precisions]: - rnd = [(indices[met_nb] - 0.1 + 0.2*(float(j)/len(accuracies))) - for j in range(len(accuracies))] - plt.scatter(rnd, metric, color='k', marker='.', alpha=0.2) - met_nb += 1 - - plt.ylim(0.5, 1) - metrics_names = ['Accuracy\n{:.1f}%'.format(means[0]*100), - 'F1-score\n{:.1f}%'.format(means[1]*100), - 'Recall\n{:.1f}%'.format(means[2]*100), - 'Precision\n{:.1f}%'.format(means[3]*100)] - plt.xticks(indices, metrics_names) - - nb_dates = float(len(accuracies))/11 - plt.title( - 'Metrics of a {}-fold random cross-validation\n{}, {}'.format(len(accuracies), location, date)) - plt.xlabel('Score type') - plt.ylabel('Scores') - - # Custom legend - custom_lines = [] - custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='k', marker='.', alpha=0.2)) - custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='b', marker='o', alpha=1)) - legend_labels = ['Single fold point', 'Mean and std of all folds'] - plt.legend(custom_lines, legend_labels, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) - - #~ plt.show() - - out_fig = op.join(statistics_dir, 'kfold_metrics.png') - plt.savefig(out_fig, bbox_inches='tight') + plot_metrics(accuracies, date, f1scores, indices, location, means, precisions, recalls, statistics_dir, stds) #~ plt.close() return means, stds, all_metrics +def plot_metrics(accuracies, date, f1scores, indices, location, means, precisions, recalls, statistics_dir, stds): + plt.errorbar(indices, means[0:4], stds[0:4], linestyle='', + marker='o', color='b') + met_nb = 0 + for metric in [accuracies, f1scores, recalls, precisions]: + rnd = [(indices[met_nb] - 0.1 + 0.2 * (float(j) / len(accuracies))) + for j in range(len(accuracies))] + plt.scatter(rnd, metric, color='k', marker='.', alpha=0.2) + met_nb += 1 + plt.ylim(0.5, 1) + metrics_names = ['Accuracy\n{:.1f}%'.format(means[0] * 100), + 'F1-score\n{:.1f}%'.format(means[1] * 100), + 'Recall\n{:.1f}%'.format(means[2] * 100), + 'Precision\n{:.1f}%'.format(means[3] * 100)] + plt.xticks(indices, metrics_names) + nb_dates = float(len(accuracies)) / 11 + plt.title( + 'Metrics of a {}-fold random cross-validation\n{}, {}'.format(len(accuracies), location, date)) + plt.xlabel('Score type') + plt.ylabel('Scores') + # Custom legend + custom_lines = [] + custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='k', marker='.', alpha=0.2)) + custom_lines.append(Line2D([0], [0], color='w', markerfacecolor='b', marker='o', alpha=1)) + legend_labels = ['Single fold point', 'Mean and std of all folds'] + plt.legend(custom_lines, legend_labels, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) + # ~ plt.show() + out_fig = op.join(statistics_dir, 'kfold_metrics.png') + plt.savefig(out_fig, bbox_inches='tight') + + def load_previous_global_parameters(location, date): paths_configuration = read_paths_parameters(op.join('parameters_files', 'paths_configuration.json')) Data_ALCD_dir = paths_configuration["data_paths"]["data_alcd"] diff --git a/split_samples.py b/split_samples.py index 6501948..3526d3a 100644 --- a/split_samples.py +++ b/split_samples.py @@ -133,36 +133,38 @@ def split_points_sample(in_shp, train_shp, validation_shp, proportion, proportio else: print('FID {} not in any list'.format(current_FID)) - - elif proportion_type == 'global': # the proportion will be respected for the set of classes, i.e. # a class can be not represented in the validation set # Thus it is not recommended #Get the info of the shapefile - features_count = len(inLayer) - train_list, validation_list = get_random_splitting_lists(features_count, proportion) - print('{} training points will be taken'.format(len(train_list))) - print('{} validation points will be taken'.format(len(validation_list))) - - # Create the feature and set values - k = 0 - for point in inLayer: - if k in train_list: - trainLayer.CreateFeature(point) - elif k in validation_list: - validationLayer.CreateFeature(point) - else: - print('Feature {} not in any list'.format(k)) - k+=1 + global_proportion_type(inLayer, proportion, trainLayer, validationLayer) # Close DataSources inDataSource.Destroy() trainDataSource.Destroy() validationDataSource.Destroy() - return - + return + + +def global_proportion_type(inLayer, proportion, trainLayer, validationLayer): + features_count = len(inLayer) + train_list, validation_list = get_random_splitting_lists(features_count, proportion) + print('{} training points will be taken'.format(len(train_list))) + print('{} validation points will be taken'.format(len(validation_list))) + # Create the feature and set values + k = 0 + for point in inLayer: + if k in train_list: + trainLayer.CreateFeature(point) + elif k in validation_list: + validationLayer.CreateFeature(point) + else: + print('Feature {} not in any list'.format(k)) + k += 1 + + def k_split(in_shp, out_dir, K): ''' Split the in_shp in K different sets @@ -182,13 +184,9 @@ def k_split(in_shp, out_dir, K): srs = inLayer.GetSpatialRef() # get the field names - field_names = [] - for i in range(layerDefinition.GetFieldCount()): - field_names.append(layerDefinition.GetFieldDefn(i).GetName()) - - shpDriver = ogr.GetDriverByName("ESRI Shapefile") - + field_names = get_field_names(layerDefinition) + shpDriver = ogr.GetDriverByName("ESRI Shapefile") # each class will respect the proportion points_classes_list = [] @@ -248,8 +246,7 @@ def k_split(in_shp, out_dir, K): print('{} validation points will be taken'.format(len(validation_FID))) inLayer.ResetReading() # needs to be reset to be readable again - - + # Remove output shapefile if it already exists for dire in [train_shp, validation_shp]: if os.path.exists(dire): @@ -262,30 +259,36 @@ def k_split(in_shp, out_dir, K): validationDataSource = shpDriver.CreateDataSource(validation_shp) validationLayer = validationDataSource.CreateLayer("buff_layer", srs, geom_type=ogr.wkbPoint) - # Add all the fields - for field_name in field_names: - newField = ogr.FieldDefn(field_name, ogr.OFTInteger) - trainLayer.CreateField(newField) - validationLayer.CreateField(newField) - - - # Create the feature and set values - for point in inLayer: - current_FID = point.GetFID() - if current_FID in train_FID: - trainLayer.CreateFeature(point) - elif current_FID in validation_FID: - validationLayer.CreateFeature(point) - else: - print('FID {} not in any list'.format(current_FID)) - - + add_all_fields(field_names, inLayer, trainLayer, train_FID, validationLayer, validation_FID) # Close DataSources trainDataSource.Destroy() validationDataSource.Destroy() - inDataSource.Destroy() return + + +def get_field_names(layerDefinition): + field_names = [] + for i in range(layerDefinition.GetFieldCount()): + field_names.append(layerDefinition.GetFieldDefn(i).GetName()) + return field_names + + +def add_all_fields(field_names, inLayer, trainLayer, train_FID, validationLayer, validation_FID): + for field_name in field_names: + newField = ogr.FieldDefn(field_name, ogr.OFTInteger) + trainLayer.CreateField(newField) + validationLayer.CreateField(newField) + + # Create the feature and set values + for point in inLayer: + current_FID = point.GetFID() + if current_FID in train_FID: + trainLayer.CreateFeature(point) + elif current_FID in validation_FID: + validationLayer.CreateFeature(point) + else: + print('FID {} not in any list'.format(current_FID)) From 17c04289fe248e78f048c2490061b289f24cc955 Mon Sep 17 00:00:00 2001 From: cadauxe Date: Tue, 26 Aug 2025 15:47:45 +0200 Subject: [PATCH 4/5] feat: corrected api tests --- tests/test_run_alcd.py | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test_run_alcd.py b/tests/test_run_alcd.py index 1fcfeb1..21e870f 100644 --- a/tests/test_run_alcd.py +++ b/tests/test_run_alcd.py @@ -198,11 +198,11 @@ def test_run_alcd(alcd_paths: ALCDTestsData) -> None: location="Toulouse", wanted_date="20240305", clear_date="20240120", - first_iteration="1", - user_input="True", # corresponds to -f True - get_dates="False", - force="False", - kfold="False" + first_iteration=True, + user_input=1, + get_dates=False, + force=False, + kfold=False ) alcd_results, details = check_expected_alcd_results( alcd_paths.data_dir / "test_run_alcd" / "Toulouse_31TCJ_20240305" / "Out") @@ -234,11 +234,11 @@ def test_scikit_alcd(alcd_paths: ALCDTestsData) -> None: location="Toulouse", wanted_date="20240305", clear_date="20240120", - first_iteration="1", - user_input="True", - get_dates="False", - force="False", - kfold="False" + first_iteration=True, + user_input=1, + get_dates=False, + force=False, + kfold=False ) alcd_results, details = check_expected_alcd_results( alcd_paths.data_dir / "test_scikit_alcd" / "Toulouse_31TCJ_20240305" / "Out") @@ -278,11 +278,11 @@ def test_user_prim_alcd(alcd_paths: ALCDTestsData) -> None: location="Toulouse", wanted_date="20240305", clear_date="20240120", - first_iteration="0", - user_input="True", - get_dates="False", - force="False", - kfold="False" + first_iteration=True, + user_input=0, + get_dates=False, + force=True, + kfold=False ) shutil.copytree(alcd_paths.s2_data / "Toulouse_31TCJ_20240305" / "In_data" / "Image", output_dir / "In_data" / "Image", dirs_exist_ok=True) @@ -293,11 +293,11 @@ def test_user_prim_alcd(alcd_paths: ALCDTestsData) -> None: location="Toulouse", wanted_date="20240305", clear_date="20240120", - first_iteration="1", - user_input="True", - get_dates="False", - force="False", - kfold="False" + first_iteration=True, + user_input=1, + get_dates=False, + force=False, + kfold=False ) alcd_results, details = check_expected_alcd_results( @@ -332,11 +332,11 @@ def test_run_alcd_gen_features(alcd_paths: ALCDTestsData) -> None: location="Toulouse", wanted_date="20240305", clear_date="20240120", - first_iteration="0", - user_input="1", - get_dates="False", - force="True", - kfold="False" + first_iteration=True, + user_input=1, + get_dates=False, + force=True, + kfold=False ) alcd_results, details = check_expected_features_alcd( alcd_paths.data_dir / "test_gen_features" / "Toulouse_31TCJ_20240305" / "In_data" / "Image") From e3812e56244e77f8fda8faaed6a2d20e9222d557 Mon Sep 17 00:00:00 2001 From: cadauxe Date: Mon, 1 Sep 2025 09:25:56 +0200 Subject: [PATCH 5/5] fix: changed ci --- .github/workflows/build_and_test_app.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_and_test_app.yml b/.github/workflows/build_and_test_app.yml index 51a03bb..d7b1404 100644 --- a/.github/workflows/build_and_test_app.yml +++ b/.github/workflows/build_and_test_app.yml @@ -4,7 +4,6 @@ on: pull_request: branches: - "master" - - "review_reco" permissions: contents: read