diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 93611a83..41465b58 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -40,8 +40,8 @@ jobs: - name: Example inference run via Docker with default config and checkpoint run: > docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/outputs/default/:/outputs/ --ipc=host --shm-size=2gb myria3d @@ -55,15 +55,15 @@ jobs: - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. run: > docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/outputs/overlap/:/outputs/ --ipc=host --shm-size=2gb myria3d python run.py --config-path /inputs/ - --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 - predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt + --config-name FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8 + predict.ckpt_path=/inputs/FRACTAL-LidarHD_7cl_randlanet.ckpt datamodule.epsg=2154 predict.src_las=/inputs/792000_6272000_subset_buildings.las predict.output_dir=/outputs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d49b208..e0eabf91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # CHANGELOG + +## 3.9.0 - Update environment: - match pdal==2.10 - ign-pdal-tools : match v1.16.0 + add missing dependency @@ -9,6 +11,8 @@ - Add a github action workflow to run a trained model on the lidar-prod thresholds optimisation dataset (in order to automate thresholds optimization) +- new model weights: Deploy model trained on Fractal dataset (cf. https://huggingface.co/IGNF/FRACTAL-LidarHD_7cl_randlanet) + ### 3.8.4 - fix: move IoU appropriately to fix wrong device error created by a breaking change in torch when using DDP. diff --git a/myria3d/_version.py b/myria3d/_version.py index 0b4921c3..4c8919fe 100644 --- a/myria3d/_version.py +++ b/myria3d/_version.py @@ -1,4 +1,4 @@ -__version__ = "3.8.4" +__version__ = "3.9.0" if __name__ == "__main__": diff --git a/run.py b/run.py index 69af4e21..d010e2a4 100755 --- a/run.py +++ b/run.py @@ -22,8 +22,8 @@ TASK_NAME_DETECTION_STRING = "task.task_name=" DEFAULT_DIRECTORY = "trained_model_assets/" -DEFAULT_CONFIG_FILE = "proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml" -DEFAULT_CHECKPOINT = "proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt" +DEFAULT_CONFIG_FILE = "FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml" +DEFAULT_CHECKPOINT = "FRACTAL-LidarHD_7cl_randlanet.ckpt" DEFAULT_ENV = "placeholder.env" diff --git a/tests/myria3d/test_train_and_predict.py b/tests/myria3d/test_train_and_predict.py index 285fee57..7921708c 100644 --- a/tests/myria3d/test_train_and_predict.py +++ b/tests/myria3d/test_train_and_predict.py @@ -96,8 +96,8 @@ def test_predict_as_command(one_epoch_trained_RandLaNet_checkpoint, tmpdir): run_hydra_decorated_command(command) output_path = Path(tmpdir) / Path(abs_path_to_toy_LAS).name metadata = las_info.las_info_metadata(output_path) - out_pesg = las_info.get_epsg_from_header_info(metadata) - assert out_pesg == DEFAULT_EPSG + out_epsg = las_info.get_epsg_from_header_info(metadata) + assert out_epsg == DEFAULT_EPSG def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir): @@ -116,6 +116,7 @@ def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir): f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}", f"predict.src_las={abs_path_to_toy_LAS}", f"predict.output_dir={tmpdir}", + "datamodule.epsg=null", "+predict.interpolator.probas_to_save=[building,unclassified]", "task.task_name=predict", ] diff --git a/trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml b/trained_model_assets/FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml old mode 100644 new mode 100755 similarity index 55% rename from trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml rename to trained_model_assets/FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml index a9970595..df3ac6d7 --- a/trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml +++ b/trained_model_assets/FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml @@ -3,47 +3,15 @@ work_dir: ${hydra:runtime.cwd} debug: false print_config: true ignore_warnings: true -trainer: - _target_: pytorch_lightning.Trainer - min_epochs: 100 - max_epochs: 150 - log_every_n_steps: 1 - num_sanity_val_steps: 2 - accumulate_grad_batches: 3 - strategy: ddp - accelerator: gpu - devices: 2 datamodule: transforms: preparations: - train: - TargetTransform: - _target_: myria3d.pctl.transforms.transforms.TargetTransform - _args_: - - ${dataset_description.classification_preprocessing_dict} - - ${dataset_description.classification_dict} - DropPointsByClass: - _target_: myria3d.pctl.transforms.transforms.DropPointsByClass - GridSampling: - _target_: torch_geometric.transforms.GridSampling - _args_: - - 0.25 - MinimumNumNodes: - _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes - _args_: - - 300 - MaximumNumNodes: - _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes - _args_: - - 40000 - Center: - _target_: torch_geometric.transforms.Center eval: TargetTransform: _target_: myria3d.pctl.transforms.transforms.TargetTransform _args_: - - ${dataset_description.classification_preprocessing_dict} - - ${dataset_description.classification_dict} + - ${dataset_description.classification_preprocessing_dict} + - ${dataset_description.classification_dict} DropPointsByClass: _target_: myria3d.pctl.transforms.transforms.DropPointsByClass CopyFullPos: @@ -53,15 +21,15 @@ datamodule: GridSampling: _target_: torch_geometric.transforms.GridSampling _args_: - - 0.25 + - 0.25 MinimumNumNodes: _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes _args_: - - 300 + - 300 MaximumNumNodes: _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes _args_: - - 40000 + - 40000 CopySampledPos: _target_: myria3d.pctl.transforms.transforms.CopySampledPos Center: @@ -74,30 +42,19 @@ datamodule: GridSampling: _target_: torch_geometric.transforms.GridSampling _args_: - - 0.25 + - 0.25 MinimumNumNodes: _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes _args_: - - 300 + - 300 MaximumNumNodes: _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes _args_: - - 40000 + - 40000 CopySampledPos: _target_: myria3d.pctl.transforms.transforms.CopySampledPos Center: _target_: torch_geometric.transforms.Center - augmentations: - x_flip: - _target_: torch_geometric.transforms.RandomFlip - _args_: - - 0 - p: 0.5 - y_flip: - _target_: torch_geometric.transforms.RandomFlip - _args_: - - 1 - p: 0.5 normalizations: NullifyLowestZ: _target_: myria3d.pctl.transforms.transforms.NullifyLowestZ @@ -106,28 +63,25 @@ datamodule: subtile_width: ${datamodule.subtile_width} StandardizeRGBAndIntensity: _target_: myria3d.pctl.transforms.transforms.StandardizeRGBAndIntensity - augmentations_list: '${oc.dict.values: datamodule.transforms.augmentations}' - preparations_train_list: '${oc.dict.values: datamodule.transforms.preparations.train}' - preparations_eval_list: '${oc.dict.values: datamodule.transforms.preparations.eval}' - preparations_predict_list: '${oc.dict.values: datamodule.transforms.preparations.predict}' - normalizations_list: '${oc.dict.values: datamodule.transforms.normalizations}' + preparations_eval_list: "${oc.dict.values: datamodule.transforms.preparations.eval}" + preparations_predict_list: "${oc.dict.values: datamodule.transforms.preparations.predict}" + normalizations_list: "${oc.dict.values: datamodule.transforms.normalizations}" _target_: myria3d.pctl.datamodule.hdf5.HDF5LidarDataModule + epsg: 2154 data_dir: null - epsg: null split_csv_path: null - hdf5_file_path: /var/data/cgaydon/data/20220607_151_dalles_proto.myria3d-V2.4.hdf5 + hdf5_file_path: null points_pre_transform: _target_: functools.partial _args_: - - ${get_method:myria3d.pctl.points_pre_transform.lidar_hd.lidar_hd_pre_transform} + - ${get_method:myria3d.pctl.points_pre_transform.lidar_hd.lidar_hd_pre_transform} pre_filter: _target_: functools.partial _args_: - - ${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points} + - ${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points} min_num_nodes: 1 tile_width: 1000 subtile_width: 50 - subtile_overlap_train: 0 subtile_overlap_predict: ${predict.subtile_overlap} batch_size: 10 num_workers: 3 @@ -137,29 +91,7 @@ dataset_description: classification_preprocessing_dict: 3: 5 4: 5 - 160: 64 - 161: 64 - 162: 64 - 0: 1 - 7: 1 - 46: 1 - 47: 1 - 48: 1 - 49: 1 - 50: 1 - 51: 1 - 52: 1 - 53: 1 - 54: 1 - 55: 1 - 56: 1 - 57: 1 - 58: 1 - 66: 1 - 67: 1 - 77: 1 - 155: 1 - 204: 1 + 66: 65 classification_dict: 1: unclassified 2: ground @@ -168,14 +100,6 @@ dataset_description: 9: water 17: bridge 64: lasting_above - class_weights: - - 0.25 - - 0.1 - - 0.1 - - 0.5 - - 2.0 - - 2.0 - - 2.0 d_in: 9 num_classes: 7 callbacks: @@ -188,9 +112,6 @@ callbacks: _target_: pytorch_lightning.callbacks.LearningRateMonitor logging_interval: step log_momentum: true - log_iou_by_class: - _target_: myria3d.callbacks.logging_callbacks.LogIoUByClass - classification_dict: ${dataset_description.classification_dict} model_checkpoint: _target_: pytorch_lightning.callbacks.ModelCheckpoint monitor: val/loss_epoch @@ -208,27 +129,11 @@ callbacks: patience: 6 min_delta: 0 model: - optimizer: - _target_: functools.partial - _args_: - - ${get_method:torch.optim.Adam} - lr: ${model.lr} - lr_scheduler: - _target_: functools.partial - _args_: - - ${get_method:torch.optim.lr_scheduler.ReduceLROnPlateau} - mode: min - factor: 0.5 - patience: 20 - cooldown: 5 - verbose: true - criterion: - _target_: torch.nn.CrossEntropyLoss - label_smoothing: 0.0 _target_: myria3d.models.model.Model d_in: ${dataset_description.d_in} num_classes: ${dataset_description.num_classes} - ckpt_path: null + classification_dict: ${dataset_description.classification_dict} + ckpt_path: FRACTAL-LidarHD_7cl_randlanet.ckpt neural_net_class_name: PyGRandLANet neural_net_hparams: num_features: ${model.d_in} @@ -238,37 +143,27 @@ model: return_logits: true interpolation_k: ${predict.interpolator.interpolation_k} num_workers: 4 - iou: - _target_: functools.partial - _args_: - - ${get_method:torchmetrics.JaccardIndex} - - ${model.num_classes} - absent_score: 1.0 - momentum: 0.9 - monitor: val/loss_epoch - lr: 0.003933709606504788 logger: comet: _target_: pytorch_lightning.loggers.comet.CometLogger api_key: ${oc.env:COMET_API_TOKEN} workspace: ${oc.env:COMET_WORKSPACE} project_name: ${oc.env:COMET_PROJECT_NAME} - experiment_name: '[V3.0.2-BS10xMAX40000pts] RandLaNet_base_run_FR_pyg_randla_net_NoRS' + experiment_name: DATAPAPER-LidarHD-20240416_100k_fractal-6GPUs auto_log_co2: false disabled: false task: task_name: predict - auto_lr_find: false predict: src_las: /path/to/input.las output_dir: /path/to/output_dir/ - ckpt_path: trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt - subtile_overlap: 0 + ckpt_path: trained_model_assets/FRACTAL-LidarHD_7cl_randlanet.ckpt gpus: 0 + subtile_overlap: 0 interpolator: _target_: myria3d.models.interpolation.Interpolator interpolation_k: 10 classification_dict: ${dataset_description.classification_dict} - probas_to_save: [building,ground] - predicted_classification_channel: confidence + probas_to_save: [building, ground] + predicted_classification_channel: PredictedClassification entropy_channel: entropy diff --git a/trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt b/trained_model_assets/FRACTAL-LidarHD_7cl_randlanet.ckpt old mode 100644 new mode 100755 similarity index 61% rename from trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt rename to trained_model_assets/FRACTAL-LidarHD_7cl_randlanet.ckpt index e1bbdf46..db1af7c7 Binary files a/trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt and b/trained_model_assets/FRACTAL-LidarHD_7cl_randlanet.ckpt differ