Staging fractal lidar hd 7cl randlanet

.github/workflows/cicd.yaml

@@ -40,8 +40,8 @@ jobs:
       - name: Example inference run via Docker with default config and checkpoint
         run: >
           docker run
-          -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-          -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
+          -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/inputs/:/inputs/
+          -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/outputs/default/:/outputs/
           --ipc=host
           --shm-size=2gb
           myria3d
@@ -55,15 +55,15 @@ jobs:
       - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results.
         run: >
           docker run
-          -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/
-          -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/
+          -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/inputs/:/inputs/
+          -v /var/data/cicd/CICD_github_assets/myria3d_V3.9.0/outputs/overlap/:/outputs/
           --ipc=host
           --shm-size=2gb
           myria3d
           python run.py
           --config-path /inputs/
-          --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0
-          predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
+          --config-name FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8
+          predict.ckpt_path=/inputs/FRACTAL-LidarHD_7cl_randlanet.ckpt
           datamodule.epsg=2154
           predict.src_las=/inputs/792000_6272000_subset_buildings.las
           predict.output_dir=/outputs/

CHANGELOG.md

@@ -1,4 +1,6 @@
 # CHANGELOG
+
+## 3.9.0
 - Update environment:
   - match pdal==2.10
   - ign-pdal-tools :  match v1.16.0 + add missing dependency
@@ -9,6 +11,8 @@
 - Add a github action workflow to run a trained model on the lidar-prod thresholds optimisation dataset
 (in order to automate thresholds optimization)
 
+- new model weights: Deploy model trained on Fractal dataset (cf. https://huggingface.co/IGNF/FRACTAL-LidarHD_7cl_randlanet)
+
 ### 3.8.4
 - fix: move IoU appropriately to fix wrong device error created by a breaking change in torch when using DDP.
 

myria3d/_version.py

@@ -1,4 +1,4 @@
-__version__ = "3.8.4"
+__version__ = "3.9.0"
 
 
 if __name__ == "__main__":

run.py

@@ -22,8 +22,8 @@
 
 TASK_NAME_DETECTION_STRING = "task.task_name="
 DEFAULT_DIRECTORY = "trained_model_assets/"
-DEFAULT_CONFIG_FILE = "proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml"
-DEFAULT_CHECKPOINT = "proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt"
+DEFAULT_CONFIG_FILE = "FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml"
+DEFAULT_CHECKPOINT = "FRACTAL-LidarHD_7cl_randlanet.ckpt"
 DEFAULT_ENV = "placeholder.env"
 
 

tests/myria3d/test_train_and_predict.py

@@ -96,8 +96,8 @@ def test_predict_as_command(one_epoch_trained_RandLaNet_checkpoint, tmpdir):
     run_hydra_decorated_command(command)
     output_path = Path(tmpdir) / Path(abs_path_to_toy_LAS).name
     metadata = las_info.las_info_metadata(output_path)
-    out_pesg = las_info.get_epsg_from_header_info(metadata)
-    assert out_pesg == DEFAULT_EPSG
+    out_epsg = las_info.get_epsg_from_header_info(metadata)
+    assert out_epsg == DEFAULT_EPSG
 
 
 def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir):
@@ -116,6 +116,7 @@ def test_command_without_epsg(one_epoch_trained_RandLaNet_checkpoint, tmpdir):
         f"predict.ckpt_path={one_epoch_trained_RandLaNet_checkpoint}",
         f"predict.src_las={abs_path_to_toy_LAS}",
         f"predict.output_dir={tmpdir}",
+        "datamodule.epsg=null",
         "+predict.interpolator.probas_to_save=[building,unclassified]",
         "task.task_name=predict",
     ]

trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0.yaml → trained_model_assets/FRACTAL-LidarHD_7cl_randlanet-inference-Myria3DV3.8.yaml

@@ -3,47 +3,15 @@ work_dir: ${hydra:runtime.cwd}
 debug: false
 print_config: true
 ignore_warnings: true
-trainer:
-  _target_: pytorch_lightning.Trainer
-  min_epochs: 100
-  max_epochs: 150
-  log_every_n_steps: 1
-  num_sanity_val_steps: 2
-  accumulate_grad_batches: 3
-  strategy: ddp
-  accelerator: gpu
-  devices: 2
 datamodule:
   transforms:
     preparations:
-      train:
-        TargetTransform:
-          _target_: myria3d.pctl.transforms.transforms.TargetTransform
-          _args_:
-          - ${dataset_description.classification_preprocessing_dict}
-          - ${dataset_description.classification_dict}
-        DropPointsByClass:
-          _target_: myria3d.pctl.transforms.transforms.DropPointsByClass
-        GridSampling:
-          _target_: torch_geometric.transforms.GridSampling
-          _args_:
-          - 0.25
-        MinimumNumNodes:
-          _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes
-          _args_:
-          - 300
-        MaximumNumNodes:
-          _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes
-          _args_:
-          - 40000
-        Center:
-          _target_: torch_geometric.transforms.Center
       eval:
         TargetTransform:
           _target_: myria3d.pctl.transforms.transforms.TargetTransform
           _args_:
-          - ${dataset_description.classification_preprocessing_dict}
-          - ${dataset_description.classification_dict}
+            - ${dataset_description.classification_preprocessing_dict}
+            - ${dataset_description.classification_dict}
         DropPointsByClass:
           _target_: myria3d.pctl.transforms.transforms.DropPointsByClass
         CopyFullPos:
@@ -53,15 +21,15 @@ datamodule:
         GridSampling:
           _target_: torch_geometric.transforms.GridSampling
           _args_:
-          - 0.25
+            - 0.25
         MinimumNumNodes:
           _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes
           _args_:
-          - 300
+            - 300
         MaximumNumNodes:
           _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes
           _args_:
-          - 40000
+            - 40000
         CopySampledPos:
           _target_: myria3d.pctl.transforms.transforms.CopySampledPos
         Center:
@@ -74,30 +42,19 @@ datamodule:
         GridSampling:
           _target_: torch_geometric.transforms.GridSampling
           _args_:
-          - 0.25
+            - 0.25
         MinimumNumNodes:
           _target_: myria3d.pctl.transforms.transforms.MinimumNumNodes
           _args_:
-          - 300
+            - 300
         MaximumNumNodes:
           _target_: myria3d.pctl.transforms.transforms.MaximumNumNodes
           _args_:
-          - 40000
+            - 40000
         CopySampledPos:
           _target_: myria3d.pctl.transforms.transforms.CopySampledPos
         Center:
           _target_: torch_geometric.transforms.Center
-    augmentations:
-      x_flip:
-        _target_: torch_geometric.transforms.RandomFlip
-        _args_:
-        - 0
-        p: 0.5
-      y_flip:
-        _target_: torch_geometric.transforms.RandomFlip
-        _args_:
-        - 1
-        p: 0.5
     normalizations:
       NullifyLowestZ:
         _target_: myria3d.pctl.transforms.transforms.NullifyLowestZ
@@ -106,28 +63,25 @@ datamodule:
         subtile_width: ${datamodule.subtile_width}
       StandardizeRGBAndIntensity:
         _target_: myria3d.pctl.transforms.transforms.StandardizeRGBAndIntensity
-    augmentations_list: '${oc.dict.values: datamodule.transforms.augmentations}'
-    preparations_train_list: '${oc.dict.values: datamodule.transforms.preparations.train}'
-    preparations_eval_list: '${oc.dict.values: datamodule.transforms.preparations.eval}'
-    preparations_predict_list: '${oc.dict.values: datamodule.transforms.preparations.predict}'
-    normalizations_list: '${oc.dict.values: datamodule.transforms.normalizations}'
+    preparations_eval_list: "${oc.dict.values: datamodule.transforms.preparations.eval}"
+    preparations_predict_list: "${oc.dict.values: datamodule.transforms.preparations.predict}"
+    normalizations_list: "${oc.dict.values: datamodule.transforms.normalizations}"
   _target_: myria3d.pctl.datamodule.hdf5.HDF5LidarDataModule
+  epsg: 2154
   data_dir: null
-  epsg: null
   split_csv_path: null
-  hdf5_file_path: /var/data/cgaydon/data/20220607_151_dalles_proto.myria3d-V2.4.hdf5
+  hdf5_file_path: null
   points_pre_transform:
     _target_: functools.partial
     _args_:
-    - ${get_method:myria3d.pctl.points_pre_transform.lidar_hd.lidar_hd_pre_transform}
+      - ${get_method:myria3d.pctl.points_pre_transform.lidar_hd.lidar_hd_pre_transform}
   pre_filter:
     _target_: functools.partial
     _args_:
-    - ${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points}
+      - ${get_method:myria3d.pctl.dataset.utils.pre_filter_below_n_points}
     min_num_nodes: 1
   tile_width: 1000
   subtile_width: 50
-  subtile_overlap_train: 0
   subtile_overlap_predict: ${predict.subtile_overlap}
   batch_size: 10
   num_workers: 3
@@ -137,29 +91,7 @@ dataset_description:
   classification_preprocessing_dict:
     3: 5
     4: 5
-    160: 64
-    161: 64
-    162: 64
-    0: 1
-    7: 1
-    46: 1
-    47: 1
-    48: 1
-    49: 1
-    50: 1
-    51: 1
-    52: 1
-    53: 1
-    54: 1
-    55: 1
-    56: 1
-    57: 1
-    58: 1
-    66: 1
-    67: 1
-    77: 1
-    155: 1
-    204: 1
+    66: 65
   classification_dict:
     1: unclassified
     2: ground
@@ -168,14 +100,6 @@ dataset_description:
     9: water
     17: bridge
     64: lasting_above
-  class_weights:
-  - 0.25
-  - 0.1
-  - 0.1
-  - 0.5
-  - 2.0
-  - 2.0
-  - 2.0
   d_in: 9
   num_classes: 7
 callbacks:
@@ -188,9 +112,6 @@ callbacks:
     _target_: pytorch_lightning.callbacks.LearningRateMonitor
     logging_interval: step
     log_momentum: true
-  log_iou_by_class:
-    _target_: myria3d.callbacks.logging_callbacks.LogIoUByClass
-    classification_dict: ${dataset_description.classification_dict}
   model_checkpoint:
     _target_: pytorch_lightning.callbacks.ModelCheckpoint
     monitor: val/loss_epoch
@@ -208,27 +129,11 @@ callbacks:
     patience: 6
     min_delta: 0
 model:
-  optimizer:
-    _target_: functools.partial
-    _args_:
-    - ${get_method:torch.optim.Adam}
-    lr: ${model.lr}
-  lr_scheduler:
-    _target_: functools.partial
-    _args_:
-    - ${get_method:torch.optim.lr_scheduler.ReduceLROnPlateau}
-    mode: min
-    factor: 0.5
-    patience: 20
-    cooldown: 5
-    verbose: true
-  criterion:
-    _target_: torch.nn.CrossEntropyLoss
-    label_smoothing: 0.0
   _target_: myria3d.models.model.Model
   d_in: ${dataset_description.d_in}
   num_classes: ${dataset_description.num_classes}
-  ckpt_path: null
+  classification_dict: ${dataset_description.classification_dict}
+  ckpt_path: FRACTAL-LidarHD_7cl_randlanet.ckpt
   neural_net_class_name: PyGRandLANet
   neural_net_hparams:
     num_features: ${model.d_in}
@@ -238,37 +143,27 @@ model:
     return_logits: true
   interpolation_k: ${predict.interpolator.interpolation_k}
   num_workers: 4
-  iou:
-    _target_: functools.partial
-    _args_:
-    - ${get_method:torchmetrics.JaccardIndex}
-    - ${model.num_classes}
-    absent_score: 1.0
-  momentum: 0.9
-  monitor: val/loss_epoch
-  lr: 0.003933709606504788
 logger:
   comet:
     _target_: pytorch_lightning.loggers.comet.CometLogger
     api_key: ${oc.env:COMET_API_TOKEN}
     workspace: ${oc.env:COMET_WORKSPACE}
     project_name: ${oc.env:COMET_PROJECT_NAME}
-    experiment_name: '[V3.0.2-BS10xMAX40000pts] RandLaNet_base_run_FR_pyg_randla_net_NoRS'
+    experiment_name: DATAPAPER-LidarHD-20240416_100k_fractal-6GPUs
     auto_log_co2: false
     disabled: false
 task:
   task_name: predict
-  auto_lr_find: false
 predict:
   src_las: /path/to/input.las
   output_dir: /path/to/output_dir/
-  ckpt_path: trained_model_assets/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt
-  subtile_overlap: 0
+  ckpt_path: trained_model_assets/FRACTAL-LidarHD_7cl_randlanet.ckpt
   gpus: 0
+  subtile_overlap: 0
   interpolator:
     _target_: myria3d.models.interpolation.Interpolator
     interpolation_k: 10
     classification_dict: ${dataset_description.classification_dict}
-    probas_to_save: [building,ground]
-    predicted_classification_channel: confidence
+    probas_to_save: [building, ground]
+    predicted_classification_channel: PredictedClassification
     entropy_channel: entropy