feat: update chapter 03 to use new LAIF container

03-file-formats/Makefile

@@ -4,29 +4,20 @@ build:
 convert:
 	sbatch convert.sh "squashfs" && sbatch convert.sh "lmdb" && sbatch convert.sh "hdf5"
 
-clean:
-	rm -rf venv-extension data-formats
-
 bench-seq:
-	sbatch run-scripts/simple-benchmarks/run-comp-seq.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-seq.sh "lmdb" && sbatch run-scripts/simple-benchmarks/run-comp-seq.sh "hdf5"
+	sbatch run-scripts/simple-benchmarks/run-comp-tiny-seq.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-tiny-seq.sh "lmdb" && sbatch run-scripts/simple-benchmarks/run-comp-tiny-seq.sh "hdf5"
 
 bench-par:
-	sbatch run-scripts/simple-benchmarks/run-comp-tiny.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-tiny.sh "lmdb" && sbatch run-scripts/simple-benchmarks/run-comp-tiny.sh "hdf5"
-
-bench-large:
-	sbatch run-scripts/simple-benchmarks/run-comp-large.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-large.sh "lmdb"
-
-bench-full:
-	sbatch run-scripts/simple-benchmarks/run-comp-large-full.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-large-full.sh "lmdb"
+	sbatch run-scripts/simple-benchmarks/run-comp-tiny-par.sh "squashfs" && sbatch run-scripts/simple-benchmarks/run-comp-tiny-par.sh "lmdb" && sbatch run-scripts/simple-benchmarks/run-comp-tiny-par.sh "hdf5"
 
 post-seq:
-	python run-scripts/simple-benchmarks/post-process.py -d 'seq'
+	python3 run-scripts/simple-benchmarks/post-process.py -d 'tiny-seq'
 
 post-par:
-	python run-scripts/simple-benchmarks/post-process.py -d 'tiny'
+	python3 run-scripts/simple-benchmarks/post-process.py -d 'tiny-par'
 
-post-large:
-	python run-scripts/simple-benchmarks/post-process.py -d 'large'
+bench-vit:
+	sbatch run-scripts/training-benchmarks/run-comp-vision-transformer.sh "squashfs" && sbatch run-scripts/training-benchmarks/run-comp-vision-transformer.sh "lmdb" && sbatch run-scripts/training-benchmarks/run-comp-vision-transformer.sh "hdf5"
 
-post-large-full:
-	python run-scripts/simple-benchmarks/post-process.py -d 'large-full'
+post-vit:
+	python3 run-scripts/training-benchmarks/post-process.py

03-file-formats/convert.sh

@@ -1,24 +1,29 @@
 #!/bin/bash
-#SBATCH --account=project_462000002
+#SBATCH --account=project_xxxxxxxxx
 #SBATCH --partition=small
 #SBATCH --ntasks=1
 #SBATCH --cpus-per-task=16
 #SBATCH --mem-per-cpu=1750
-#SBATCH --time=0:10:00
+#SBATCH --time=0:30:00
 
-module use /appl/local/training/modules/AI-20240529
-module load singularity-userfilesystems singularity-CPEbits
+# this module facilitates the use of singularity containers on LUMI
+module purge
+module use /appl/local/laifs/modules
+module load lumi-aif-singularity-bindings
+
+# choose container
+SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260415_130625/lumi-multitorch-full-u24r70f21m50t210-20260415_130625.sif
 
-CONTAINER=/appl/local/containers/sif-images/lumi-pytorch-rocm-6.2.1-python-3.12-pytorch-20240918-vllm-4075b35.sif
 
 if [[ $1 == 'squashfs' ]]; then
     mkdir -p data-formats/squashfs/
-    time srun bash -c 'mksquashfs data-formats/raw/tiny-imagenet-200/val/ data-formats/squashfs/val.squashfs -processors 16 -no-progress'
-    time srun bash -c 'mksquashfs data-formats/raw/tiny-imagenet-200/train/ data-formats/squashfs/train.squashfs -processors 16 -no-progress'
+    time srun bash -c 'mksquashfs data-formats/raw/tiny-imagenet-200/val/ data-formats/squashfs/val.squashfs -processors 16 -no-progress -no-xattrs -noappend'
+    time srun bash -c 'mksquashfs data-formats/raw/tiny-imagenet-200/train/ data-formats/squashfs/train.squashfs -processors 16 -no-progress -no-xattrs -noappend'
 elif [[ $1 == 'lmdb' ]]; then
     mkdir -p data-formats/lmdb/
-    time srun singularity exec $CONTAINER bash -c '$WITH_CONDA && source venv-extension/bin/activate && python scripts/lmdb/convert_to_lmdb.py'
+    export SINGULARITYENV_PREPEND_PATH=/user-software/bin # gives access to packages inside the container
+    time srun singularity run -B venv.sqsh:/user-software:image-src=/ $SIF bash -c 'python scripts/lmdb/convert_to_lmdb.py'
 elif [[ $1 == 'hdf5' ]]; then
     mkdir -p data-formats/hdf5/
-    time srun singularity exec $CONTAINER bash -c '$WITH_CONDA && source venv-extension/bin/activate && python scripts/hdf5/convert_to_hdf5.py'
+    time srun singularity run $SIF bash -c 'python scripts/hdf5/convert_to_hdf5.py'
 fi

03-file-formats/install_venv.sh

@@ -1,11 +1,20 @@
 #!/bin/bash
-module use /appl/local/training/modules/AI-20240529
-module load singularity-userfilesystems singularity-CPEbits
-
-CONTAINER=/appl/local/containers/sif-images/lumi-pytorch-rocm-6.2.1-python-3.12-pytorch-20240918-vllm-4075b35.sif
-
-if [ -d "venv-extension" ]; then echo 'Removing existing venv-extension'; rm -Rf venv-extension; fi
-
-singularity exec $CONTAINER bash -c '$WITH_CONDA && python -m venv venv-extension --system-site-packages && source venv-extension/bin/activate && python -m pip install -r venv-requirements.txt'
+module purge
+export BUILD_DIR=env-temp
+export SQUASHFS_NAME=venv.sqsh
+export SIF=/appl/local/laifs/containers/lumi-multitorch-u24r70f21m50t210-20260415_130625/lumi-multitorch-full-u24r70f21m50t210-20260415_130625.sif
 
+# creating the venv
+echo "Creating venv in: $BUILD_DIR"
+mkdir $BUILD_DIR
+singularity exec -B "$BUILD_DIR":/user-software "$SIF" bash -c '
+set -euo pipefail
+python -m venv /user-software --system-site-packages
+/user-software/bin/python -m pip install lmdb
+'
 
+# creating the squashfs file and removing the venv
+echo "Creating squashfs in: $SQUASHFS_NAME"
+mksquashfs $BUILD_DIR $SQUASHFS_NAME -processors 1 -no-xattrs
+rm -rf $BUILD_DIR
+echo "done"