Decimals are actually decimals now, fix issue with params in debug, add max_iterations to validate

magbak · magbak · commit 24a11ca8b628 · 2026-04-12T08:50:56.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -79,6 +79,7 @@ ryu = "1.0.20"
 nohash-hasher = "0.2.0"
 arrow = "56"
 ordered-float = "5.1.0"
+rust_decimal = "1.41.0"
 
 [patch.crates-io]
 #polars = { git = 'https://github.com/pola-rs/polars', rev="665722ac3f3664c589c4827208d173cc16f0ec68" }
diff --git a/lib/maplib/src/model.rs b/lib/maplib/src/model.rs
@@ -474,6 +474,7 @@ impl Model {
         deactivate_shapes: Vec<NamedNode>,
         dry_run: bool,
         serial: bool,
+        max_iterations: Option<usize>,
         debug_rules: bool,
     ) -> Result<ValidationReport, MaplibError> {
         let mut res = validate(
@@ -492,6 +493,7 @@ impl Model {
             dry_run,
             Some(self.prefixes.clone()),
             serial,
+            max_iterations,
             debug_rules,
         )
         .map_err(|x| MaplibError::ShaclError(x))?;
diff --git a/lib/query_processing/src/aggregates.rs b/lib/query_processing/src/aggregates.rs
@@ -97,7 +97,6 @@ pub fn min(solution_mappings: &SolutionMappings, column_context: &Context) -> (E
     } else {
         expr_rdf_node_type.clone()
     };
-
     let out_expr = col(column_context.as_str()).min();
 
     (out_expr, out_rdf_node_type)
diff --git a/lib/query_processing/src/expressions.rs b/lib/query_processing/src/expressions.rs
@@ -475,8 +475,8 @@ pub fn if_expression(
     let right_exploded = exploded.pop().unwrap();
     let mid_exploded = exploded.pop().unwrap();
 
-    let mut right_exploded_types: HashSet<_> = right_exploded.iter().map(|(t,..)|t.clone()).collect();
-    let mut mid_exploded_types: HashSet<_> = mid_exploded.iter().map(|(t,..)|t.clone()).collect();
+    let right_exploded_types: HashSet<_> = right_exploded.iter().map(|(t,..)|t.clone()).collect();
+    let mid_exploded_types: HashSet<_> = mid_exploded.iter().map(|(t,..)|t.clone()).collect();
 
 
     let mut mid_exprs = vec![];
diff --git a/lib/representation/Cargo.toml b/lib/representation/Cargo.toml
@@ -27,6 +27,7 @@ nohash-hasher.workspace = true
 uuid.workspace = true
 arrow.workspace = true
 memchr.workspace = true
+rust_decimal.workspace = true
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("gil-refs", "rdf-star"))'] }
diff --git a/lib/representation/src/base_rdf_type.rs b/lib/representation/src/base_rdf_type.rs
@@ -10,7 +10,7 @@ use oxrdf::{NamedNode, NamedNodeRef, NamedOrBlankNode, Term};
 use polars::datatypes::{DataType, Field, PlSmallStr};
 use spargebra::term::GroundTerm;
 use std::fmt::{Display, Formatter};
-use crate::rdf_to_polars::{default_time_unit, default_time_zone};
+use crate::rdf_to_polars::{default_decimal_type, default_time_unit, default_time_zone};
 
 #[derive(Debug, Clone, Ord, PartialOrd, PartialEq, Eq, Hash)]
 pub enum BaseRDFNodeType {
@@ -215,7 +215,8 @@ fn literal_type(
         xsd::INT => DataType::Int32,
         xsd::SHORT => DataType::Int16,
         xsd::BYTE => DataType::Int8,
-        xsd::DOUBLE | xsd::DECIMAL => DataType::Float64,
+        xsd::DOUBLE => DataType::Float64,
+        xsd::DECIMAL => default_decimal_type(),
         xsd::FLOAT => DataType::Float32,
         xsd::BOOLEAN => DataType::Boolean,
         rdf::LANG_STRING => {
diff --git a/lib/representation/src/rdf_to_polars.rs b/lib/representation/src/rdf_to_polars.rs
@@ -8,6 +8,8 @@ use polars::prelude::{
     as_struct, lit, AnyValue, DataType, Expr, LiteralValue, NamedFrom, PlSmallStr, Scalar, Series,
     TimeUnit, TimeZone,
 };
+use polars_core::prelude::{Int128Chunked, IntoSeries, NewChunkedArray};
+use rust_decimal::Decimal;
 use std::ops::Deref;
 use std::str::FromStr;
 use tracing::warn;
@@ -239,11 +241,16 @@ pub fn rdf_literal_to_polars_literal_value_impl(
             LiteralValue::Scalar(Scalar::null(DataType::Date))
         }
     } else if datatype == xsd::DECIMAL {
-        if let Ok(d) = f64::from_str(value) {
-            LiteralValue::Scalar(Scalar::from(d))
+        if let Ok(mut d) = Decimal::from_str(value) {
+            d.rescale(default_decimal_scale() as u32);
+            LiteralValue::Scalar(Scalar::new_decimal(
+                d.mantissa(),
+                default_decimal_precision(),
+                default_decimal_scale(),
+            ))
         } else {
             warn!("Could not parse xsd:decimal {value}");
-            LiteralValue::Scalar(Scalar::null(DataType::Float64))
+            LiteralValue::Scalar(Scalar::null(default_decimal_type()))
         }
     } else {
         LiteralValue::Scalar(Scalar::from(PlSmallStr::from_string(value.to_string())))
@@ -431,6 +438,24 @@ pub fn polars_literal_values_to_series(literal_values: Vec<LiteralValue>, name:
                         })
                         .collect::<Vec<Option<f64>>>(),
                 ),
+                AnyValue::Decimal(_, ..) => {
+                    let i128ch = Int128Chunked::from_iter_options(
+                        name.into(),
+                        values.into_iter().map(|x| {
+                            if let AnyValue::Decimal(v, ..) = x {
+                                Some(v)
+                            } else {
+                                None
+                            }
+                        }),
+                    );
+                    let dec = i128ch
+                        .into_decimal(default_decimal_precision(), default_decimal_scale())
+                        .unwrap();
+                    let mut ser = dec.into_series();
+                    ser.rename(PlSmallStr::from_str(name));
+                    ser
+                }
                 AnyValue::Datetime(_, t, tz) => Series::new(
                     name.into(),
                     values
@@ -515,3 +540,15 @@ pub fn default_time_unit() -> TimeUnit {
 pub fn default_time_zone() -> TimeZone {
     TimeZone::UTC
 }
+
+pub fn default_decimal_type() -> DataType {
+    DataType::Decimal(default_decimal_precision(), default_decimal_scale())
+}
+
+pub fn default_decimal_precision() -> usize {
+    38
+}
+
+pub fn default_decimal_scale() -> usize {
+    12
+}
diff --git a/lib/triplestore/src/sparql/debug.rs b/lib/triplestore/src/sparql/debug.rs
@@ -217,7 +217,7 @@ impl Triplestore {
                     },
                     None,
                     &Context::new(),
-                    &None,
+                    &parameters,
                     Pushdowns::new(),
                     qs,
                     qg,
diff --git a/lib/triplestore/src/storage/so_index.rs b/lib/triplestore/src/storage/so_index.rs
@@ -6,17 +6,14 @@ use polars_core::datatypes::{
     BooleanChunked, Int16Chunked, Int8Chunked, UInt16Chunked, UInt8Chunked,
 };
 use polars_core::frame::DataFrame;
-use polars_core::prelude::{
-    Column, Float32Chunked, Float64Chunked, Int32Chunked, Int64Chunked, IntoColumn, UInt32Chunked,
-    UInt64Chunked,
-};
+use polars_core::prelude::{Column, DecimalChunked, Float32Chunked, Float64Chunked, Int128Chunked, Int32Chunked, Int64Chunked, IntoColumn, NewChunkedArray, UInt32Chunked, UInt64Chunked};
 use polars_core::series::Series;
 use representation::{
     BaseRDFNodeType, LANG_STRING_LANG_FIELD, LANG_STRING_VALUE_FIELD, OBJECT_COL_NAME,
     SUBJECT_COL_NAME,
 };
 use std::collections::HashSet;
-use representation::rdf_to_polars::{default_time_unit, default_time_zone};
+use representation::rdf_to_polars::{default_decimal_precision, default_decimal_scale, default_time_unit, default_time_zone};
 
 pub fn bool_chunked(c: &Column) -> &BooleanChunked {
     c.bool().unwrap()
@@ -66,6 +63,10 @@ pub fn datetime_chunked(c: &Column) -> &Int64Chunked {
     c.datetime().unwrap().physical()
 }
 
+pub fn decimal_chunked(c: &Column) -> &Int128Chunked {
+    c.decimal().unwrap().physical()
+}
+
 
 pub fn bool_vec_to_column(col_name: &str, vec: Vec<bool>) -> Column {
     let mut c = Series::from_iter(vec).into_column();
@@ -117,6 +118,11 @@ pub fn datetime_vec_to_column(col_name: &str, vec: Vec<i64>) -> Column {
     o_series.into_column()
 }
 
+pub fn decimal_vec_to_column(col_name: &str, vec: Vec<i128>) -> Column {
+    let o_series = Int128Chunked::from_vec(PlSmallStr::from_str(col_name), vec).into_decimal(default_decimal_precision(), default_decimal_scale()).unwrap();
+    o_series.into_column()
+}
+
 pub fn unwrap_ordered_float<T>(o: Option<T>) -> OrderedFloat<T> {
     OrderedFloat(o.unwrap())
 }
@@ -510,6 +516,18 @@ binary_nonlang_nonlang_index_impl!(
     datetime_vec_to_column
 );
 
+binary_nonlang_nonlang_index_impl!(
+    U32DecimalIndex,
+    (u32, i128),
+    u32_chunked,
+    decimal_chunked,
+    unwrap_t,
+    noop_t,
+    noop_t,
+    u32_vec_to_column,
+    decimal_vec_to_column
+);
+
 binary_nonlang_lang_index_impl!(
     U32LangIndex,
     (u32, u32, u32),
@@ -533,6 +551,7 @@ pub enum SubjectObjectIndex {
     U32I64Index(U32I64Index),
     U32F32Index(U32F32Index),
     U32F64Index(U32F64Index),
+    U32DecimalIndex(U32DecimalIndex),
     U32DateIndex(U32DateIndex),
     U32DateTimeIndex(U32DateTimeIndex),
 }
@@ -553,9 +572,11 @@ impl SubjectObjectIndex {
                 SubjectObjectIndex::U32BoolIndex(U32BoolIndex::new())
             } else if object_type.is_lit_type(xsd::FLOAT) {
                 SubjectObjectIndex::U32F32Index(U32F32Index::new())
-            } else if object_type.is_lit_type(xsd::DOUBLE) || object_type.is_lit_type(xsd::DECIMAL)
+            } else if object_type.is_lit_type(xsd::DOUBLE)
             {
                 SubjectObjectIndex::U32F64Index(U32F64Index::new())
+            } else if object_type.is_lit_type(xsd::DECIMAL) {
+                SubjectObjectIndex::U32DecimalIndex(U32DecimalIndex::new())
             } else if object_type.is_lit_type(xsd::BYTE) {
                 SubjectObjectIndex::U32I8Index(U32I8Index::new())
             } else if object_type.is_lit_type(xsd::UNSIGNED_BYTE) {
@@ -592,6 +613,7 @@ impl SubjectObjectIndex {
             SubjectObjectIndex::U32I64Index(i) => i.insert(df),
             SubjectObjectIndex::U32F32Index(i) => i.insert(df),
             SubjectObjectIndex::U32F64Index(i) => i.insert(df),
+            SubjectObjectIndex::U32DecimalIndex(i) => i.insert(df),
             SubjectObjectIndex::U32DateIndex(i) => i.insert(df),
             SubjectObjectIndex::U32DateTimeIndex(i) => i.insert(df),
             SubjectObjectIndex::U32BoolIndex(i) => i.insert(df),
@@ -611,6 +633,7 @@ impl SubjectObjectIndex {
             SubjectObjectIndex::U32I64Index(i) => i.delete(df),
             SubjectObjectIndex::U32F32Index(i) => i.delete(df),
             SubjectObjectIndex::U32F64Index(i) => i.delete(df),
+            SubjectObjectIndex::U32DecimalIndex(i) => i.delete(df),
             SubjectObjectIndex::U32DateIndex(i) => i.delete(df),
             SubjectObjectIndex::U32DateTimeIndex(i) => i.delete(df),
             SubjectObjectIndex::U32BoolIndex(i) => i.delete(df),
diff --git a/lib/triplestore/src/triples_write/serializers.rs b/lib/triplestore/src/triples_write/serializers.rs
@@ -7,6 +7,9 @@ use polars_core::utils::arrow::array::{Array, BooleanArray, PrimitiveArray, Utf8
 use polars_core::utils::arrow::types::NativeType;
 use representation::BaseRDFNodeType;
 use std::io::Write;
+use polars::polars_compute;
+use polars_core::utils::arrow;
+use representation::rdf_to_polars::default_decimal_scale;
 
 const TOO_MANY_MSG: &str = "too many items requested from CSV serializer";
 const ARRAY_MISMATCH_MSG: &str = "wrong array type";
@@ -99,6 +102,25 @@ fn float_serializer_no_precision_autoformat<I: NativeType + ryu::Float>(
     })
 }
 
+fn decimal_serializer(array: &PrimitiveArray<i128>, scale: usize) -> impl Serializer<'_> {
+    let mut fmt_buf = polars_compute::decimal::DecimalFmtBuffer::new();
+    let f = move |&item, buf: &mut Vec<u8> | {
+        buf.extend_from_slice(
+            fmt_buf
+                .format_dec128(item, scale, true, false)
+                .as_bytes(),
+        );
+    };
+
+    make_serializer::<_, _,>(f, array.iter(), |array| {
+        array
+            .as_any()
+            .downcast_ref::<PrimitiveArray<i128>>()
+            .expect(ARRAY_MISMATCH_MSG)
+            .iter()
+    })
+}
+
 fn bool_serializer(array: &BooleanArray) -> impl Serializer<'_> {
     let f = move |item, buf: &mut Vec<u8>| {
         let s = if item { "true" } else { "false" };
@@ -397,6 +419,9 @@ pub(super) fn serializer_for<'a>(
             },
             array,
         ),
+        (DataType::Decimal(..), _) => {
+            with_quoted_serializer!(decimal_serializer, default_decimal_scale())
+        }
         _ => {
             panic!("datatype {dtype} cannot be written to triples")
         }
diff --git a/py_maplib/Cargo.toml b/py_maplib/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "py_maplib"
-version = "0.20.7"
+version = "0.20.8"
 edition.workspace = true
 
 [features]
diff --git a/py_maplib/maplib/__init__.pyi b/py_maplib/maplib/__init__.pyi
@@ -776,6 +776,7 @@ class Model:
         dry_run: bool = False,
         max_rows: int = None,
         serial: bool = False,
+        max_iterations: Optional[int] = 100_000,
         debug_rules: bool = False,
     ) -> ValidationReport:
         """
@@ -796,6 +797,7 @@ class Model:
         :param dry_run: Only find targets of shapes, but do not validate them.
         :param max_rows: Maximum estimated rows in underlying SPARQL results, helps avoid out-of-memory errors.
         :param serial: Turns off most parallell validation of shapes.
+        :param max_iterations: Maximum number of iterations for SHACL rules.
         :param debug_rules: Debug why rules returning no results do so. Included in rule log.
         :return: Validation report containing shape performance details and target counts and whether the graph conforms (report.conforms)
         """
diff --git a/py_maplib/src/lib.rs b/py_maplib/src/lib.rs
@@ -489,6 +489,7 @@ impl PyModel {
         dry_run=None,
         max_rows=None,
         serial=None,
+        max_iterations=100_000,
         debug_rules=None,
     ))]
     #[instrument(skip_all)]
@@ -508,6 +509,7 @@ impl PyModel {
         dry_run: Option<bool>,
         max_rows: Option<usize>,
         serial: Option<bool>,
+        max_iterations: Option<usize>,
         debug_rules: Option<bool>,
     ) -> PyResult<PyValidationReport> {
         let res = py.detach(move || {
@@ -527,6 +529,7 @@ impl PyModel {
                 dry_run,
                 max_rows,
                 serial,
+                max_iterations,
                 debug_rules,
             )
         })?;
@@ -1231,6 +1234,7 @@ fn validate_mutex(
     dry_run: Option<bool>,
     max_rows: Option<usize>,
     serial: Option<bool>,
+    max_iterations: Option<usize>,
     debug_rules: Option<bool>,
 ) -> PyResult<PyValidationReport> {
     let data_graph = parse_optional_named_node(data_graph)?;
@@ -1292,6 +1296,7 @@ fn validate_mutex(
             deactivate_shapes,
             dry_run.unwrap_or(false),
             serial.unwrap_or(false),
+            max_iterations,
             debug_rules.unwrap_or(false),
         )
         .map_err(PyMaplibError::from)?;
diff --git a/py_maplib/tests/test_multi_expressions.py b/py_maplib/tests/test_multi_expressions.py
diff --git a/py_maplib/tests/testdata/multi_concat.csv b/py_maplib/tests/testdata/multi_concat.csv
diff --git a/py_maplib/tests/testdata/multi_many_comp.csv b/py_maplib/tests/testdata/multi_many_comp.csv
