From 43cb00bb9a9a0826fedcf93459ebe7e4879359ba Mon Sep 17 00:00:00 2001 From: james_teo Date: Mon, 27 Apr 2026 22:34:28 +0800 Subject: [PATCH 1/4] Update Changelog --- CHANGELOG.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99c0117..f12c35b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Nil +## [0.0.2] - 2026-04-27 + +### 🎉 vowl is now an official ODCS vendor! + +We're thrilled to announce that **vowl** has been recognised as an official [Open Data Contract Standard (ODCS)](https://bitol.io/open-data-contract-standard/) vendor. This is a proud milestone for the project and a testament to the community's commitment to open, interoperable data contracts. + +### Fixed +- Getting outputs no longer crashes when a contract contains a list-valued `mustBe` field (#23). +- Broken link on the Known Issues documentation page (#19). +- Git link sanitisation issue in documentation (#15). + +### Changed (Breaking) +- `contract_definition` is now the single source of truth for check metadata (#21). This replaces several top-level metadata fields with a single `contract_definition` dict that carries the raw ODCS quality entry: + - **Renamed fields:** `schema` → `schema_name`, `rule` → `rendered_implementation`. + - **Moved into `contract_definition`:** `dimension`, `type`, `description`, `severity`, `unit` — these are no longer top-level keys in `CheckResultMetadata`. + - **Output DataFrame:** `get_check_results_df()` now flattens `contract_definition` into top-level columns, so all contract fields appear automatically. The column order in the results CSV has changed accordingly. + - Code that accesses `metadata["schema"]`, `metadata["rule"]`, `metadata["unit"]`, etc. must be updated to use the new key names or read from `metadata["contract_definition"]`. + +### Added +- DuckDB attach example demonstrating cross-database validation (#18). +- Open Graph and SEO metadata for the documentation site (#14). +- Google site verification meta tag (#12). +- Updated Jupyter notebook examples (#16). +- README badge and data contract example link (#13, #22). + +### Dependencies +- Bumped `pytest` from 9.0.2 to 9.0.3 (#20). +- Bumped `cryptography` (#17). +- Updated Python Docker tag to 3.14.3 (#11). + ## [0.0.1] - 2026-04-02 ### 🎉 Celebrating Open Source @@ -34,5 +64,6 @@ Initial public release of **vowl**. - `THIRD_PARTY_NOTICES` and `LICENSE_AUDIT_REPORT.md`. - `CONTRIBUTING.md` with development setup and release workflow. -[Unreleased]: https://github.com/govtech-data-practice/vowl/compare/v0.0.1...HEAD +[Unreleased]: https://github.com/govtech-data-practice/vowl/compare/v0.0.2...HEAD +[0.0.2]: https://github.com/govtech-data-practice/vowl/compare/v0.0.1...v0.0.2 [0.0.1]: https://github.com/govtech-data-practice/vowl/releases/tag/v0.0.1 From eb708148929308d48d8a7b3b055c1c312546d046 Mon Sep 17 00:00:00 2001 From: james_teo Date: Tue, 28 Apr 2026 14:34:33 +0800 Subject: [PATCH 2/4] update changelog + examples Co-authored-by: Copilot --- CHANGELOG.md | 1 + README.md | 4 +- docs/roadmap.md | 2 + .../vowl_demo_HDB_results_check_results.csv | 48 +- examples/vowl_demo_HDB_results_summary.json | 390 +++-- ...demo_multi_table_results_check_results.csv | 175 +- ...vowl_demo_multi_table_results_summary.json | 1413 ++++++++++------ examples/vowl_usage_patterns_demo.ipynb | 1458 +++++++++-------- 8 files changed, 1996 insertions(+), 1495 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f12c35b..66c7aa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We're thrilled to announce that **vowl** has been recognised as an official [Ope - Code that accesses `metadata["schema"]`, `metadata["rule"]`, `metadata["unit"]`, etc. must be updated to use the new key names or read from `metadata["contract_definition"]`. ### Added +- Logical type `options.format` validation — contracts can now specify format constraints (e.g. date formats) on logical types, and vowl will auto-generate format checks. See the [Format Checks](https://govtech-data-practice.github.io/vowl/contracts/#format-checks) docs for details (#25). - DuckDB attach example demonstrating cross-database validation (#18). - Open Graph and SEO metadata for the documentation site (#14). - Google site verification meta tag (#12). diff --git a/README.md b/README.md index 9a259cd..eaa0154 100644 --- a/README.md +++ b/README.md @@ -855,8 +855,8 @@ result.display_full_report() | Capability | Description | Status | |------------|-------------|--------| | � **Alternative Check Engines** | Support for dqx, dbt, Soda, Great Expectations (subject to licensing review) | Planned | -| 📅 **Parallel Check Execution** | Run checks in parallel for faster validation across large contracts | Planned | - +| 📅 **Parallel Check Execution** | Run checks in parallel for faster validation across large contracts | Planned || 📅 **CLI Interface** | Command-line interface for running validations directly from the terminal | Planned | +| 📅 **vowl-ui** | Web-based validation interface for vowl | Planned | --- ## Contributing diff --git a/docs/roadmap.md b/docs/roadmap.md index d842e4b..7cb13b9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -28,3 +28,5 @@ description: >- |------------|-------------|--------| | **Alternative Check Engines** | Support for dqx, dbt, Soda, Great Expectations (subject to licensing review) | Planned | | **Parallel Check Execution** | Run checks in parallel for faster validation across large contracts | Planned | +| **CLI Interface** | Command-line interface for running validations directly from the terminal | Planned | +| **vowl-ui** | Web-based validation interface for vowl | Planned | diff --git a/examples/vowl_demo_HDB_results_check_results.csv b/examples/vowl_demo_HDB_results_check_results.csv index 3861b25..f717f12 100644 --- a/examples/vowl_demo_HDB_results_check_results.csv +++ b/examples/vowl_demo_HDB_results_check_results.csv @@ -1,21 +1,27 @@ -"check_name","target","schema","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rule","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms" -"month_column_exists_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Column 'month' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string",true,5.757582999649458 -"month_logical_type_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['hdb_resale_prices']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string",true,9.819083992624655 -"town_column_exists_check","hdb_resale_prices.town","hdb_resale_prices","sql","sql","conformity","Column 'town' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference",,true,5.337791008059867 -"flat_type_column_exists_check","hdb_resale_prices.flat_type","hdb_resale_prices","sql","sql","conformity","Column 'flat_type' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference",,true,5.779207989689894 -"block_column_exists_check","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","Column 'block' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference",,true,5.379000009270385 -"street_name_column_exists_check","hdb_resale_prices.street_name","hdb_resale_prices","sql","sql","conformity","Column 'street_name' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference",,true,5.304541002260521 -"storey_range_column_exists_check","hdb_resale_prices.storey_range","hdb_resale_prices","sql","sql","conformity","Column 'storey_range' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference",,true,5.266625012154691 -"floor_area_sqm_column_exists_check","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","conformity","Column 'floor_area_sqm' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference",,true,5.168709001736715 -"flat_model_column_exists_check","hdb_resale_prices.flat_model","hdb_resale_prices","sql","sql","conformity","Column 'flat_model' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference",,true,5.213583004660904 -"lease_commence_date_column_exists_check","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Column 'lease_commence_date' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference",,true,4.999249998945743 -"remaining_lease_column_exists_check","hdb_resale_prices.remaining_lease","hdb_resale_prices","sql","sql","conformity","Column 'remaining_lease' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference",,true,5.113916995469481 -"resale_price_column_exists_check","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Column 'resale_price' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference",,true,5.010666005546227 -"hdb_resale_prices_rowCount","hdb_resale_prices","hdb_resale_prices","sql","library","completeness","Total row count for 'hdb_resale_prices'","PASSED",,"mustBeBetween","201879","[0, 30000000]",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices""","['hdb_resale_prices']","$.schema[0].quality[0]","RowCountCheckReference",,true,7.587959000375122 -"Month","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['hdb_resale_prices']","$.schema[0].properties[0].quality[0]","SQLColumnCheckReference","string",false,21.381875005317852 -"town_nullValues","hdb_resale_prices.town","hdb_resale_prices","sql","library","completeness","Count of NULL values in 'town' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE ""town"" IS NULL","['hdb_resale_prices']","$.schema[0].properties[1].quality[0]","NullValuesCheckReference",,true,14.130416995612904 -"flat_type_invalidValues","hdb_resale_prices.flat_type","hdb_resale_prices","sql","library","conformity","Count of invalid values in 'flat_type' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""flat_type"" IS NULL AND NOT TRY_CAST(""flat_type"" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')","['hdb_resale_prices']","$.schema[0].properties[2].quality[0]","InvalidValuesCheckReference",,true,21.572665995336138 -"AddressBlockHouseNumber","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')","['hdb_resale_prices']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference",,false,20.722290995763615 -"floor_area_must_be_less_than_200","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","consistency","Validates that floor area must be less than 200","FAILED","error","mustBe","12","0",12,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200","['hdb_resale_prices']","$.schema[0].properties[6].quality[0]","SQLColumnCheckReference",,false,17.006708003464155 -"Year","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')","['hdb_resale_prices']","$.schema[0].properties[8].quality[0]","SQLColumnCheckReference",,false,20.985333001590334 -"resale_price_must_not_exceed_2m","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Resale price must not be more than 2 million SGD","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000","['hdb_resale_prices']","$.schema[0].properties[10].quality[0]","SQLColumnCheckReference",,false,18.29566700325813 +"check_name","target","schema_name","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms","name","query","mustBe","metric","mustBeBetween","tags","arguments" +"month_column_exists_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Column 'month' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",15.560625004582107,"month_column_exists_check","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"month_logical_type_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['hdb_resale_prices']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",14.568041005986743,"month_logical_type_check","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND CAST(""month"" AS VARCHAR) IS NULL","0",,,, +"town_column_exists_check","hdb_resale_prices.town","hdb_resale_prices","sql","sql","conformity","Column 'town' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference",,"True",7.94391700765118,"town_column_exists_check","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"flat_type_column_exists_check","hdb_resale_prices.flat_type","hdb_resale_prices","sql","sql","conformity","Column 'flat_type' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference",,"True",6.528957994305529,"flat_type_column_exists_check","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"block_column_exists_check","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","Column 'block' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference",,"True",7.354166998993605,"block_column_exists_check","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"street_name_column_exists_check","hdb_resale_prices.street_name","hdb_resale_prices","sql","sql","conformity","Column 'street_name' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference",,"True",7.871333000366576,"street_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"storey_range_column_exists_check","hdb_resale_prices.storey_range","hdb_resale_prices","sql","sql","conformity","Column 'storey_range' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference",,"True",7.073707995004952,"storey_range_column_exists_check","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"floor_area_sqm_column_exists_check","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","conformity","Column 'floor_area_sqm' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference",,"True",7.138625005609356,"floor_area_sqm_column_exists_check","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"flat_model_column_exists_check","hdb_resale_prices.flat_model","hdb_resale_prices","sql","sql","conformity","Column 'flat_model' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference",,"True",7.1229590103030205,"flat_model_column_exists_check","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"lease_commence_date_column_exists_check","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Column 'lease_commence_date' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference",,"True",8.973290998255834,"lease_commence_date_column_exists_check","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"remaining_lease_column_exists_check","hdb_resale_prices.remaining_lease","hdb_resale_prices","sql","sql","conformity","Column 'remaining_lease' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference",,"True",6.421375001082197,"remaining_lease_column_exists_check","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"resale_price_column_exists_check","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Column 'resale_price' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference",,"True",7.602665995364077,"resale_price_column_exists_check","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, +"hdb_resale_prices_rowCount","hdb_resale_prices","hdb_resale_prices","sql","library","completeness","Total row count for 'hdb_resale_prices'","PASSED",,"mustBeBetween","201879","[0, 30000000]",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices""","['hdb_resale_prices']","$.schema[0].quality[0]","RowCountCheckReference",,"True",11.940374999539927,,,,"rowCount","[0, 30000000]",, +"Month","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['hdb_resale_prices']","$.schema[0].properties[0].quality[0]","SQLColumnCheckReference","string","False",67.6931250054622,"Month","SELECT COUNT(*) +FROM ""hdb_resale_prices"" +WHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';","0",,,"['SG-DRM v5.0']", +"town_nullValues","hdb_resale_prices.town","hdb_resale_prices","sql","library","completeness","Count of NULL values in 'town' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE ""town"" IS NULL","['hdb_resale_prices']","$.schema[0].properties[1].quality[0]","NullValuesCheckReference",,"True",18.517291988246143,,,"0","nullValues",,, +"flat_type_invalidValues","hdb_resale_prices.flat_type","hdb_resale_prices","sql","library","conformity","Count of invalid values in 'flat_type' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""flat_type"" IS NULL AND NOT TRY_CAST(""flat_type"" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')","['hdb_resale_prices']","$.schema[0].properties[2].quality[0]","InvalidValuesCheckReference",,"True",30.758624998270534,,,"0","invalidValues",,,"{'validValues': ['1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION']}" +"AddressBlockHouseNumber","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')","['hdb_resale_prices']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference",,"False",28.13729199988302,"AddressBlockHouseNumber","SELECT COUNT(*) +FROM ""hdb_resale_prices"" +WHERE CAST(block AS TEXT) !~ '^[A-Za-z0-9]{1,10}$';","0",,,"['SG-DRM v5.0']", +"floor_area_must_be_less_than_200","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","consistency","Validates that floor area must be less than 200","FAILED","error","mustBe","12","0",12,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200","['hdb_resale_prices']","$.schema[0].properties[6].quality[0]","SQLColumnCheckReference",,"False",23.854999992181547,"floor_area_must_be_less_than_200","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE floor_area_sqm >= 200","0",,,"['User-Defined Quality Rule']", +"Year","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')","['hdb_resale_prices']","$.schema[0].properties[8].quality[0]","SQLColumnCheckReference",,"False",26.971458006300963,"Year","SELECT COUNT(*) +FROM ""hdb_resale_prices"" +WHERE CAST(lease_commence_date AS TEXT) !~ '^[0-9]{4}$';","0",,,"['SG-DRM v5.0']", +"resale_price_must_not_exceed_2m","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Resale price must not be more than 2 million SGD","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000","['hdb_resale_prices']","$.schema[0].properties[10].quality[0]","SQLColumnCheckReference",,"False",20.45279199955985,"resale_price_must_not_exceed_2m","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE resale_price > 2000000","0",,,"['Business Metadata Enrichment']", diff --git a/examples/vowl_demo_HDB_results_summary.json b/examples/vowl_demo_HDB_results_summary.json index db9e98a..2cc1bb0 100644 --- a/examples/vowl_demo_HDB_results_summary.json +++ b/examples/vowl_demo_HDB_results_summary.json @@ -14,7 +14,7 @@ "use_try_cast": true }, "failed_rows": 18, - "total_execution_time_ms": 209.8328730207868, + "total_execution_time_ms": 332.48562400694937, "success_rate": 80.0, "connection_results": { "hdb_resale_prices": { @@ -30,24 +30,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.757582999649458, + "execution_time_ms": 15.560625004582107, "check_path": "$.schema[0].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'month' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'month' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "month_logical_type_check", @@ -56,24 +60,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 9.819083992624655, + "execution_time_ms": 14.568041005986743, "check_path": "$.schema[0].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'month' must be valid string", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'month' must be valid string", + "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"month\" IS NULL AND CAST(\"month\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "hdb_resale_prices.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"month\" IS NULL AND TRY_CAST(\"month\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"month\" IS NULL AND TRY_CAST(\"month\" AS TEXT) IS NULL" }, { "name": "town_column_exists_check", @@ -82,23 +90,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.337791008059867, + "execution_time_ms": 7.94391700765118, "check_path": "$.schema[0].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'town' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "town_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'town' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.town", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "flat_type_column_exists_check", @@ -107,23 +119,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.779207989689894, + "execution_time_ms": 6.528957994305529, "check_path": "$.schema[0].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'flat_type' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "flat_type_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'flat_type' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.flat_type", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "block_column_exists_check", @@ -132,23 +148,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.379000009270385, + "execution_time_ms": 7.354166998993605, "check_path": "$.schema[0].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'block' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "block_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'block' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.block", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "street_name_column_exists_check", @@ -157,23 +177,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.304541002260521, + "execution_time_ms": 7.871333000366576, "check_path": "$.schema[0].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'street_name' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "street_name_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'street_name' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"street_name\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.street_name", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"street_name\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"street_name\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "storey_range_column_exists_check", @@ -182,23 +206,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.266625012154691, + "execution_time_ms": 7.073707995004952, "check_path": "$.schema[0].properties[5].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'storey_range' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "storey_range_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'storey_range' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"storey_range\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.storey_range", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"storey_range\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"storey_range\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "floor_area_sqm_column_exists_check", @@ -207,23 +235,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.168709001736715, + "execution_time_ms": 7.138625005609356, "check_path": "$.schema[0].properties[6].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'floor_area_sqm' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "floor_area_sqm_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'floor_area_sqm' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.floor_area_sqm", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "flat_model_column_exists_check", @@ -232,23 +264,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.213583004660904, + "execution_time_ms": 7.1229590103030205, "check_path": "$.schema[0].properties[7].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'flat_model' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "flat_model_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'flat_model' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.flat_model", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "lease_commence_date_column_exists_check", @@ -257,23 +293,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.999249998945743, + "execution_time_ms": 8.973290998255834, "check_path": "$.schema[0].properties[8].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'lease_commence_date' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "lease_commence_date_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'lease_commence_date' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"lease_commence_date\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.lease_commence_date", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"lease_commence_date\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"lease_commence_date\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "remaining_lease_column_exists_check", @@ -282,23 +322,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.113916995469481, + "execution_time_ms": 6.421375001082197, "check_path": "$.schema[0].properties[9].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'remaining_lease' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "remaining_lease_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'remaining_lease' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"remaining_lease\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.remaining_lease", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"remaining_lease\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"remaining_lease\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "resale_price_column_exists_check", @@ -307,23 +351,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.010666005546227, + "execution_time_ms": 7.602665995364077, "check_path": "$.schema[0].properties[10].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'resale_price' must exist in 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "resale_price_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'resale_price' must exist in 'hdb_resale_prices'", + "query": "SELECT COUNT(*) FROM (SELECT \"resale_price\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "hdb_resale_prices.resale_price", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"resale_price\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"resale_price\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists" }, { "name": "hdb_resale_prices_rowCount", @@ -335,23 +383,29 @@ ], "actual_value": 201879, "failed_rows_count": 0, - "execution_time_ms": 7.587959000375122, + "execution_time_ms": 11.940374999539927, "check_path": "$.schema[0].quality[0]", "check_ref_type": "RowCountCheckReference", - "dimension": "completeness", - "type": "library", - "description": "Total row count for 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBeBetween", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "rowCount", + "mustBeBetween": [ + 0, + 30000000 + ], + "dimension": "completeness", + "description": "Total row count for 'hdb_resale_prices'" + }, "target": "hdb_resale_prices", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\"" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\"" }, { "name": "Month", @@ -360,24 +414,31 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 21.381875005317852, + "execution_time_ms": 67.6931250054622, "check_path": "$.schema[0].properties[0].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "Month", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "hdb_resale_prices.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')" }, { "name": "town_nullValues", @@ -386,23 +447,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.130416995612904, + "execution_time_ms": 18.517291988246143, "check_path": "$.schema[0].properties[1].quality[0]", "check_ref_type": "NullValuesCheckReference", - "dimension": "completeness", - "type": "library", - "description": "Count of NULL values in 'town' of 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness", + "description": "Count of NULL values in 'town' of 'hdb_resale_prices'" + }, "target": "hdb_resale_prices.town", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE \"town\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE \"town\" IS NULL" }, { "name": "flat_type_invalidValues", @@ -411,23 +475,37 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 21.572665995336138, + "execution_time_ms": 30.758624998270534, "check_path": "$.schema[0].properties[2].quality[0]", "check_ref_type": "InvalidValuesCheckReference", - "dimension": "conformity", - "type": "library", - "description": "Count of invalid values in 'flat_type' of 'hdb_resale_prices'", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "invalidValues", + "mustBe": 0, + "dimension": "conformity", + "arguments": { + "validValues": [ + "1 ROOM", + "2 ROOM", + "3 ROOM", + "4 ROOM", + "5 ROOM", + "EXECUTIVE", + "MULTI-GENERATION" + ] + }, + "description": "Count of invalid values in 'flat_type' of 'hdb_resale_prices'" + }, "target": "hdb_resale_prices.flat_type", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"flat_type\" IS NULL AND NOT TRY_CAST(\"flat_type\" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"flat_type\" IS NULL AND NOT TRY_CAST(\"flat_type\" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')" }, { "name": "AddressBlockHouseNumber", @@ -436,23 +514,30 @@ "expected_value": 0, "actual_value": 1, "failed_rows_count": 1, - "execution_time_ms": 20.722290995763615, + "execution_time_ms": 28.13729199988302, "check_path": "$.schema[0].properties[3].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "AddressBlockHouseNumber", + "description": "House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(block AS TEXT) !~ '^[A-Za-z0-9]{1,10}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "hdb_resale_prices.block", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')" }, { "name": "floor_area_must_be_less_than_200", @@ -461,23 +546,31 @@ "expected_value": 0, "actual_value": 12, "failed_rows_count": 12, - "execution_time_ms": 17.006708003464155, + "execution_time_ms": 23.854999992181547, "check_path": "$.schema[0].properties[6].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Validates that floor area must be less than 200", - "severity": "error", - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "floor_area_must_be_less_than_200", + "description": "Validates that floor area must be less than 200", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE floor_area_sqm >= 200", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, "target": "hdb_resale_prices.floor_area_sqm", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200" }, { "name": "Year", @@ -486,23 +579,30 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 20.985333001590334, + "execution_time_ms": 26.971458006300963, "check_path": "$.schema[0].properties[8].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY", - "severity": null, - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "Year", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(lease_commence_date AS TEXT) !~ '^[0-9]{4}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "hdb_resale_prices.lease_commence_date", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')" }, { "name": "resale_price_must_not_exceed_2m", @@ -511,23 +611,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 18.29566700325813, + "execution_time_ms": 20.45279199955985, "check_path": "$.schema[0].properties[10].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Resale price must not be more than 2 million SGD", - "severity": "error", - "schema": "hdb_resale_prices", + "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "resale_price_must_not_exceed_2m", + "description": "Resale price must not be more than 2 million SGD", + "type": "sql", + "dimension": "conformity", + "severity": "error", + "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE resale_price > 2000000", + "mustBe": 0, + "tags": [ + "Business Metadata Enrichment" + ] + }, "target": "hdb_resale_prices.resale_price", "aggregation_type": "count", "tables_in_query": [ "hdb_resale_prices" ], - "rule": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000" + "rendered_implementation": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000" } ], "contract_metadata": { diff --git a/examples/vowl_demo_multi_table_results_check_results.csv b/examples/vowl_demo_multi_table_results_check_results.csv index 457b601..769b4f4 100644 --- a/examples/vowl_demo_multi_table_results_check_results.csv +++ b/examples/vowl_demo_multi_table_results_check_results.csv @@ -1,77 +1,98 @@ -"check_name","target","schema","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rule","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms" -"employee_id_column_exists_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string",true,6.11975000356324 -"employee_id_logical_type_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string",true,8.546666009351611 -"employee_id_logical_type_options_minLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string",true,25.609999996959232 -"employee_id_logical_type_options_maxLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string",true,25.552249993779697 -"employee_id_logical_type_options_pattern_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]{6}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]{6}$')","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,25.541707989759743 -"employee_id_required_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].required","RequiredCheckReference","string",true,5.14758299686946 -"employee_id_unique_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[0].unique","UniqueCheckReference","string",true,6.010749988490716 -"employee_id_primary_key_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[0].primaryKey","PrimaryKeyCheckReference","string",true,4.767832986544818 -"payroll_id_column_exists_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference","string",true,5.433333994005807 -"payroll_id_logical_type_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND TRY_CAST(""payroll_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].logicalType","LogicalTypeCheckReference","string",true,8.019708999199793 -"payroll_id_logical_type_options_minLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have minimum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) < 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string",true,25.0603340100497 -"payroll_id_logical_type_options_maxLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have maximum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) > 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string",true,25.716958989505656 -"payroll_id_logical_type_options_pattern_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""payroll_id"" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,25.72074999625329 -"payroll_id_required_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","completeness","Column 'payroll_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].required","RequiredCheckReference","string",true,5.311874992912635 -"payroll_id_unique_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Column 'payroll_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[1].unique","UniqueCheckReference","string",true,5.644874996505678 -"payroll_id_primary_key_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'payroll_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[1].primaryKey","PrimaryKeyCheckReference","string",true,4.597375009325333 -"month_column_exists_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference","string",true,3.79183299082797 -"month_logical_type_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].logicalType","LogicalTypeCheckReference","string",true,5.933833002927713 -"month_logical_type_options_pattern_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must match pattern '^\d{4}-\d{2}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-\d{2}$')","['demo_employee_payroll']","$.schema[0].properties[2].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,22.392333004972897 -"month_required_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","completeness","Column 'month' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].required","RequiredCheckReference","string",true,3.6305000103311613 -"payroll_start_dt_column_exists_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_start_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference","string",true,3.7929579993942752 -"payroll_start_dt_logical_type_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_start_dt' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND TRY_CAST(""payroll_start_dt"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[3].logicalType","LogicalTypeCheckReference","string",true,6.0861669917358086 -"payroll_end_dt_column_exists_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_end_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference","date",true,3.690333993290551 -"payroll_end_dt_logical_type_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_end_dt' must be valid date","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND TRY_CAST(""payroll_end_dt"" AS DATE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[4].logicalType","LogicalTypeCheckReference","date",true,5.9707079926738515 -"total_amt_column_exists_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Column 'total_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference","integer",true,3.760708001209423 -"total_amt_logical_type_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt' must be valid integer","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (TRY_CAST(""total_amt"" AS DOUBLE) IS NULL OR TRY_CAST(""total_amt"" AS DOUBLE) <> TRY_CAST(""total_amt"" AS BIGINT))","['demo_employee_payroll']","$.schema[0].properties[5].logicalType","LogicalTypeCheckReference","integer",true,6.376332996296696 -"employer_cpf_amt_column_exists_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference","number",true,3.631291998317465 -"employer_cpf_amt_logical_type_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employer_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND TRY_CAST(""employer_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[6].logicalType","LogicalTypeCheckReference","number",true,20.88683401234448 -"total_amt_employee_column_exists_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Column 'total_amt_employee' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference","number",true,3.625041004852392 -"total_amt_employee_logical_type_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt_employee' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND TRY_CAST(""total_amt_employee"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].logicalType","LogicalTypeCheckReference","number",true,5.843083010404371 -"total_amt_employee_required_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","completeness","Column 'total_amt_employee' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].required","RequiredCheckReference","number",true,3.5080829984508455 -"employee_cpf_amt_column_exists_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference","number",true,3.5907499986933544 -"employee_cpf_amt_logical_type_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND TRY_CAST(""employee_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].logicalType","LogicalTypeCheckReference","number",true,8.018790991627611 -"employee_cpf_amt_required_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_cpf_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].required","RequiredCheckReference","number",true,4.540499998256564 -"employee_gross_amt_column_exists_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_gross_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference","number",true,5.041374999564141 -"employee_gross_amt_logical_type_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_gross_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND TRY_CAST(""employee_gross_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].logicalType","LogicalTypeCheckReference","number",true,7.9375420027645305 -"employee_gross_amt_required_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_gross_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].required","RequiredCheckReference","number",true,4.815750013221987 -"phone_number_column_exists_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference","string",true,5.021249991841614 -"phone_number_logical_type_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[10].logicalType","LogicalTypeCheckReference","string",true,8.666124995215796 -"phone_number_logical_type_options_minLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have minimum length of 8","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) < 8","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string",true,25.763958008610643 -"phone_number_logical_type_options_maxLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have maximum length of 11","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) > 11","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string",true,30.281333994935267 -"phone_number_logical_type_options_pattern_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must match pattern '^\+65[0-9]{8}$'","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+65[0-9]{8}$')","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,26.03491699846927 -"total_amt_employee_equals_gross_plus_cpf","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)","['demo_employee_payroll']","$.schema[0].quality[0]","SQLTableCheckReference",,false,15.561499996692874 -"demo_employee_payroll_rowCount","demo_employee_payroll","demo_employee_payroll","sql","library","completeness","Total row count for 'demo_employee_payroll'","PASSED",,"mustBeGreaterThan","3","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll""","['demo_employee_payroll']","$.schema[0].quality[3]","RowCountCheckReference",,true,7.83212500391528 -"employee_id_nullValues","demo_employee_payroll.employee_id","demo_employee_payroll","sql","library","completeness","Count of NULL values in 'employee_id' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].quality[0]","NullValuesCheckReference","string",true,13.46854200528469 -"Month","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[0]","SQLColumnCheckReference","string",false,18.365625001024455 -"month_invalidValues","demo_employee_payroll.month","demo_employee_payroll","sql","library","conformity","Count of invalid values in 'month' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[1]","InvalidValuesCheckReference","string",true,19.918834004784003 -"Date With Timezone","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference","string",false,17.863624991150573 -"Date With Timezone","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[4].quality[0]","SQLColumnCheckReference","date",false,17.999208997935057 -"total_amt_equals_employer_cpf_plus_total_amt_employee","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","consistency","Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","['demo_employee_payroll']","$.schema[0].properties[5].quality[0]","SQLColumnCheckReference","integer",false,14.988583003287204 -"total_amt_missingValues","demo_employee_payroll.total_amt","demo_employee_payroll","sql","library","completeness","Count of missing values in 'total_amt' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt"" IS NULL OR TRY_CAST(""total_amt"" AS TEXT) IN ('')","['demo_employee_payroll']","$.schema[0].properties[5].quality[1]","MissingValuesCheckReference","integer",true,19.144125006278045 -"employee_id_column_exists_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[0].name","DeclaredColumnExistsCheckReference","string",true,5.450958997244015 -"employee_id_logical_type_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[0].logicalType","LogicalTypeCheckReference","string",true,8.749791988520883 -"employee_id_logical_type_options_pattern_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]+$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]+$')","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,25.176167007884942 -"employee_id_logical_type_options_minLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string",true,25.518875001580454 -"employee_id_logical_type_options_maxLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string",true,27.36829199420754 -"employee_id_required_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","['demo_employee_list']","$.schema[1].properties[0].required","RequiredCheckReference","string",true,5.373791995225474 -"employee_id_unique_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[0].unique","UniqueCheckReference","string",true,6.475375004811212 -"employee_id_primary_key_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_list']","$.schema[1].properties[0].primaryKey","PrimaryKeyCheckReference","string",true,5.115499996463768 -"employee_last_name_column_exists_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Column 'employee_last_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[1].name","DeclaredColumnExistsCheckReference","string",true,6.529666992719285 -"employee_last_name_logical_type_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Values in 'employee_last_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND TRY_CAST(""employee_last_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[1].logicalType","LogicalTypeCheckReference","string",true,10.200917007750832 -"employee_last_name_required_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","completeness","Column 'employee_last_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[1].required","RequiredCheckReference","string",true,5.171458004042506 -"employee_first_name_column_exists_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Column 'employee_first_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[2].name","DeclaredColumnExistsCheckReference","string",true,5.425541996373795 -"employee_first_name_logical_type_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Values in 'employee_first_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND TRY_CAST(""employee_first_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[2].logicalType","LogicalTypeCheckReference","string",true,8.74150000163354 -"employee_first_name_required_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","completeness","Column 'employee_first_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[2].required","RequiredCheckReference","string",true,6.4498749998165295 -"current_title_column_exists_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Column 'current_title' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[3].name","DeclaredColumnExistsCheckReference","string",true,7.187124996562488 -"current_title_logical_type_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Values in 'current_title' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND TRY_CAST(""current_title"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[3].logicalType","LogicalTypeCheckReference","string",true,9.6572080074111 -"phone_number_column_exists_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[4].name","DeclaredColumnExistsCheckReference","string",true,5.327791994204745 -"phone_number_logical_type_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[4].logicalType","LogicalTypeCheckReference","string",true,8.912875011446886 -"phone_number_logical_type_options_pattern_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must match pattern '^\+[0-9]+$'","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+[0-9]+$')","['demo_employee_list']","$.schema[1].properties[4].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string",true,44.16029200365301 -"phone_number_unique_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","consistency","Column 'phone_number' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[4].unique","UniqueCheckReference","string",true,4.733582987682894 -"PersonName","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[1].quality[0]","SQLColumnCheckReference","string",false,14.657958003226668 -"PersonName","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[2].quality[0]","SQLColumnCheckReference","string",false,14.439665988902561 -"TelephoneNumber","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Based on ITU T E.164 | e.g. 095236","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')","['demo_employee_list']","$.schema[1].properties[4].quality[0]","SQLColumnCheckReference","string",false,14.721458006533794 -"employee_id_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[1]","SQLTableCheckReference",,false,47.93762500048615 -"phone_number_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Ensures the phone numbers recorded in payroll matches an entry in the master employee list.","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[2]","SQLTableCheckReference",,false,14.986333000706509 +"check_name","target","schema_name","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms","name","query","mustBe","tags","metric","mustBeGreaterThan","arguments" +"employee_id_column_exists_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",8.266415999969468,"employee_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_id_logical_type_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",10.77183299639728,"employee_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND CAST(""employee_id"" AS VARCHAR) IS NULL","0",,,, +"employee_id_logical_type_options_minLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",34.72320800938178,"employee_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) < 7","0",,,, +"employee_id_logical_type_options_maxLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",33.42779100057669,"employee_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) > 7","0",,,, +"employee_id_logical_type_options_pattern_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]{6}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]{6}$')","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",39.862334000645205,"employee_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT CAST(""employee_id"" AS VARCHAR) ~ '^e[0-9]{6}$'","0",,,, +"employee_id_required_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].required","RequiredCheckReference","string","True",7.818917001713999,"employee_id_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","0",,,, +"employee_id_unique_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[0].unique","UniqueCheckReference","string","True",8.558542001992464,"employee_id_unique_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","0",,,, +"employee_id_primary_key_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",6.696457989164628,"employee_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","0",,,, +"payroll_id_column_exists_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference","string","True",6.77716700010933,"payroll_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"payroll_id_logical_type_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND TRY_CAST(""payroll_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].logicalType","LogicalTypeCheckReference","string","True",12.050000004819594,"payroll_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND CAST(""payroll_id"" AS VARCHAR) IS NULL","0",,,, +"payroll_id_logical_type_options_minLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have minimum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) < 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",37.43920900160447,"payroll_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(CAST(""payroll_id"" AS VARCHAR)) < 36","0",,,, +"payroll_id_logical_type_options_maxLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have maximum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) > 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",36.900374994729646,"payroll_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(CAST(""payroll_id"" AS VARCHAR)) > 36","0",,,, +"payroll_id_logical_type_options_pattern_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""payroll_id"" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",30.695709006977268,"payroll_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT CAST(""payroll_id"" AS VARCHAR) ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'","0",,,, +"payroll_id_required_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","completeness","Column 'payroll_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].required","RequiredCheckReference","string","True",6.23133301269263,"payroll_id_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","0",,,, +"payroll_id_unique_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Column 'payroll_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[1].unique","UniqueCheckReference","string","True",6.7342499969527125,"payroll_id_unique_check","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","0",,,, +"payroll_id_primary_key_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'payroll_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[1].primaryKey","PrimaryKeyCheckReference","string","True",5.641625000862405,"payroll_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","0",,,, +"month_column_exists_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference","string","True",6.300708002527244,"month_column_exists_check","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"month_logical_type_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].logicalType","LogicalTypeCheckReference","string","True",10.405374996480532,"month_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND CAST(""month"" AS VARCHAR) IS NULL","0",,,, +"month_logical_type_options_pattern_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must match pattern '^\d{4}-\d{2}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-\d{2}$')","['demo_employee_payroll']","$.schema[0].properties[2].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",25.42725000239443,"month_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT CAST(""month"" AS VARCHAR) ~ '^\d{4}-\d{2}$'","0",,,, +"month_required_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","completeness","Column 'month' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].required","RequiredCheckReference","string","True",4.294291997211985,"month_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","0",,,, +"payroll_start_dt_column_exists_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_start_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference","string","True",4.076666999026202,"payroll_start_dt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"payroll_start_dt_logical_type_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_start_dt' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND TRY_CAST(""payroll_start_dt"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[3].logicalType","LogicalTypeCheckReference","string","True",6.958707992453128,"payroll_start_dt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND CAST(""payroll_start_dt"" AS VARCHAR) IS NULL","0",,,, +"payroll_end_dt_column_exists_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_end_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference","date","True",4.2034159996546805,"payroll_end_dt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"payroll_end_dt_logical_type_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_end_dt' must be valid date","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND TRY_CAST(""payroll_end_dt"" AS DATE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[4].logicalType","LogicalTypeCheckReference","date","True",7.174208003561944,"payroll_end_dt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND CAST(""payroll_end_dt"" AS DATE) IS NULL","0",,,, +"total_amt_column_exists_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Column 'total_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference","integer","True",4.167917009908706,"total_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"total_amt_logical_type_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt' must be valid integer","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (TRY_CAST(""total_amt"" AS DOUBLE) IS NULL OR TRY_CAST(""total_amt"" AS DOUBLE) <> TRY_CAST(""total_amt"" AS BIGINT))","['demo_employee_payroll']","$.schema[0].properties[5].logicalType","LogicalTypeCheckReference","integer","True",7.325166996452026,"total_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (CAST(""total_amt"" AS DOUBLE PRECISION) IS NULL OR CAST(""total_amt"" AS DOUBLE PRECISION) <> CAST(""total_amt"" AS BIGINT))","0",,,, +"employer_cpf_amt_column_exists_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference","number","True",4.097916011232883,"employer_cpf_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employer_cpf_amt_logical_type_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employer_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND TRY_CAST(""employer_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[6].logicalType","LogicalTypeCheckReference","number","True",8.272374994703569,"employer_cpf_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND CAST(""employer_cpf_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, +"total_amt_employee_column_exists_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Column 'total_amt_employee' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference","number","True",5.967875011265278,"total_amt_employee_column_exists_check","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"total_amt_employee_logical_type_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt_employee' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND TRY_CAST(""total_amt_employee"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].logicalType","LogicalTypeCheckReference","number","True",39.624583994736895,"total_amt_employee_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND CAST(""total_amt_employee"" AS DOUBLE PRECISION) IS NULL","0",,,, +"total_amt_employee_required_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","completeness","Column 'total_amt_employee' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].required","RequiredCheckReference","number","True",5.84237500152085,"total_amt_employee_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","0",,,, +"employee_cpf_amt_column_exists_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference","number","True",6.1369589966489,"employee_cpf_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_cpf_amt_logical_type_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND TRY_CAST(""employee_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].logicalType","LogicalTypeCheckReference","number","True",9.304834005888551,"employee_cpf_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND CAST(""employee_cpf_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, +"employee_cpf_amt_required_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_cpf_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].required","RequiredCheckReference","number","True",5.339750001439825,"employee_cpf_amt_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","0",,,, +"employee_gross_amt_column_exists_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_gross_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference","number","True",8.089125010883436,"employee_gross_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_gross_amt_logical_type_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_gross_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND TRY_CAST(""employee_gross_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].logicalType","LogicalTypeCheckReference","number","True",12.091583994333632,"employee_gross_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND CAST(""employee_gross_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, +"employee_gross_amt_required_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_gross_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].required","RequiredCheckReference","number","True",7.5386660028016195,"employee_gross_amt_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","0",,,, +"phone_number_column_exists_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference","string","True",8.80279199918732,"phone_number_column_exists_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, +"phone_number_logical_type_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[10].logicalType","LogicalTypeCheckReference","string","True",14.585208002245054,"phone_number_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND CAST(""phone_number"" AS VARCHAR) IS NULL","0",,,, +"phone_number_logical_type_options_minLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have minimum length of 8","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) < 8","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",32.70300000440329,"phone_number_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(CAST(""phone_number"" AS VARCHAR)) < 8","0",,,, +"phone_number_logical_type_options_maxLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have maximum length of 11","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) > 11","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",34.6164999937173,"phone_number_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(CAST(""phone_number"" AS VARCHAR)) > 11","0",,,, +"phone_number_logical_type_options_pattern_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must match pattern '^\+65[0-9]{8}$'","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+65[0-9]{8}$')","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",38.351875002263114,"phone_number_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT CAST(""phone_number"" AS VARCHAR) ~ '^\+65[0-9]{8}$'","0",,,, +"total_amt_employee_equals_gross_plus_cpf","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)","['demo_employee_payroll']","$.schema[0].quality[0]","SQLTableCheckReference",,"False",23.633499993593432,"total_amt_employee_equals_gross_plus_cpf","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee != (employee_gross_amt + employee_cpf_amt)","0","['User-Defined Quality Rule']",,, +"demo_employee_payroll_rowCount","demo_employee_payroll","demo_employee_payroll","sql","library","completeness","Total row count for 'demo_employee_payroll'","PASSED",,"mustBeGreaterThan","3","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll""","['demo_employee_payroll']","$.schema[0].quality[3]","RowCountCheckReference",,"True",12.046417003148235,,,,,"rowCount","0", +"employee_id_nullValues","demo_employee_payroll.employee_id","demo_employee_payroll","sql","library","completeness","Count of NULL values in 'employee_id' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].quality[0]","NullValuesCheckReference","string","True",17.949457993381657,,,"0",,"nullValues",, +"Month","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[0]","SQLColumnCheckReference","string","False",23.35858299920801,"Month","SELECT COUNT(*) +FROM demo_employee_payroll +WHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';","0","['SG-DRM v5.0']",,, +"month_invalidValues","demo_employee_payroll.month","demo_employee_payroll","sql","library","conformity","Count of invalid values in 'month' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[1]","InvalidValuesCheckReference","string","True",25.47229200717993,,,"0",,"invalidValues",,"{'pattern': '^\\d{4}-(0[1-9]|1[0-2])$'}" +"Date With Timezone","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference","string","False",23.1642079888843,"Date With Timezone","SELECT COUNT(*) +FROM demo_employee_payroll +WHERE CAST(payroll_start_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$' + OR LENGTH(CAST(payroll_start_dt AS TEXT)) > 22;","0","['SG-DRM v5.0']",,, +"Date With Timezone","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[4].quality[0]","SQLColumnCheckReference","date","False",22.935790999326855,"Date With Timezone","SELECT COUNT(*) +FROM demo_employee_payroll +WHERE CAST(payroll_end_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$' + OR LENGTH(CAST(payroll_end_dt AS TEXT)) > 22;","0","['SG-DRM v5.0']",,, +"total_amt_equals_employer_cpf_plus_total_amt_employee","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","consistency","Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","['demo_employee_payroll']","$.schema[0].properties[5].quality[0]","SQLColumnCheckReference","integer","False",17.85162500164006,"total_amt_equals_employer_cpf_plus_total_amt_employee","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt != (employer_cpf_amt + total_amt_employee) OR (total_amt IS NOT NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","0","['User-Defined Quality Rule']",,, +"total_amt_missingValues","demo_employee_payroll.total_amt","demo_employee_payroll","sql","library","completeness","Count of missing values in 'total_amt' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt"" IS NULL OR TRY_CAST(""total_amt"" AS TEXT) IN ('')","['demo_employee_payroll']","$.schema[0].properties[5].quality[1]","MissingValuesCheckReference","integer","True",22.719083004631102,,,"0",,"missingValues",,"{'missingValues': [None, '']}" +"employee_id_column_exists_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[0].name","DeclaredColumnExistsCheckReference","string","True",5.673040999681689,"employee_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_id_logical_type_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[0].logicalType","LogicalTypeCheckReference","string","True",8.802083000773564,"employee_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND CAST(""employee_id"" AS VARCHAR) IS NULL","0",,,, +"employee_id_logical_type_options_pattern_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]+$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]+$')","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",27.994290998321958,"employee_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT CAST(""employee_id"" AS VARCHAR) ~ '^e[0-9]+$'","0",,,, +"employee_id_logical_type_options_minLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",27.982749990769662,"employee_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) < 7","0",,,, +"employee_id_logical_type_options_maxLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",28.017333999741822,"employee_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) > 7","0",,,, +"employee_id_required_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","['demo_employee_list']","$.schema[1].properties[0].required","RequiredCheckReference","string","True",5.478375009261072,"employee_id_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","0",,,, +"employee_id_unique_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[0].unique","UniqueCheckReference","string","True",7.041749995551072,"employee_id_unique_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","0",,,, +"employee_id_primary_key_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_list']","$.schema[1].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",5.410500001744367,"employee_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","0",,,, +"employee_last_name_column_exists_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Column 'employee_last_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[1].name","DeclaredColumnExistsCheckReference","string","True",6.55170799291227,"employee_last_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_last_name_logical_type_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Values in 'employee_last_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND TRY_CAST(""employee_last_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[1].logicalType","LogicalTypeCheckReference","string","True",10.189666994847357,"employee_last_name_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND CAST(""employee_last_name"" AS VARCHAR) IS NULL","0",,,, +"employee_last_name_required_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","completeness","Column 'employee_last_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[1].required","RequiredCheckReference","string","True",6.203083001310006,"employee_last_name_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","0",,,, +"employee_first_name_column_exists_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Column 'employee_first_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[2].name","DeclaredColumnExistsCheckReference","string","True",6.034209000063129,"employee_first_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, +"employee_first_name_logical_type_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Values in 'employee_first_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND TRY_CAST(""employee_first_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[2].logicalType","LogicalTypeCheckReference","string","True",10.555083004874177,"employee_first_name_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND CAST(""employee_first_name"" AS VARCHAR) IS NULL","0",,,, +"employee_first_name_required_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","completeness","Column 'employee_first_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[2].required","RequiredCheckReference","string","True",6.4019589917734265,"employee_first_name_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","0",,,, +"current_title_column_exists_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Column 'current_title' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[3].name","DeclaredColumnExistsCheckReference","string","True",6.396415992639959,"current_title_column_exists_check","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, +"current_title_logical_type_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Values in 'current_title' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND TRY_CAST(""current_title"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[3].logicalType","LogicalTypeCheckReference","string","True",10.182790996623226,"current_title_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND CAST(""current_title"" AS VARCHAR) IS NULL","0",,,, +"phone_number_column_exists_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[4].name","DeclaredColumnExistsCheckReference","string","True",6.789667007979006,"phone_number_column_exists_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, +"phone_number_logical_type_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[4].logicalType","LogicalTypeCheckReference","string","True",10.848208010429516,"phone_number_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND CAST(""phone_number"" AS VARCHAR) IS NULL","0",,,, +"phone_number_logical_type_options_pattern_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must match pattern '^\+[0-9]+$'","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+[0-9]+$')","['demo_employee_list']","$.schema[1].properties[4].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",32.44387501035817,"phone_number_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT CAST(""phone_number"" AS VARCHAR) ~ '^\+[0-9]+$'","0",,,, +"phone_number_unique_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","consistency","Column 'phone_number' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[4].unique","UniqueCheckReference","string","True",6.953083007829264,"phone_number_unique_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","0",,,, +"PersonName","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[1].quality[0]","SQLColumnCheckReference","string","False",22.425041999667883,"PersonName","SELECT COUNT(*) +FROM demo_employee_list +WHERE employee_last_name IS NOT NULL + AND LENGTH(CAST(employee_last_name AS TEXT)) > 66;","0","['SG-DRM v5.0']",,, +"PersonName","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[2].quality[0]","SQLColumnCheckReference","string","False",18.33512500161305,"PersonName","SELECT COUNT(*) +FROM demo_employee_list +WHERE employee_first_name IS NOT NULL + AND LENGTH(CAST(employee_first_name AS TEXT)) > 66;","0","['SG-DRM v5.0']",,, +"TelephoneNumber","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Based on ITU T E.164 | e.g. 095236","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')","['demo_employee_list']","$.schema[1].properties[4].quality[0]","SQLColumnCheckReference","string","False",18.17870800732635,"TelephoneNumber","SELECT COUNT(*) +FROM demo_employee_list +WHERE CAST(phone_number AS TEXT) !~ '^[0-9]{1,12}$';","0","['SG-DRM v5.0']",,, +"employee_id_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[1]","SQLTableCheckReference",,"False",62.502582994056866,"employee_id_exists_in_master_list","SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref + ON payroll.employee_id = ref.employee_id +WHERE ref.employee_id IS NULL","0","['Cross-Table Validation']",,, +"phone_number_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Ensures the phone numbers recorded in payroll matches an entry in the master employee list.","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[2]","SQLTableCheckReference",,"False",18.972791003761813,"phone_number_exists_in_master_list","SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref + ON payroll.phone_number = ref.phone_number +WHERE payroll.phone_number IS NOT NULL + AND ref.phone_number IS NULL","0","['Cross-Table Validation']",,, diff --git a/examples/vowl_demo_multi_table_results_summary.json b/examples/vowl_demo_multi_table_results_summary.json index d553b49..055c097 100644 --- a/examples/vowl_demo_multi_table_results_summary.json +++ b/examples/vowl_demo_multi_table_results_summary.json @@ -15,7 +15,7 @@ "use_try_cast": true }, "failed_rows": 14, - "total_execution_time_ms": 929.0458769537508, + "total_execution_time_ms": 1179.7893720504362, "success_rate": 92.10526315789474, "connection_results": { "demo_employee_payroll": { @@ -35,24 +35,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.11975000356324, + "execution_time_ms": 8.266415999969468, "check_path": "$.schema[0].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_id_logical_type_check", @@ -61,24 +65,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.546666009351611, + "execution_time_ms": 10.77183299639728, "check_path": "$.schema[0].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_id' must be valid string", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_id' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND CAST(\"employee_id\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND TRY_CAST(\"employee_id\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND TRY_CAST(\"employee_id\" AS TEXT) IS NULL" }, { "name": "employee_id_logical_type_options_minLength_check", @@ -87,24 +95,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.609999996959232, + "execution_time_ms": 34.72320800938178, "check_path": "$.schema[0].properties[0].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must have minimum length of 7", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_minLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must have minimum length of 7", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) < 7", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) < 7" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) < 7" }, { "name": "employee_id_logical_type_options_maxLength_check", @@ -113,24 +125,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.552249993779697, + "execution_time_ms": 33.42779100057669, "check_path": "$.schema[0].properties[0].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must have maximum length of 7", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_maxLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must have maximum length of 7", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) > 7", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) > 7" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) > 7" }, { "name": "employee_id_logical_type_options_pattern_check", @@ -139,24 +155,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.541707989759743, + "execution_time_ms": 39.862334000645205, "check_path": "$.schema[0].properties[0].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must match pattern '^e[0-9]{6}$'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must match pattern '^e[0-9]{6}$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND NOT CAST(\"employee_id\" AS VARCHAR) ~ '^e[0-9]{6}$'", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"employee_id\" AS TEXT), '^e[0-9]{6}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"employee_id\" AS TEXT), '^e[0-9]{6}$')" }, { "name": "employee_id_required_check", @@ -165,24 +185,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.14758299686946, + "execution_time_ms": 7.818917001713999, "check_path": "$.schema[0].properties[0].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_id' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_id' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL" }, { "name": "employee_id_unique_check", @@ -191,24 +215,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.010749988490716, + "execution_time_ms": 8.558542001992464, "check_path": "$.schema[0].properties[0].unique", "check_ref_type": "UniqueCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Column 'employee_id' must contain unique values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_unique_check", + "type": "sql", + "dimension": "consistency", + "description": "Column 'employee_id' must contain unique values", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)" }, { "name": "employee_id_primary_key_check", @@ -217,24 +245,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.767832986544818, + "execution_time_ms": 6.696457989164628, "check_path": "$.schema[0].properties[0].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Primary key column 'employee_id' must be unique and not null", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_primary_key_check", + "type": "sql", + "dimension": "consistency", + "description": "Primary key column 'employee_id' must be unique and not null", + "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "custom", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))" + "rendered_implementation": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))" }, { "name": "payroll_id_column_exists_check", @@ -243,24 +275,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.433333994005807, + "execution_time_ms": 6.77716700010933, "check_path": "$.schema[0].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_id' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_id' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "payroll_id_logical_type_check", @@ -269,24 +305,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.019708999199793, + "execution_time_ms": 12.050000004819594, "check_path": "$.schema[0].properties[1].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'payroll_id' must be valid string", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'payroll_id' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND CAST(\"payroll_id\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND TRY_CAST(\"payroll_id\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND TRY_CAST(\"payroll_id\" AS TEXT) IS NULL" }, { "name": "payroll_id_logical_type_options_minLength_check", @@ -295,24 +335,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.0603340100497, + "execution_time_ms": 37.43920900160447, "check_path": "$.schema[0].properties[1].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_id' must have minimum length of 36", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_logical_type_options_minLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_id' must have minimum length of 36", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(CAST(\"payroll_id\" AS VARCHAR)) < 36", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(TRY_CAST(\"payroll_id\" AS TEXT)) < 36" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(TRY_CAST(\"payroll_id\" AS TEXT)) < 36" }, { "name": "payroll_id_logical_type_options_maxLength_check", @@ -321,24 +365,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.716958989505656, + "execution_time_ms": 36.900374994729646, "check_path": "$.schema[0].properties[1].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_id' must have maximum length of 36", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_logical_type_options_maxLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_id' must have maximum length of 36", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(CAST(\"payroll_id\" AS VARCHAR)) > 36", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(TRY_CAST(\"payroll_id\" AS TEXT)) > 36" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(TRY_CAST(\"payroll_id\" AS TEXT)) > 36" }, { "name": "payroll_id_logical_type_options_pattern_check", @@ -347,24 +395,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.72074999625329, + "execution_time_ms": 30.695709006977268, "check_path": "$.schema[0].properties[1].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND NOT CAST(\"payroll_id\" AS VARCHAR) ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"payroll_id\" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"payroll_id\" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')" }, { "name": "payroll_id_required_check", @@ -373,24 +425,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.311874992912635, + "execution_time_ms": 6.23133301269263, "check_path": "$.schema[0].properties[1].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'payroll_id' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'payroll_id' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL" }, { "name": "payroll_id_unique_check", @@ -399,24 +455,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.644874996505678, + "execution_time_ms": 6.7342499969527125, "check_path": "$.schema[0].properties[1].unique", "check_ref_type": "UniqueCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Column 'payroll_id' must contain unique values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_unique_check", + "type": "sql", + "dimension": "consistency", + "description": "Column 'payroll_id' must contain unique values", + "query": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)" }, { "name": "payroll_id_primary_key_check", @@ -425,24 +485,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.597375009325333, + "execution_time_ms": 5.641625000862405, "check_path": "$.schema[0].properties[1].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Primary key column 'payroll_id' must be unique and not null", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_id_primary_key_check", + "type": "sql", + "dimension": "consistency", + "description": "Primary key column 'payroll_id' must be unique and not null", + "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)))", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", "aggregation_type": "custom", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)))" + "rendered_implementation": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)))" }, { "name": "month_column_exists_check", @@ -451,24 +515,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.79183299082797, + "execution_time_ms": 6.300708002527244, "check_path": "$.schema[0].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'month' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'month' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "month_logical_type_check", @@ -477,24 +545,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.933833002927713, + "execution_time_ms": 10.405374996480532, "check_path": "$.schema[0].properties[2].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'month' must be valid string", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'month' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND CAST(\"month\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND TRY_CAST(\"month\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND TRY_CAST(\"month\" AS TEXT) IS NULL" }, { "name": "month_logical_type_options_pattern_check", @@ -503,24 +575,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 22.392333004972897, + "execution_time_ms": 25.42725000239443, "check_path": "$.schema[0].properties[2].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'month' must match pattern '^\\d{4}-\\d{2}$'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'month' must match pattern '^\\d{4}-\\d{2}$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT CAST(\"month\" AS VARCHAR) ~ '^\\d{4}-\\d{2}$'", + "mustBe": 0 + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"month\" AS TEXT), '^\\d{4}-\\d{2}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"month\" AS TEXT), '^\\d{4}-\\d{2}$')" }, { "name": "month_required_check", @@ -529,24 +605,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.6305000103311613, + "execution_time_ms": 4.294291997211985, "check_path": "$.schema[0].properties[2].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'month' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "month_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'month' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"month\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"month\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"month\" IS NULL" }, { "name": "payroll_start_dt_column_exists_check", @@ -555,24 +635,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.7929579993942752, + "execution_time_ms": 4.076666999026202, "check_path": "$.schema[0].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_start_dt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_start_dt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_start_dt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"payroll_start_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_start_dt", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"payroll_start_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"payroll_start_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "payroll_start_dt_logical_type_check", @@ -581,24 +665,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.0861669917358086, + "execution_time_ms": 6.958707992453128, "check_path": "$.schema[0].properties[3].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'payroll_start_dt' must be valid string", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_start_dt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'payroll_start_dt' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_start_dt\" IS NULL AND CAST(\"payroll_start_dt\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_start_dt", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_start_dt\" IS NULL AND TRY_CAST(\"payroll_start_dt\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_start_dt\" IS NULL AND TRY_CAST(\"payroll_start_dt\" AS TEXT) IS NULL" }, { "name": "payroll_end_dt_column_exists_check", @@ -607,24 +695,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.690333993290551, + "execution_time_ms": 4.2034159996546805, "check_path": "$.schema[0].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'payroll_end_dt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_end_dt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'payroll_end_dt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"payroll_end_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_end_dt", "logical_type": "date", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"payroll_end_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"payroll_end_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "payroll_end_dt_logical_type_check", @@ -633,24 +725,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.9707079926738515, + "execution_time_ms": 7.174208003561944, "check_path": "$.schema[0].properties[4].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'payroll_end_dt' must be valid date", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "payroll_end_dt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'payroll_end_dt' must be valid date", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_end_dt\" IS NULL AND CAST(\"payroll_end_dt\" AS DATE) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.payroll_end_dt", "logical_type": "date", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_end_dt\" IS NULL AND TRY_CAST(\"payroll_end_dt\" AS DATE) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_end_dt\" IS NULL AND TRY_CAST(\"payroll_end_dt\" AS DATE) IS NULL" }, { "name": "total_amt_column_exists_check", @@ -659,24 +755,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.760708001209423, + "execution_time_ms": 4.167917009908706, "check_path": "$.schema[0].properties[5].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'total_amt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "total_amt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'total_amt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"total_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"total_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"total_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "total_amt_logical_type_check", @@ -685,24 +785,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.376332996296696, + "execution_time_ms": 7.325166996452026, "check_path": "$.schema[0].properties[5].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'total_amt' must be valid integer", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "total_amt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'total_amt' must be valid integer", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt\" IS NULL AND (CAST(\"total_amt\" AS DOUBLE PRECISION) IS NULL OR CAST(\"total_amt\" AS DOUBLE PRECISION) <> CAST(\"total_amt\" AS BIGINT))", + "mustBe": 0 + }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt\" IS NULL AND (TRY_CAST(\"total_amt\" AS DOUBLE) IS NULL OR TRY_CAST(\"total_amt\" AS DOUBLE) <> TRY_CAST(\"total_amt\" AS BIGINT))" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt\" IS NULL AND (TRY_CAST(\"total_amt\" AS DOUBLE) IS NULL OR TRY_CAST(\"total_amt\" AS DOUBLE) <> TRY_CAST(\"total_amt\" AS BIGINT))" }, { "name": "employer_cpf_amt_column_exists_check", @@ -711,24 +815,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.631291998317465, + "execution_time_ms": 4.097916011232883, "check_path": "$.schema[0].properties[6].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employer_cpf_amt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"employer_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.employer_cpf_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employer_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employer_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employer_cpf_amt_logical_type_check", @@ -737,24 +845,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 20.88683401234448, + "execution_time_ms": 8.272374994703569, "check_path": "$.schema[0].properties[6].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employer_cpf_amt' must be valid number", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employer_cpf_amt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employer_cpf_amt' must be valid number", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employer_cpf_amt\" IS NULL AND CAST(\"employer_cpf_amt\" AS DOUBLE PRECISION) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employer_cpf_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employer_cpf_amt\" IS NULL AND TRY_CAST(\"employer_cpf_amt\" AS DOUBLE) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employer_cpf_amt\" IS NULL AND TRY_CAST(\"employer_cpf_amt\" AS DOUBLE) IS NULL" }, { "name": "total_amt_employee_column_exists_check", @@ -763,24 +875,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.625041004852392, + "execution_time_ms": 5.967875011265278, "check_path": "$.schema[0].properties[7].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'total_amt_employee' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "total_amt_employee_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'total_amt_employee' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"total_amt_employee\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"total_amt_employee\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"total_amt_employee\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "total_amt_employee_logical_type_check", @@ -789,24 +905,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.843083010404371, + "execution_time_ms": 39.624583994736895, "check_path": "$.schema[0].properties[7].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'total_amt_employee' must be valid number", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "total_amt_employee_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'total_amt_employee' must be valid number", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt_employee\" IS NULL AND CAST(\"total_amt_employee\" AS DOUBLE PRECISION) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt_employee\" IS NULL AND TRY_CAST(\"total_amt_employee\" AS DOUBLE) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt_employee\" IS NULL AND TRY_CAST(\"total_amt_employee\" AS DOUBLE) IS NULL" }, { "name": "total_amt_employee_required_check", @@ -815,24 +935,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.5080829984508455, + "execution_time_ms": 5.84237500152085, "check_path": "$.schema[0].properties[7].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'total_amt_employee' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "total_amt_employee_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'total_amt_employee' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt_employee\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt_employee\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt_employee\" IS NULL" }, { "name": "employee_cpf_amt_column_exists_check", @@ -841,24 +965,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 3.5907499986933544, + "execution_time_ms": 6.1369589966489, "check_path": "$.schema[0].properties[8].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_cpf_amt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_cpf_amt_logical_type_check", @@ -867,24 +995,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.018790991627611, + "execution_time_ms": 9.304834005888551, "check_path": "$.schema[0].properties[8].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_cpf_amt' must be valid number", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_cpf_amt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_cpf_amt' must be valid number", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_cpf_amt\" IS NULL AND CAST(\"employee_cpf_amt\" AS DOUBLE PRECISION) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_cpf_amt\" IS NULL AND TRY_CAST(\"employee_cpf_amt\" AS DOUBLE) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_cpf_amt\" IS NULL AND TRY_CAST(\"employee_cpf_amt\" AS DOUBLE) IS NULL" }, { "name": "employee_cpf_amt_required_check", @@ -893,24 +1025,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.540499998256564, + "execution_time_ms": 5.339750001439825, "check_path": "$.schema[0].properties[8].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_cpf_amt' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_cpf_amt_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_cpf_amt' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_cpf_amt\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_cpf_amt\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_cpf_amt\" IS NULL" }, { "name": "employee_gross_amt_column_exists_check", @@ -919,24 +1055,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.041374999564141, + "execution_time_ms": 8.089125010883436, "check_path": "$.schema[0].properties[9].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_gross_amt' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_gross_amt_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_gross_amt' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_gross_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_gross_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_gross_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_gross_amt_logical_type_check", @@ -945,24 +1085,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.9375420027645305, + "execution_time_ms": 12.091583994333632, "check_path": "$.schema[0].properties[9].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_gross_amt' must be valid number", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_gross_amt_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_gross_amt' must be valid number", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_gross_amt\" IS NULL AND CAST(\"employee_gross_amt\" AS DOUBLE PRECISION) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_gross_amt\" IS NULL AND TRY_CAST(\"employee_gross_amt\" AS DOUBLE) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_gross_amt\" IS NULL AND TRY_CAST(\"employee_gross_amt\" AS DOUBLE) IS NULL" }, { "name": "employee_gross_amt_required_check", @@ -971,24 +1115,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.815750013221987, + "execution_time_ms": 7.5386660028016195, "check_path": "$.schema[0].properties[9].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_gross_amt' must not contain NULL values", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_gross_amt_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_gross_amt' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_gross_amt\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_gross_amt\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_gross_amt\" IS NULL" }, { "name": "phone_number_column_exists_check", @@ -997,24 +1145,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.021249991841614, + "execution_time_ms": 8.80279199918732, "check_path": "$.schema[0].properties[10].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must exist in 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must exist in 'demo_employee_payroll'", + "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists" }, { "name": "phone_number_logical_type_check", @@ -1023,24 +1175,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.666124995215796, + "execution_time_ms": 14.585208002245054, "check_path": "$.schema[0].properties[10].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'phone_number' must be valid string", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'phone_number' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND CAST(\"phone_number\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND TRY_CAST(\"phone_number\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND TRY_CAST(\"phone_number\" AS TEXT) IS NULL" }, { "name": "phone_number_logical_type_options_minLength_check", @@ -1049,24 +1205,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.763958008610643, + "execution_time_ms": 32.70300000440329, "check_path": "$.schema[0].properties[10].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must have minimum length of 8", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_options_minLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must have minimum length of 8", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(CAST(\"phone_number\" AS VARCHAR)) < 8", + "mustBe": 0 + }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(TRY_CAST(\"phone_number\" AS TEXT)) < 8" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(TRY_CAST(\"phone_number\" AS TEXT)) < 8" }, { "name": "phone_number_logical_type_options_maxLength_check", @@ -1075,24 +1235,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 30.281333994935267, + "execution_time_ms": 34.6164999937173, "check_path": "$.schema[0].properties[10].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must have maximum length of 11", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_options_maxLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must have maximum length of 11", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(CAST(\"phone_number\" AS VARCHAR)) > 11", + "mustBe": 0 + }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(TRY_CAST(\"phone_number\" AS TEXT)) > 11" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(TRY_CAST(\"phone_number\" AS TEXT)) > 11" }, { "name": "phone_number_logical_type_options_pattern_check", @@ -1101,24 +1265,28 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 26.03491699846927, + "execution_time_ms": 38.351875002263114, "check_path": "$.schema[0].properties[10].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must match pattern '^\\+65[0-9]{8}$'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must match pattern '^\\+65[0-9]{8}$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND NOT CAST(\"phone_number\" AS VARCHAR) ~ '^\\+65[0-9]{8}$'", + "mustBe": 0 + }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"phone_number\" AS TEXT), '^\\+65[0-9]{8}$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"phone_number\" AS TEXT), '^\\+65[0-9]{8}$')" }, { "name": "total_amt_employee_equals_gross_plus_cpf", @@ -1127,23 +1295,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 15.561499996692874, + "execution_time_ms": 23.633499993593432, "check_path": "$.schema[0].quality[0]", "check_ref_type": "SQLTableCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency", - "severity": "error", - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "total_amt_employee_equals_gross_plus_cpf", + "description": "Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee != (employee_gross_amt + employee_cpf_amt)", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, "target": "demo_employee_payroll", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)" }, { "name": "demo_employee_payroll_rowCount", @@ -1152,23 +1328,26 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 0, - "execution_time_ms": 7.83212500391528, + "execution_time_ms": 12.046417003148235, "check_path": "$.schema[0].quality[3]", "check_ref_type": "RowCountCheckReference", - "dimension": "completeness", - "type": "library", - "description": "Total row count for 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBeGreaterThan", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "rowCount", + "mustBeGreaterThan": 0, + "dimension": "completeness", + "description": "Total row count for 'demo_employee_payroll'" + }, "target": "demo_employee_payroll", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\"" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\"" }, { "name": "employee_id_nullValues", @@ -1177,24 +1356,27 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 13.46854200528469, + "execution_time_ms": 17.949457993381657, "check_path": "$.schema[0].properties[0].quality[0]", "check_ref_type": "NullValuesCheckReference", - "dimension": "completeness", - "type": "library", - "description": "Count of NULL values in 'employee_id' of 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness", + "description": "Count of NULL values in 'employee_id' of 'demo_employee_payroll'" + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL" }, { "name": "Month", @@ -1203,24 +1385,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 18.365625001024455, + "execution_time_ms": 23.35858299920801, "check_path": "$.schema[0].properties[2].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "Month", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')" }, { "name": "month_invalidValues", @@ -1229,24 +1418,30 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 19.918834004784003, + "execution_time_ms": 25.47229200717993, "check_path": "$.schema[0].properties[2].quality[1]", "check_ref_type": "InvalidValuesCheckReference", - "dimension": "conformity", - "type": "library", - "description": "Count of invalid values in 'month' of 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "invalidValues", + "mustBe": 0, + "dimension": "conformity", + "arguments": { + "pattern": "^\\d{4}-(0[1-9]|1[0-2])$" + }, + "description": "Count of invalid values in 'month' of 'demo_employee_payroll'" + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"month\" AS TEXT), '^\\d{4}-(0[1-9]|1[0-2])$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"month\" AS TEXT), '^\\d{4}-(0[1-9]|1[0-2])$')" }, { "name": "Date With Timezone", @@ -1255,24 +1450,31 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 17.863624991150573, + "execution_time_ms": 23.1642079888843, "check_path": "$.schema[0].properties[3].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "Date With Timezone", + "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(payroll_start_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$'\n OR LENGTH(CAST(payroll_start_dt AS TEXT)) > 22;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_payroll.payroll_start_dt", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22" }, { "name": "Date With Timezone", @@ -1281,24 +1483,31 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 17.999208997935057, + "execution_time_ms": 22.935790999326855, "check_path": "$.schema[0].properties[4].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "Date With Timezone", + "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(payroll_end_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$'\n OR LENGTH(CAST(payroll_end_dt AS TEXT)) > 22;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_payroll.payroll_end_dt", "logical_type": "date", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22" }, { "name": "total_amt_equals_employer_cpf_plus_total_amt_employee", @@ -1307,24 +1516,32 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.988583003287204, + "execution_time_ms": 17.85162500164006, "check_path": "$.schema[0].properties[5].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency", - "severity": "error", - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "total_amt_equals_employer_cpf_plus_total_amt_employee", + "description": "Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt != (employer_cpf_amt + total_amt_employee) OR (total_amt IS NOT NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))" }, { "name": "total_amt_missingValues", @@ -1333,24 +1550,33 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 19.144125006278045, + "execution_time_ms": 22.719083004631102, "check_path": "$.schema[0].properties[5].quality[1]", "check_ref_type": "MissingValuesCheckReference", - "dimension": "completeness", - "type": "library", - "description": "Count of missing values in 'total_amt' of 'demo_employee_payroll'", - "severity": null, - "schema": "demo_employee_payroll", - "operator": "mustBe", - "is_generated": true, - "engine": "sql", + "schema_name": "demo_employee_payroll", + "operator": "mustBe", + "is_generated": true, + "engine": "sql", + "contract_definition": { + "type": "library", + "metric": "missingValues", + "mustBe": 0, + "dimension": "completeness", + "arguments": { + "missingValues": [ + null, + "" + ] + }, + "description": "Count of missing values in 'total_amt' of 'demo_employee_payroll'" + }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", "aggregation_type": "count", "tables_in_query": [ "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt\" IS NULL OR TRY_CAST(\"total_amt\" AS TEXT) IN ('')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt\" IS NULL OR TRY_CAST(\"total_amt\" AS TEXT) IN ('')" }, { "name": "employee_id_column_exists_check", @@ -1359,24 +1585,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.450958997244015, + "execution_time_ms": 5.673040999681689, "check_path": "$.schema[1].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must exist in 'demo_employee_list'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must exist in 'demo_employee_list'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_id_logical_type_check", @@ -1385,24 +1615,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.749791988520883, + "execution_time_ms": 8.802083000773564, "check_path": "$.schema[1].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_id' must be valid string", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_id' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND CAST(\"employee_id\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND TRY_CAST(\"employee_id\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND TRY_CAST(\"employee_id\" AS TEXT) IS NULL" }, { "name": "employee_id_logical_type_options_pattern_check", @@ -1411,24 +1645,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.176167007884942, + "execution_time_ms": 27.994290998321958, "check_path": "$.schema[1].properties[0].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must match pattern '^e[0-9]+$'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must match pattern '^e[0-9]+$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND NOT CAST(\"employee_id\" AS VARCHAR) ~ '^e[0-9]+$'", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"employee_id\" AS TEXT), '^e[0-9]+$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"employee_id\" AS TEXT), '^e[0-9]+$')" }, { "name": "employee_id_logical_type_options_minLength_check", @@ -1437,24 +1675,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.518875001580454, + "execution_time_ms": 27.982749990769662, "check_path": "$.schema[1].properties[0].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must have minimum length of 7", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_minLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must have minimum length of 7", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) < 7", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) < 7" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) < 7" }, { "name": "employee_id_logical_type_options_maxLength_check", @@ -1463,24 +1705,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 27.36829199420754, + "execution_time_ms": 28.017333999741822, "check_path": "$.schema[1].properties[0].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_id' must have maximum length of 7", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_logical_type_options_maxLength_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_id' must have maximum length of 7", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) > 7", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) > 7" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(TRY_CAST(\"employee_id\" AS TEXT)) > 7" }, { "name": "employee_id_required_check", @@ -1489,24 +1735,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.373791995225474, + "execution_time_ms": 5.478375009261072, "check_path": "$.schema[1].properties[0].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_id' must not contain NULL values", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_id' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL" }, { "name": "employee_id_unique_check", @@ -1515,24 +1765,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.475375004811212, + "execution_time_ms": 7.041749995551072, "check_path": "$.schema[1].properties[0].unique", "check_ref_type": "UniqueCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Column 'employee_id' must contain unique values", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_unique_check", + "type": "sql", + "dimension": "consistency", + "description": "Column 'employee_id' must contain unique values", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)" }, { "name": "employee_id_primary_key_check", @@ -1541,24 +1795,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.115499996463768, + "execution_time_ms": 5.410500001744367, "check_path": "$.schema[1].properties[0].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Primary key column 'employee_id' must be unique and not null", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_id_primary_key_check", + "type": "sql", + "dimension": "consistency", + "description": "Primary key column 'employee_id' must be unique and not null", + "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))", + "mustBe": 0 + }, "target": "demo_employee_list.employee_id", "logical_type": "string", "aggregation_type": "custom", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))" + "rendered_implementation": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))" }, { "name": "employee_last_name_column_exists_check", @@ -1567,24 +1825,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.529666992719285, + "execution_time_ms": 6.55170799291227, "check_path": "$.schema[1].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_last_name' must exist in 'demo_employee_list'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_last_name_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_last_name' must exist in 'demo_employee_list'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_last_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_last_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_last_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_last_name_logical_type_check", @@ -1593,24 +1855,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.200917007750832, + "execution_time_ms": 10.189666994847357, "check_path": "$.schema[1].properties[1].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_last_name' must be valid string", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_last_name_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_last_name' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_last_name\" IS NULL AND CAST(\"employee_last_name\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_last_name\" IS NULL AND TRY_CAST(\"employee_last_name\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_last_name\" IS NULL AND TRY_CAST(\"employee_last_name\" AS TEXT) IS NULL" }, { "name": "employee_last_name_required_check", @@ -1619,24 +1885,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.171458004042506, + "execution_time_ms": 6.203083001310006, "check_path": "$.schema[1].properties[1].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_last_name' must not contain NULL values", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_last_name_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_last_name' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_last_name\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_last_name\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_last_name\" IS NULL" }, { "name": "employee_first_name_column_exists_check", @@ -1645,24 +1915,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.425541996373795, + "execution_time_ms": 6.034209000063129, "check_path": "$.schema[1].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'employee_first_name' must exist in 'demo_employee_list'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_first_name_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'employee_first_name' must exist in 'demo_employee_list'", + "query": "SELECT COUNT(*) FROM (SELECT \"employee_first_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"employee_first_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"employee_first_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" }, { "name": "employee_first_name_logical_type_check", @@ -1671,24 +1945,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.74150000163354, + "execution_time_ms": 10.555083004874177, "check_path": "$.schema[1].properties[2].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'employee_first_name' must be valid string", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_first_name_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'employee_first_name' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_first_name\" IS NULL AND CAST(\"employee_first_name\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_first_name\" IS NULL AND TRY_CAST(\"employee_first_name\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_first_name\" IS NULL AND TRY_CAST(\"employee_first_name\" AS TEXT) IS NULL" }, { "name": "employee_first_name_required_check", @@ -1697,24 +1975,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.4498749998165295, + "execution_time_ms": 6.4019589917734265, "check_path": "$.schema[1].properties[2].required", "check_ref_type": "RequiredCheckReference", - "dimension": "completeness", - "type": "sql", - "description": "Column 'employee_first_name' must not contain NULL values", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "employee_first_name_required_check", + "type": "sql", + "dimension": "completeness", + "description": "Column 'employee_first_name' must not contain NULL values", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_first_name\" IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_first_name\" IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_first_name\" IS NULL" }, { "name": "current_title_column_exists_check", @@ -1723,24 +2005,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.187124996562488, + "execution_time_ms": 6.396415992639959, "check_path": "$.schema[1].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'current_title' must exist in 'demo_employee_list'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "current_title_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'current_title' must exist in 'demo_employee_list'", + "query": "SELECT COUNT(*) FROM (SELECT \"current_title\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_list.current_title", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"current_title\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"current_title\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" }, { "name": "current_title_logical_type_check", @@ -1749,24 +2035,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 9.6572080074111, + "execution_time_ms": 10.182790996623226, "check_path": "$.schema[1].properties[3].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'current_title' must be valid string", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "current_title_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'current_title' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"current_title\" IS NULL AND CAST(\"current_title\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.current_title", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"current_title\" IS NULL AND TRY_CAST(\"current_title\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"current_title\" IS NULL AND TRY_CAST(\"current_title\" AS TEXT) IS NULL" }, { "name": "phone_number_column_exists_check", @@ -1775,24 +2065,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.327791994204745, + "execution_time_ms": 6.789667007979006, "check_path": "$.schema[1].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must exist in 'demo_employee_list'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_column_exists_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must exist in 'demo_employee_list'", + "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", + "mustBe": 0 + }, "target": "demo_employee_list.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists" }, { "name": "phone_number_logical_type_check", @@ -1801,24 +2095,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.912875011446886, + "execution_time_ms": 10.848208010429516, "check_path": "$.schema[1].properties[4].logicalType", "check_ref_type": "LogicalTypeCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Values in 'phone_number' must be valid string", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_check", + "type": "sql", + "dimension": "conformity", + "description": "Values in 'phone_number' must be valid string", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND CAST(\"phone_number\" AS VARCHAR) IS NULL", + "mustBe": 0 + }, "target": "demo_employee_list.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND TRY_CAST(\"phone_number\" AS TEXT) IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND TRY_CAST(\"phone_number\" AS TEXT) IS NULL" }, { "name": "phone_number_logical_type_options_pattern_check", @@ -1827,24 +2125,28 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 44.16029200365301, + "execution_time_ms": 32.44387501035817, "check_path": "$.schema[1].properties[4].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Column 'phone_number' must match pattern '^\\+[0-9]+$'", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_logical_type_options_pattern_check", + "type": "sql", + "dimension": "conformity", + "description": "Column 'phone_number' must match pattern '^\\+[0-9]+$'", + "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND NOT CAST(\"phone_number\" AS VARCHAR) ~ '^\\+[0-9]+$'", + "mustBe": 0 + }, "target": "demo_employee_list.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"phone_number\" AS TEXT), '^\\+[0-9]+$')" + "rendered_implementation": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(\"phone_number\" AS TEXT), '^\\+[0-9]+$')" }, { "name": "phone_number_unique_check", @@ -1853,24 +2155,28 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.733582987682894, + "execution_time_ms": 6.953083007829264, "check_path": "$.schema[1].properties[4].unique", "check_ref_type": "UniqueCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Column 'phone_number' must contain unique values", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", + "contract_definition": { + "name": "phone_number_unique_check", + "type": "sql", + "dimension": "consistency", + "description": "Column 'phone_number' must contain unique values", + "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL GROUP BY \"phone_number\" HAVING COUNT(*) > 1)", + "mustBe": 0 + }, "target": "demo_employee_list.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL GROUP BY \"phone_number\" HAVING COUNT(*) > 1)" + "rendered_implementation": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL GROUP BY \"phone_number\" HAVING COUNT(*) > 1)" }, { "name": "PersonName", @@ -1879,24 +2185,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.657958003226668, + "execution_time_ms": 22.425041999667883, "check_path": "$.schema[1].properties[1].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "PersonName", + "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE employee_last_name IS NOT NULL\n AND LENGTH(CAST(employee_last_name AS TEXT)) > 66;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66" }, { "name": "PersonName", @@ -1905,24 +2218,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.439665988902561, + "execution_time_ms": 18.33512500161305, "check_path": "$.schema[1].properties[2].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "PersonName", + "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE employee_first_name IS NOT NULL\n AND LENGTH(CAST(employee_first_name AS TEXT)) > 66;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66" }, { "name": "TelephoneNumber", @@ -1931,24 +2251,31 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.721458006533794, + "execution_time_ms": 18.17870800732635, "check_path": "$.schema[1].properties[4].quality[0]", "check_ref_type": "SQLColumnCheckReference", - "dimension": "conformity", - "type": "sql", - "description": "Based on ITU T E.164 | e.g. 095236", - "severity": null, - "schema": "demo_employee_list", + "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "type": "sql", + "name": "TelephoneNumber", + "description": "Based on ITU T E.164 | e.g. 095236", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE CAST(phone_number AS TEXT) !~ '^[0-9]{1,12}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "target": "demo_employee_list.phone_number", "logical_type": "string", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list" ], - "rule": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')" }, { "name": "employee_id_exists_in_master_list", @@ -1957,24 +2284,31 @@ "expected_value": 0, "actual_value": 1, "failed_rows_count": 1, - "execution_time_ms": 47.93762500048615, + "execution_time_ms": 62.502582994056866, "check_path": "$.schema[0].quality[1]", "check_ref_type": "SQLTableCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "employee_id_exists_in_master_list", + "description": "Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.", + "type": "sql", + "dimension": "consistency", + "query": "SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref \n ON payroll.employee_id = ref.employee_id\nWHERE ref.employee_id IS NULL", + "mustBe": 0, + "tags": [ + "Cross-Table Validation" + ] + }, "target": "demo_employee_payroll", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list", "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL" }, { "name": "phone_number_exists_in_master_list", @@ -1983,24 +2317,31 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 14.986333000706509, + "execution_time_ms": 18.972791003761813, "check_path": "$.schema[0].quality[2]", "check_ref_type": "SQLTableCheckReference", - "dimension": "consistency", - "type": "sql", - "description": "Ensures the phone numbers recorded in payroll matches an entry in the master employee list.", - "severity": null, - "schema": "demo_employee_payroll", + "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "contract_definition": { + "name": "phone_number_exists_in_master_list", + "description": "Ensures the phone numbers recorded in payroll matches an entry in the master employee list.", + "type": "sql", + "dimension": "consistency", + "query": "SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref \n ON payroll.phone_number = ref.phone_number\nWHERE payroll.phone_number IS NOT NULL \n AND ref.phone_number IS NULL", + "mustBe": 0, + "tags": [ + "Cross-Table Validation" + ] + }, "target": "demo_employee_payroll", "aggregation_type": "count", "tables_in_query": [ "demo_employee_list", "demo_employee_payroll" ], - "rule": "SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL" + "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL" } ], "contract_metadata": { diff --git a/examples/vowl_usage_patterns_demo.ipynb b/examples/vowl_usage_patterns_demo.ipynb index cb7f5d1..4dff11a 100644 --- a/examples/vowl_usage_patterns_demo.ipynb +++ b/examples/vowl_usage_patterns_demo.ipynb @@ -125,7 +125,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/1521035067.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/1521035067.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " df = pd.read_csv(HDB_CSV).fillna(\"\")\n" ] }, @@ -322,29 +322,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 18.96 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 19.63 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 35.66 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 17.68 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 19.77 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 27.02 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 21.95 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 16.29 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.03 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.69 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.32 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 19.35 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.11 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 12.47 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.63 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.03 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.51 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.42 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.37 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.31 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.30 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.25 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.09 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.83 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.53 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.08 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.57 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.66 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.84 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 11.97 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.20 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.47 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.96 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.60 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.81 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 27.17 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.42 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.13 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.99 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.37 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 224.63 ms\n", + "Total Execution: 237.81 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -482,29 +482,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 20.28 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 16.87 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 13.73 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 28.15 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 23.29 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 20.10 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 19.86 ms |\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.88 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.79 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.77 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.23 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.76 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 5.94 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.68 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.81 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.60 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.51 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.63 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 13.75 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.67 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.61 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.71 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 13.13 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 59.53 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.59 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 40.14 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.23 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 28.74 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.38 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 13.72 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.27 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.57 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.99 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.28 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.91 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.51 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.15 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.94 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.08 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.33 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 191.20 ms\n", + "Total Execution: 310.90 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -659,7 +659,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/276778508.py:7: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/276778508.py:7: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " hdb_df = pd.read_csv(HDB_CSV).fillna(\"\")\n" ] }, @@ -696,29 +696,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 17.21 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 21.10 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 16.76 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 13.88 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 22.10 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 25.62 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 25.88 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 18.95 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.84 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.01 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.76 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.07 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.42 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 7.76 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.34 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.05 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.06 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.71 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.46 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 13.95 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.52 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.78 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.23 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.45 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.58 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.52 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.44 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.85 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.80 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 8.31 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.93 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 4.20 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.99 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.75 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.66 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 24.57 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.54 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.19 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.08 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.04 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 191.35 ms\n", + "Total Execution: 242.01 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -850,7 +850,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/1131799543.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/1131799543.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " hdb_df = pd.read_csv(HDB_CSV).fillna(\"\")\n" ] }, @@ -887,29 +887,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 32.37 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 46.19 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.83 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.90 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.33 ms |\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.98 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.54 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.41 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.20 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.19 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 32729 | 29.50 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.89 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.73 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.28 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.05 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.25 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.07 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.93 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.62 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.31 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 13.88 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.58 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.91 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.22 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.95 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.43 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.02 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 24.91 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.11 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 32729 | 36.21 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.87 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.39 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 13.31 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.34 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.98 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 26.75 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.09 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.63 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.74 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.08 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 292.24 ms\n" + "Total Execution: 353.72 ms\n" ] }, { @@ -986,28 +986,28 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 39.19 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.92 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.65 ms |\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.05 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.63 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.67 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.83 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.12 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.55 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 1264 | 10.29 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.97 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.12 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.63 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 27.04 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.85 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 19.84 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.67 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.60 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.65 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 16.65 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.98 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.41 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 29.02 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.96 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.85 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.77 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 25.19 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.14 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.14 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 1264 | 12.00 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.06 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.09 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.73 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 15.68 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 36.03 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 26.25 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.06 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.53 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.55 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.95 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 290.94 ms\n" + "Total Execution: 340.38 ms\n" ] }, { @@ -1076,29 +1076,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 7 | 25.39 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 7 | 26.30 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 24.12 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.79 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.03 ms |\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.61 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.75 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.03 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.09 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.03 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 137623 | 12.16 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.30 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.02 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.81 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.34 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.24 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.16 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.80 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.75 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.53 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 31.40 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 29.64 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 30.96 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 31.17 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.56 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.93 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.36 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 26.77 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.52 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 137623 | 18.38 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 12.15 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.53 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.84 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.53 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 11.48 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 27.49 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.63 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.32 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.79 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.45 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 291.35 ms\n" + "Total Execution: 342.81 ms\n" ] }, { @@ -1193,85 +1193,85 @@ "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 47.94 ms |\n", - "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 14.99 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 17.86 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 18.00 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 26.03 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 44.16 ms |\n", + "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 62.50 ms |\n", + "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 18.97 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 23.16 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 22.94 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 38.35 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 32.44 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 18.37 ms |\n", - "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 7.83 ms |\n", - "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.59 ms |\n", - "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.02 ms |\n", - "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.54 ms |\n", - "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.04 ms |\n", - "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.94 ms |\n", - "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.82 ms |\n", - "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.12 ms |\n", - "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.55 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.55 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.61 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.54 ms |\n", - "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 13.47 ms |\n", - "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.77 ms |\n", - "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.15 ms |\n", - "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.01 ms |\n", - "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.63 ms |\n", - "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 20.89 ms |\n", - "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.79 ms |\n", - "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 19.92 ms |\n", - "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.93 ms |\n", - "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.39 ms |\n", - "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.63 ms |\n", - "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.69 ms |\n", - "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.97 ms |\n", - "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.43 ms |\n", - "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.02 ms |\n", - "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.72 ms |\n", - "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.06 ms |\n", - "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.72 ms |\n", - "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.60 ms |\n", - "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.31 ms |\n", - "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.64 ms |\n", - "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.79 ms |\n", - "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.09 ms |\n", - "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.02 ms |\n", - "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.67 ms |\n", - "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.28 ms |\n", - "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.76 ms |\n", - "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.76 ms |\n", - "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.63 ms |\n", - "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.56 ms |\n", - "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.84 ms |\n", - "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 3.51 ms |\n", - "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.99 ms |\n", - "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.38 ms |\n", - "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 19.14 ms |\n", - "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.66 ms |\n", - "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.44 ms |\n", - "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.72 ms |\n", - "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 7.19 ms |\n", - "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 9.66 ms |\n", - "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.43 ms |\n", - "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.74 ms |\n", - "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.45 ms |\n", - "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.45 ms |\n", - "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.75 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 27.37 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 25.52 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 25.18 ms |\n", - "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.12 ms |\n", - "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.37 ms |\n", - "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.48 ms |\n", - "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.53 ms |\n", - "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.20 ms |\n", - "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.17 ms |\n", - "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.33 ms |\n", - "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.91 ms |\n", - "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 4.73 ms |\n", + "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 23.36 ms |\n", + "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 12.05 ms |\n", + "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.14 ms |\n", + "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.30 ms |\n", + "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.34 ms |\n", + "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.09 ms |\n", + "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.09 ms |\n", + "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.54 ms |\n", + "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.27 ms |\n", + "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.77 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 33.43 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.72 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 39.86 ms |\n", + "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 17.95 ms |\n", + "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.70 ms |\n", + "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.82 ms |\n", + "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.56 ms |\n", + "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.10 ms |\n", + "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.27 ms |\n", + "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.30 ms |\n", + "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.47 ms |\n", + "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.41 ms |\n", + "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.43 ms |\n", + "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.29 ms |\n", + "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.20 ms |\n", + "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.17 ms |\n", + "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.78 ms |\n", + "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.05 ms |\n", + "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 36.90 ms |\n", + "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 37.44 ms |\n", + "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.70 ms |\n", + "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.64 ms |\n", + "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.23 ms |\n", + "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.73 ms |\n", + "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.08 ms |\n", + "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.96 ms |\n", + "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.80 ms |\n", + "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.59 ms |\n", + "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.62 ms |\n", + "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 32.70 ms |\n", + "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.17 ms |\n", + "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.97 ms |\n", + "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 23.63 ms |\n", + "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 39.62 ms |\n", + "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.84 ms |\n", + "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 17.85 ms |\n", + "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.33 ms |\n", + "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.72 ms |\n", + "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 22.43 ms |\n", + "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 18.34 ms |\n", + "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 18.18 ms |\n", + "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.40 ms |\n", + "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.18 ms |\n", + "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.03 ms |\n", + "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.56 ms |\n", + "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.40 ms |\n", + "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.67 ms |\n", + "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.80 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 28.02 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 27.98 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 27.99 ms |\n", + "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.41 ms |\n", + "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.48 ms |\n", + "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 7.04 ms |\n", + "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.55 ms |\n", + "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.19 ms |\n", + "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.20 ms |\n", + "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.79 ms |\n", + "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.85 ms |\n", + "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.95 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "Total Execution: 929.05 ms\n", + "Total Execution: 1179.79 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -1508,85 +1508,85 @@ "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 15.56 ms |\n", - "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 15.05 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 18.18 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 18.13 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 23.97 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 25.93 ms |\n", + "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 19.60 ms |\n", + "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 19.22 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 33.03 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 29.10 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 27.58 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 34.25 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 31.29 ms |\n", - "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 6.32 ms |\n", - "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.34 ms |\n", - "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.28 ms |\n", - "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.31 ms |\n", - "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.33 ms |\n", - "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.48 ms |\n", - "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.54 ms |\n", - "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.70 ms |\n", - "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.58 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.59 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.73 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.54 ms |\n", - "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.45 ms |\n", - "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.76 ms |\n", - "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.13 ms |\n", - "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.02 ms |\n", - "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.52 ms |\n", - "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.52 ms |\n", - "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.27 ms |\n", - "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 17.27 ms |\n", - "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.42 ms |\n", - "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 25.92 ms |\n", - "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.85 ms |\n", - "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.26 ms |\n", - "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.15 ms |\n", - "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.58 ms |\n", - "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.32 ms |\n", - "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.25 ms |\n", - "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 27.59 ms |\n", - "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 27.09 ms |\n", - "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 4.94 ms |\n", - "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.07 ms |\n", - "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.79 ms |\n", - "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.07 ms |\n", - "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.11 ms |\n", - "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 21.22 ms |\n", - "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.88 ms |\n", - "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 24.13 ms |\n", - "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 26.45 ms |\n", - "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.28 ms |\n", - "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.50 ms |\n", - "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.20 ms |\n", - "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.57 ms |\n", - "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.23 ms |\n", - "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.21 ms |\n", - "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.85 ms |\n", - "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 19.38 ms |\n", - "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 17.30 ms |\n", - "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 17.11 ms |\n", - "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 16.96 ms |\n", - "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.63 ms |\n", - "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.41 ms |\n", - "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.66 ms |\n", - "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.59 ms |\n", - "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 4.99 ms |\n", - "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.54 ms |\n", - "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.06 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 25.67 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 26.59 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 27.04 ms |\n", - "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 4.88 ms |\n", - "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.57 ms |\n", - "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.98 ms |\n", - "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.81 ms |\n", - "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 9.24 ms |\n", - "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.30 ms |\n", - "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.27 ms |\n", - "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.70 ms |\n", - "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 5.49 ms |\n", + "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 18.26 ms |\n", + "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 7.78 ms |\n", + "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.32 ms |\n", + "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.55 ms |\n", + "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.84 ms |\n", + "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.65 ms |\n", + "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.92 ms |\n", + "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.49 ms |\n", + "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.08 ms |\n", + "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.94 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 32.97 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 36.35 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 33.75 ms |\n", + "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.29 ms |\n", + "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.44 ms |\n", + "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.55 ms |\n", + "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.74 ms |\n", + "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.88 ms |\n", + "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.25 ms |\n", + "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.75 ms |\n", + "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 21.22 ms |\n", + "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.34 ms |\n", + "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 28.87 ms |\n", + "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.18 ms |\n", + "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.28 ms |\n", + "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.74 ms |\n", + "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.07 ms |\n", + "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.36 ms |\n", + "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.04 ms |\n", + "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.17 ms |\n", + "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 29.45 ms |\n", + "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.16 ms |\n", + "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.31 ms |\n", + "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.11 ms |\n", + "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.52 ms |\n", + "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.38 ms |\n", + "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.28 ms |\n", + "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.73 ms |\n", + "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 28.53 ms |\n", + "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 33.01 ms |\n", + "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.19 ms |\n", + "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.01 ms |\n", + "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.52 ms |\n", + "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.89 ms |\n", + "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.94 ms |\n", + "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 20.53 ms |\n", + "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.83 ms |\n", + "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.81 ms |\n", + "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 23.15 ms |\n", + "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 23.96 ms |\n", + "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 31.66 ms |\n", + "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.89 ms |\n", + "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.71 ms |\n", + "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.94 ms |\n", + "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.97 ms |\n", + "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.59 ms |\n", + "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.14 ms |\n", + "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 9.16 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 34.49 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 34.22 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 28.95 ms |\n", + "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.01 ms |\n", + "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.81 ms |\n", + "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.94 ms |\n", + "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.76 ms |\n", + "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.43 ms |\n", + "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.22 ms |\n", + "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.92 ms |\n", + "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 11.75 ms |\n", + "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 7.58 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "Total Execution: 923.84 ms\n", + "Total Execution: 1136.30 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -1738,7 +1738,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/541641534.py:2: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/541641534.py:2: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " result = validate_data(contract=str(HDB_CONTRACT), df=pd.read_csv(HDB_CSV).fillna(\"\"))\n", "/Users/jamestth/DEP/DCP/dqmk/src/vowl/validation/runner.py:72: UserWarning: Arrow type conversion failed, loading problematic columns as strings: lease_commence_date. Type validation will occur in DQ checks. Original error: (\"Expected bytes, got a 'int' object\", 'Conversion failed for column lease_commence_date with type object')\n", " resolved[schema_name] = mapper.get_adapter(adapter_input, schema_name)\n" @@ -1778,29 +1778,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 20.72 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 21.38 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 20.99 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 17.01 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 28.14 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 67.69 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 26.97 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 23.85 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.38 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.21 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.78 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 21.57 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.17 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 7.59 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.00 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.76 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 9.82 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.11 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.01 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.30 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.27 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.30 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 5.34 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.13 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.35 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.12 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.53 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 30.76 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.14 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 11.94 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 8.97 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 15.56 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 14.57 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 6.42 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.60 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 20.45 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.07 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.87 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 7.94 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 18.52 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 209.83 ms\n" + "Total Execution: 332.49 ms\n" ] }, { @@ -1864,7 +1864,7 @@ " \n", " check_name\n", " target\n", - " schema\n", + " schema_name\n", " engine\n", " type\n", " dimension\n", @@ -1873,16 +1873,16 @@ " severity\n", " operator\n", " ...\n", - " failed_rows_count\n", - " aggregation_type\n", - " message\n", - " rule\n", - " tables_in_query\n", - " check_path\n", - " check_ref_type\n", " logical_type\n", " is_generated\n", " execution_time_ms\n", + " name\n", + " query\n", + " mustBe\n", + " metric\n", + " mustBeBetween\n", + " tags\n", + " arguments\n", " \n", " \n", " \n", @@ -1899,16 +1899,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[0].name\n", - " DeclaredColumnExistsCheckReference\n", " string\n", " True\n", - " 5.757583\n", + " 15.560625\n", + " month_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 1\n", @@ -1923,16 +1923,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[0].logicalType\n", - " LogicalTypeCheckReference\n", " string\n", " True\n", - " 9.819084\n", + " 14.568041\n", + " month_logical_type_check\n", + " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 2\n", @@ -1947,16 +1947,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[1].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.337791\n", + " 7.943917\n", + " town_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 3\n", @@ -1971,16 +1971,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM ...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[2].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.779208\n", + " 6.528958\n", + " flat_type_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM ...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 4\n", @@ -1995,16 +1995,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[3].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.379000\n", + " 7.354167\n", + " block_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 5\n", @@ -2019,16 +2019,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"street_name\" FRO...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[4].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.304541\n", + " 7.871333\n", + " street_name_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"street_name\" FRO...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 6\n", @@ -2043,16 +2043,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"storey_range\" FR...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[5].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.266625\n", + " 7.073708\n", + " storey_range_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"storey_range\" FR...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 7\n", @@ -2067,16 +2067,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" ...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[6].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.168709\n", + " 7.138625\n", + " floor_area_sqm_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" ...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 8\n", @@ -2091,16 +2091,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[7].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.213583\n", + " 7.122959\n", + " flat_model_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 9\n", @@ -2115,16 +2115,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"lease_commence_d...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[8].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 4.999250\n", + " 8.973291\n", + " lease_commence_date_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"lease_commence_d...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 10\n", @@ -2139,16 +2139,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"remaining_lease\"...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[9].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.113917\n", + " 6.421375\n", + " remaining_lease_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"remaining_lease\"...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 11\n", @@ -2163,16 +2163,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM (SELECT \"resale_price\" FR...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[10].name\n", - " DeclaredColumnExistsCheckReference\n", " NaN\n", " True\n", - " 5.010666\n", + " 7.602666\n", + " resale_price_column_exists_check\n", + " SELECT COUNT(*) FROM (SELECT \"resale_price\" FR...\n", + " 0\n", + " NaN\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 12\n", @@ -2187,16 +2187,16 @@ " NaN\n", " mustBeBetween\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\"\n", - " ['hdb_resale_prices']\n", - " $.schema[0].quality[0]\n", - " RowCountCheckReference\n", " NaN\n", " True\n", - " 7.587959\n", + " 11.940375\n", + " NaN\n", + " NaN\n", + " NaN\n", + " rowCount\n", + " [0, 30000000]\n", + " NaN\n", + " NaN\n", " \n", " \n", " 13\n", @@ -2211,16 +2211,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 2\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[0].quality[0]\n", - " SQLColumnCheckReference\n", " string\n", " False\n", - " 21.381875\n", + " 67.693125\n", + " Month\n", + " SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " ['SG-DRM v5.0']\n", + " NaN\n", " \n", " \n", " 14\n", @@ -2235,16 +2235,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[1].quality[0]\n", - " NullValuesCheckReference\n", " NaN\n", " True\n", - " 14.130417\n", + " 18.517292\n", + " NaN\n", + " NaN\n", + " 0\n", + " nullValues\n", + " NaN\n", + " NaN\n", + " NaN\n", " \n", " \n", " 15\n", @@ -2259,16 +2259,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[2].quality[0]\n", - " InvalidValuesCheckReference\n", " NaN\n", " True\n", - " 21.572666\n", + " 30.758625\n", + " NaN\n", + " NaN\n", + " 0\n", + " invalidValues\n", + " NaN\n", + " NaN\n", + " {'validValues': ['1 ROOM', '2 ROOM', '3 ROOM',...\n", " \n", " \n", " 16\n", @@ -2283,16 +2283,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 1\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[3].quality[0]\n", - " SQLColumnCheckReference\n", " NaN\n", " False\n", - " 20.722291\n", + " 28.137292\n", + " AddressBlockHouseNumber\n", + " SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " ['SG-DRM v5.0']\n", + " NaN\n", " \n", " \n", " 17\n", @@ -2307,16 +2307,16 @@ " error\n", " mustBe\n", " ...\n", - " 12\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[6].quality[0]\n", - " SQLColumnCheckReference\n", " NaN\n", " False\n", - " 17.006708\n", + " 23.855000\n", + " floor_area_must_be_less_than_200\n", + " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " ['User-Defined Quality Rule']\n", + " NaN\n", " \n", " \n", " 18\n", @@ -2331,16 +2331,16 @@ " NaN\n", " mustBe\n", " ...\n", - " 3\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[8].quality[0]\n", - " SQLColumnCheckReference\n", " NaN\n", " False\n", - " 20.985333\n", + " 26.971458\n", + " Year\n", + " SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " ['SG-DRM v5.0']\n", + " NaN\n", " \n", " \n", " 19\n", @@ -2355,20 +2355,20 @@ " error\n", " mustBe\n", " ...\n", - " 0\n", - " count\n", - " \n", - " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", - " ['hdb_resale_prices']\n", - " $.schema[0].properties[10].quality[0]\n", - " SQLColumnCheckReference\n", " NaN\n", " False\n", - " 18.295667\n", + " 20.452792\n", + " resale_price_must_not_exceed_2m\n", + " SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE...\n", + " 0\n", + " NaN\n", + " NaN\n", + " ['Business Metadata Enrichment']\n", + " NaN\n", " \n", " \n", "\n", - "

20 rows × 22 columns

\n", + "

20 rows × 29 columns

\n", "" ], "text/plain": [ @@ -2394,7 +2394,7 @@ "18 Year \n", "19 resale_price_must_not_exceed_2m \n", "\n", - " target schema engine type \\\n", + " target schema_name engine type \\\n", "0 hdb_resale_prices.month hdb_resale_prices sql sql \n", "1 hdb_resale_prices.month hdb_resale_prices sql sql \n", "2 hdb_resale_prices.town hdb_resale_prices sql sql \n", @@ -2438,95 +2438,117 @@ "18 conformity Based on ISO 8601, assumed to be in UTC +8 | YYYY FAILED \n", "19 conformity Resale price must not be more than 2 million SGD PASSED \n", "\n", - " severity operator ... failed_rows_count aggregation_type message \\\n", - "0 NaN mustBe ... 0 count \n", - "1 NaN mustBe ... 0 count \n", - "2 NaN mustBe ... 0 count \n", - "3 NaN mustBe ... 0 count \n", - "4 NaN mustBe ... 0 count \n", - "5 NaN mustBe ... 0 count \n", - "6 NaN mustBe ... 0 count \n", - "7 NaN mustBe ... 0 count \n", - "8 NaN mustBe ... 0 count \n", - "9 NaN mustBe ... 0 count \n", - "10 NaN mustBe ... 0 count \n", - "11 NaN mustBe ... 0 count \n", - "12 NaN mustBeBetween ... 0 count \n", - "13 NaN mustBe ... 2 count \n", - "14 NaN mustBe ... 0 count \n", - "15 NaN mustBe ... 0 count \n", - "16 NaN mustBe ... 1 count \n", - "17 error mustBe ... 12 count \n", - "18 NaN mustBe ... 3 count \n", - "19 error mustBe ... 0 count \n", + " severity operator ... logical_type is_generated execution_time_ms \\\n", + "0 NaN mustBe ... string True 15.560625 \n", + "1 NaN mustBe ... string True 14.568041 \n", + "2 NaN mustBe ... NaN True 7.943917 \n", + "3 NaN mustBe ... NaN True 6.528958 \n", + "4 NaN mustBe ... NaN True 7.354167 \n", + "5 NaN mustBe ... NaN True 7.871333 \n", + "6 NaN mustBe ... NaN True 7.073708 \n", + "7 NaN mustBe ... NaN True 7.138625 \n", + "8 NaN mustBe ... NaN True 7.122959 \n", + "9 NaN mustBe ... NaN True 8.973291 \n", + "10 NaN mustBe ... NaN True 6.421375 \n", + "11 NaN mustBe ... NaN True 7.602666 \n", + "12 NaN mustBeBetween ... NaN True 11.940375 \n", + "13 NaN mustBe ... string False 67.693125 \n", + "14 NaN mustBe ... NaN True 18.517292 \n", + "15 NaN mustBe ... NaN True 30.758625 \n", + "16 NaN mustBe ... NaN False 28.137292 \n", + "17 error mustBe ... NaN False 23.855000 \n", + "18 NaN mustBe ... NaN False 26.971458 \n", + "19 error mustBe ... NaN False 20.452792 \n", + "\n", + " name \\\n", + "0 month_column_exists_check \n", + "1 month_logical_type_check \n", + "2 town_column_exists_check \n", + "3 flat_type_column_exists_check \n", + "4 block_column_exists_check \n", + "5 street_name_column_exists_check \n", + "6 storey_range_column_exists_check \n", + "7 floor_area_sqm_column_exists_check \n", + "8 flat_model_column_exists_check \n", + "9 lease_commence_date_column_exists_check \n", + "10 remaining_lease_column_exists_check \n", + "11 resale_price_column_exists_check \n", + "12 NaN \n", + "13 Month \n", + "14 NaN \n", + "15 NaN \n", + "16 AddressBlockHouseNumber \n", + "17 floor_area_must_be_less_than_200 \n", + "18 Year \n", + "19 resale_price_must_not_exceed_2m \n", "\n", - " rule tables_in_query \\\n", - "0 SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb... ['hdb_resale_prices'] \n", - "1 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "2 SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_... ['hdb_resale_prices'] \n", - "3 SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM ... ['hdb_resale_prices'] \n", - "4 SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb... ['hdb_resale_prices'] \n", - "5 SELECT COUNT(*) FROM (SELECT \"street_name\" FRO... ['hdb_resale_prices'] \n", - "6 SELECT COUNT(*) FROM (SELECT \"storey_range\" FR... ['hdb_resale_prices'] \n", - "7 SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" ... ['hdb_resale_prices'] \n", - "8 SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM... ['hdb_resale_prices'] \n", - "9 SELECT COUNT(*) FROM (SELECT \"lease_commence_d... ['hdb_resale_prices'] \n", - "10 SELECT COUNT(*) FROM (SELECT \"remaining_lease\"... ['hdb_resale_prices'] \n", - "11 SELECT COUNT(*) FROM (SELECT \"resale_price\" FR... ['hdb_resale_prices'] \n", - "12 SELECT COUNT(*) FROM \"hdb_resale_prices\" ['hdb_resale_prices'] \n", - "13 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "14 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "15 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "16 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "17 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "18 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", - "19 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... ['hdb_resale_prices'] \n", + " query mustBe metric \\\n", + "0 SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb... 0 NaN \n", + "1 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... 0 NaN \n", + "2 SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_... 0 NaN \n", + "3 SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM ... 0 NaN \n", + "4 SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb... 0 NaN \n", + "5 SELECT COUNT(*) FROM (SELECT \"street_name\" FRO... 0 NaN \n", + "6 SELECT COUNT(*) FROM (SELECT \"storey_range\" FR... 0 NaN \n", + "7 SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" ... 0 NaN \n", + "8 SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM... 0 NaN \n", + "9 SELECT COUNT(*) FROM (SELECT \"lease_commence_d... 0 NaN \n", + "10 SELECT COUNT(*) FROM (SELECT \"remaining_lease\"... 0 NaN \n", + "11 SELECT COUNT(*) FROM (SELECT \"resale_price\" FR... 0 NaN \n", + "12 NaN NaN rowCount \n", + "13 SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE... 0 NaN \n", + "14 NaN 0 nullValues \n", + "15 NaN 0 invalidValues \n", + "16 SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE... 0 NaN \n", + "17 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... 0 NaN \n", + "18 SELECT COUNT(*)\\nFROM \"hdb_resale_prices\"\\nWHE... 0 NaN \n", + "19 SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE... 0 NaN \n", "\n", - " check_path check_ref_type \\\n", - "0 $.schema[0].properties[0].name DeclaredColumnExistsCheckReference \n", - "1 $.schema[0].properties[0].logicalType LogicalTypeCheckReference \n", - "2 $.schema[0].properties[1].name DeclaredColumnExistsCheckReference \n", - "3 $.schema[0].properties[2].name DeclaredColumnExistsCheckReference \n", - "4 $.schema[0].properties[3].name DeclaredColumnExistsCheckReference \n", - "5 $.schema[0].properties[4].name DeclaredColumnExistsCheckReference \n", - "6 $.schema[0].properties[5].name DeclaredColumnExistsCheckReference \n", - "7 $.schema[0].properties[6].name DeclaredColumnExistsCheckReference \n", - "8 $.schema[0].properties[7].name DeclaredColumnExistsCheckReference \n", - "9 $.schema[0].properties[8].name DeclaredColumnExistsCheckReference \n", - "10 $.schema[0].properties[9].name DeclaredColumnExistsCheckReference \n", - "11 $.schema[0].properties[10].name DeclaredColumnExistsCheckReference \n", - "12 $.schema[0].quality[0] RowCountCheckReference \n", - "13 $.schema[0].properties[0].quality[0] SQLColumnCheckReference \n", - "14 $.schema[0].properties[1].quality[0] NullValuesCheckReference \n", - "15 $.schema[0].properties[2].quality[0] InvalidValuesCheckReference \n", - "16 $.schema[0].properties[3].quality[0] SQLColumnCheckReference \n", - "17 $.schema[0].properties[6].quality[0] SQLColumnCheckReference \n", - "18 $.schema[0].properties[8].quality[0] SQLColumnCheckReference \n", - "19 $.schema[0].properties[10].quality[0] SQLColumnCheckReference \n", + " mustBeBetween tags \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "5 NaN NaN \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 NaN NaN \n", + "10 NaN NaN \n", + "11 NaN NaN \n", + "12 [0, 30000000] NaN \n", + "13 NaN ['SG-DRM v5.0'] \n", + "14 NaN NaN \n", + "15 NaN NaN \n", + "16 NaN ['SG-DRM v5.0'] \n", + "17 NaN ['User-Defined Quality Rule'] \n", + "18 NaN ['SG-DRM v5.0'] \n", + "19 NaN ['Business Metadata Enrichment'] \n", "\n", - " logical_type is_generated execution_time_ms \n", - "0 string True 5.757583 \n", - "1 string True 9.819084 \n", - "2 NaN True 5.337791 \n", - "3 NaN True 5.779208 \n", - "4 NaN True 5.379000 \n", - "5 NaN True 5.304541 \n", - "6 NaN True 5.266625 \n", - "7 NaN True 5.168709 \n", - "8 NaN True 5.213583 \n", - "9 NaN True 4.999250 \n", - "10 NaN True 5.113917 \n", - "11 NaN True 5.010666 \n", - "12 NaN True 7.587959 \n", - "13 string False 21.381875 \n", - "14 NaN True 14.130417 \n", - "15 NaN True 21.572666 \n", - "16 NaN False 20.722291 \n", - "17 NaN False 17.006708 \n", - "18 NaN False 20.985333 \n", - "19 NaN False 18.295667 \n", + " arguments \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "5 NaN \n", + "6 NaN \n", + "7 NaN \n", + "8 NaN \n", + "9 NaN \n", + "10 NaN \n", + "11 NaN \n", + "12 NaN \n", + "13 NaN \n", + "14 NaN \n", + "15 {'validValues': ['1 ROOM', '2 ROOM', '3 ROOM',... \n", + "16 NaN \n", + "17 NaN \n", + "18 NaN \n", + "19 NaN \n", "\n", - "[20 rows x 22 columns]" + "[20 rows x 29 columns]" ] }, "execution_count": 15, @@ -3368,30 +3390,30 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 89.63 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 127.84 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 55.89 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 93.62 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 160.29 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 59.79 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | ERROR | mustBe | | | 6.54 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | ERROR | mustBe | | | 6.23 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | ERROR | mustBe | | | 8.04 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | ERROR | mustBe | | | 7.08 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.44 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 2.49 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.82 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 36.51 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.50 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 23.17 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 2.44 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.92 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 22.52 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.59 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 2.44 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.46 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 2.91 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.38 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 15.43 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.50 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.39 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.36 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 36.61 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.39 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 10.54 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.31 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 3.05 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 10.14 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.43 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.28 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.51 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.27 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.30 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 17.85 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 407.16 ms\n", + "Total Execution: 420.77 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -3497,7 +3519,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/4060035347.py:16: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/4060035347.py:16: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " hdb_df = pd.read_csv(HDB_CSV).fillna(\"\").astype(str)\n" ] }, @@ -3578,29 +3600,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 72.46 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 102.15 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 64.81 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 67.15 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 74.26 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 78.58 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 66.15 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | FAILED | mustBe | 0 | 12 | 70.42 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.47 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.18 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.48 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 67.21 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.30 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 21.32 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.25 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.83 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 52.16 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.28 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.13 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 67.68 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.16 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.31 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.23 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 42.42 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.34 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.29 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.33 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 71.98 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.29 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 11.12 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.37 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 2.43 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 43.69 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.30 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.35 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 71.27 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.43 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.48 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1.55 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 43.47 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 571.97 ms\n", + "Total Execution: 547.11 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -3711,27 +3733,27 @@ "Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "26/04/09 13:04:08 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_35770/2696327818.py:24: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "26/04/28 14:32:15 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/2696327818.py:24: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " pd.read_csv(HDB_CSV).fillna(\"\").astype(str)\n", - "26/04/09 13:04:15 WARN TaskSetManager: Stage 0 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "Exception ignored while finalizing file <_io.BufferedWriter name=5>: \n", + "26/04/28 14:32:23 WARN TaskSetManager: Stage 0 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "Exception ignored while finalizing file <_io.BufferedWriter name=5>:(0 + 1) / 1]\n", "Traceback (most recent call last):\n", " File \"/Users/jamestth/DEP/DCP/dqmk/.venv/lib/python3.14/site-packages/pyspark/python/lib/pyspark.zip/pyspark/daemon.py\", line 200, in manager\n", "BrokenPipeError: [Errno 32] Broken pipe\n", - "26/04/09 13:04:16 WARN TaskSetManager: Stage 3 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:17 WARN TaskSetManager: Stage 26 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:17 WARN TaskSetManager: Stage 29 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:17 WARN TaskSetManager: Stage 32 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:18 WARN TaskSetManager: Stage 35 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:18 WARN TaskSetManager: Stage 38 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:18 WARN TaskSetManager: Stage 41 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:20 WARN TaskSetManager: Stage 44 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:20 WARN TaskSetManager: Stage 47 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:21 WARN TaskSetManager: Stage 50 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:22 WARN TaskSetManager: Stage 53 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:22 WARN TaskSetManager: Stage 54 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", - "26/04/09 13:04:22 WARN TaskSetManager: Stage 55 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n" + "26/04/28 14:32:25 WARN TaskSetManager: Stage 3 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:26 WARN TaskSetManager: Stage 26 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:26 WARN TaskSetManager: Stage 29 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:27 WARN TaskSetManager: Stage 32 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:27 WARN TaskSetManager: Stage 35 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:28 WARN TaskSetManager: Stage 38 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:28 WARN TaskSetManager: Stage 41 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:30 WARN TaskSetManager: Stage 44 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:30 WARN TaskSetManager: Stage 47 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:31 WARN TaskSetManager: Stage 50 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:32 WARN TaskSetManager: Stage 53 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:32 WARN TaskSetManager: Stage 54 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n", + "26/04/28 14:32:32 WARN TaskSetManager: Stage 55 contains a task of very large size (24936 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -3767,29 +3789,29 @@ "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 247.61 ms |\n", - "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 275.47 ms |\n", - "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 350.66 ms |\n", + "| AddressBlockHouseNumber | hdb_resale_prices.block | hdb_resale_prices | FAILED | mustBe | 0 | 1 | 321.11 ms |\n", + "| Month | hdb_resale_prices.month | hdb_resale_prices | FAILED | mustBe | 0 | 2 | 709.63 ms |\n", + "| Year | hdb_resale_prices.lease_commence_date | hdb_resale_prices | FAILED | mustBe | 0 | 3 | 356.39 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 26.37 ms |\n", - "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 25.68 ms |\n", - "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 27.43 ms |\n", - "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 267.41 ms |\n", - "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1297.35 ms |\n", - "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 25.05 ms |\n", - "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 243.90 ms |\n", - "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 24.44 ms |\n", - "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 156.84 ms |\n", - "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 448.01 ms |\n", - "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 23.90 ms |\n", - "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 23.34 ms |\n", - "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1132.20 ms |\n", - "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 27.02 ms |\n", - "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 28.03 ms |\n", - "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 28.85 ms |\n", - "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 245.10 ms |\n", + "| block_column_exists_check | hdb_resale_prices.block | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 33.57 ms |\n", + "| flat_model_column_exists_check | hdb_resale_prices.flat_model | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 32.45 ms |\n", + "| flat_type_column_exists_check | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 41.11 ms |\n", + "| flat_type_invalidValues | hdb_resale_prices.flat_type | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 337.98 ms |\n", + "| floor_area_must_be_less_than_200 | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1678.28 ms |\n", + "| floor_area_sqm_column_exists_check | hdb_resale_prices.floor_area_sqm | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 33.80 ms |\n", + "| hdb_resale_prices_rowCount | hdb_resale_prices | hdb_resale_prices | PASSED | mustBeBetween | [0, 30000000] | 201879 | 302.14 ms |\n", + "| lease_commence_date_column_exists_check | hdb_resale_prices.lease_commence_date | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 30.95 ms |\n", + "| month_column_exists_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 223.94 ms |\n", + "| month_logical_type_check | hdb_resale_prices.month | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 540.22 ms |\n", + "| remaining_lease_column_exists_check | hdb_resale_prices.remaining_lease | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 33.29 ms |\n", + "| resale_price_column_exists_check | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 31.51 ms |\n", + "| resale_price_must_not_exceed_2m | hdb_resale_prices.resale_price | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 1342.07 ms |\n", + "| storey_range_column_exists_check | hdb_resale_prices.storey_range | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 32.16 ms |\n", + "| street_name_column_exists_check | hdb_resale_prices.street_name | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 33.20 ms |\n", + "| town_column_exists_check | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 42.77 ms |\n", + "| town_nullValues | hdb_resale_prices.town | hdb_resale_prices | PASSED | mustBe | 0 | 0 | 411.73 ms |\n", "+-----------------------------------------+---------------------------------------+-------------------+--------+---------------+---------------+--------+----------------+\n", - "Total Execution: 4924.64 ms\n", + "Total Execution: 6568.30 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", @@ -3964,86 +3986,86 @@ "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", "| check_id | Target | tables_in_query | status | operator | expected | actual | execution time |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 19.63 ms |\n", - "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 25.33 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 24.05 ms |\n", - "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 20.93 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 31.13 ms |\n", - "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 31.11 ms |\n", + "| employee_id_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 1 | 22.34 ms |\n", + "| phone_number_exists_in_master_list | demo_employee_payroll | demo_employee_list, demo_employee_payroll | FAILED | mustBe | 0 | 2 | 21.83 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 28.75 ms |\n", + "| Date With Timezone | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 26.35 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_payroll.phone_number | demo_employee_payroll | FAILED | mustBe | 0 | 3 | 29.79 ms |\n", + "| phone_number_logical_type_options_pattern_check | demo_employee_list.phone_number | demo_employee_list | FAILED | mustBe | 0 | 2 | 37.43 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | ERROR | mustBe | | | 11.68 ms |\n", - "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | ERROR | mustBe | | | 11.47 ms |\n", + "| total_amt_employee_equals_gross_plus_cpf | demo_employee_payroll | demo_employee_payroll | ERROR | mustBe | | | 9.91 ms |\n", + "| total_amt_equals_employer_cpf_plus_total_amt_employee | demo_employee_payroll.total_amt | demo_employee_payroll | ERROR | mustBe | | | 10.20 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.10 ms |\n", - "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 11.02 ms |\n", - "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.33 ms |\n", - "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.83 ms |\n", - "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.75 ms |\n", - "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 26.99 ms |\n", - "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.48 ms |\n", - "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.50 ms |\n", - "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 21.97 ms |\n", - "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 20.38 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 38.10 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 45.44 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 35.50 ms |\n", - "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 18.59 ms |\n", - "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.36 ms |\n", - "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 17.50 ms |\n", - "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.34 ms |\n", - "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.05 ms |\n", - "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.19 ms |\n", - "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.74 ms |\n", - "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 26.99 ms |\n", - "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.16 ms |\n", - "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 38.65 ms |\n", - "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.21 ms |\n", - "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.77 ms |\n", - "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.41 ms |\n", - "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.27 ms |\n", - "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 23.58 ms |\n", - "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.90 ms |\n", - "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 34.99 ms |\n", - "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 35.39 ms |\n", - "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.15 ms |\n", - "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.72 ms |\n", - "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 11.94 ms |\n", - "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.41 ms |\n", - "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 67.69 ms |\n", - "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.02 ms |\n", - "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.37 ms |\n", - "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 31.01 ms |\n", - "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.69 ms |\n", - "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.34 ms |\n", - "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.27 ms |\n", - "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 12.42 ms |\n", - "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.31 ms |\n", - "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.27 ms |\n", - "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 23.83 ms |\n", - "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 23.80 ms |\n", - "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 25.18 ms |\n", - "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 23.11 ms |\n", - "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.33 ms |\n", - "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.22 ms |\n", - "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.69 ms |\n", - "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.49 ms |\n", - "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.76 ms |\n", - "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.36 ms |\n", - "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 13.37 ms |\n", - "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 29.79 ms |\n", - "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 29.49 ms |\n", - "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 28.42 ms |\n", - "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.64 ms |\n", - "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 9.38 ms |\n", - "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.63 ms |\n", - "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.33 ms |\n", - "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 13.06 ms |\n", - "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.47 ms |\n", - "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 7.98 ms |\n", - "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 13.20 ms |\n", - "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.99 ms |\n", + "| Month | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 27.66 ms |\n", + "| demo_employee_payroll_rowCount | demo_employee_payroll | demo_employee_payroll | PASSED | mustBeGreaterThan | 0 | 3 | 19.02 ms |\n", + "| employee_cpf_amt_column_exists_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.71 ms |\n", + "| employee_cpf_amt_logical_type_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.33 ms |\n", + "| employee_cpf_amt_required_check | demo_employee_payroll.employee_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.80 ms |\n", + "| employee_gross_amt_column_exists_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.42 ms |\n", + "| employee_gross_amt_logical_type_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.26 ms |\n", + "| employee_gross_amt_required_check | demo_employee_payroll.employee_gross_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.37 ms |\n", + "| employee_id_column_exists_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.64 ms |\n", + "| employee_id_logical_type_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.21 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 38.14 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 37.98 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 42.30 ms |\n", + "| employee_id_nullValues | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.95 ms |\n", + "| employee_id_primary_key_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.58 ms |\n", + "| employee_id_required_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.35 ms |\n", + "| employee_id_unique_check | demo_employee_payroll.employee_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.42 ms |\n", + "| employer_cpf_amt_column_exists_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.75 ms |\n", + "| employer_cpf_amt_logical_type_check | demo_employee_payroll.employer_cpf_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 20.45 ms |\n", + "| month_column_exists_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.85 ms |\n", + "| month_invalidValues | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 33.05 ms |\n", + "| month_logical_type_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.06 ms |\n", + "| month_logical_type_options_pattern_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 39.85 ms |\n", + "| month_required_check | demo_employee_payroll.month | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.57 ms |\n", + "| payroll_end_dt_column_exists_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.13 ms |\n", + "| payroll_end_dt_logical_type_check | demo_employee_payroll.payroll_end_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 15.37 ms |\n", + "| payroll_id_column_exists_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.62 ms |\n", + "| payroll_id_logical_type_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 13.02 ms |\n", + "| payroll_id_logical_type_options_maxLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 36.76 ms |\n", + "| payroll_id_logical_type_options_minLength_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 36.77 ms |\n", + "| payroll_id_logical_type_options_pattern_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 42.52 ms |\n", + "| payroll_id_primary_key_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.55 ms |\n", + "| payroll_id_required_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 9.33 ms |\n", + "| payroll_id_unique_check | demo_employee_payroll.payroll_id | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.62 ms |\n", + "| payroll_start_dt_column_exists_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.51 ms |\n", + "| payroll_start_dt_logical_type_check | demo_employee_payroll.payroll_start_dt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 16.04 ms |\n", + "| phone_number_column_exists_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.68 ms |\n", + "| phone_number_logical_type_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 10.45 ms |\n", + "| phone_number_logical_type_options_maxLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 64.09 ms |\n", + "| phone_number_logical_type_options_minLength_check | demo_employee_payroll.phone_number | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 30.43 ms |\n", + "| total_amt_column_exists_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 7.84 ms |\n", + "| total_amt_employee_column_exists_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 5.86 ms |\n", + "| total_amt_employee_logical_type_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 8.93 ms |\n", + "| total_amt_employee_required_check | demo_employee_payroll.total_amt_employee | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 6.31 ms |\n", + "| total_amt_logical_type_check | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 14.93 ms |\n", + "| total_amt_missingValues | demo_employee_payroll.total_amt | demo_employee_payroll | PASSED | mustBe | 0 | 0 | 22.75 ms |\n", + "| PersonName | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 30.36 ms |\n", + "| PersonName | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 25.88 ms |\n", + "| TelephoneNumber | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 24.56 ms |\n", + "| current_title_column_exists_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.55 ms |\n", + "| current_title_logical_type_check | demo_employee_list.current_title | demo_employee_list | PASSED | mustBe | 0 | 0 | 14.15 ms |\n", + "| employee_first_name_column_exists_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.50 ms |\n", + "| employee_first_name_logical_type_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 12.99 ms |\n", + "| employee_first_name_required_check | demo_employee_list.employee_first_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 9.73 ms |\n", + "| employee_id_column_exists_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 6.80 ms |\n", + "| employee_id_logical_type_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 11.26 ms |\n", + "| employee_id_logical_type_options_maxLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 33.80 ms |\n", + "| employee_id_logical_type_options_minLength_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 32.77 ms |\n", + "| employee_id_logical_type_options_pattern_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 31.48 ms |\n", + "| employee_id_primary_key_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.08 ms |\n", + "| employee_id_required_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.72 ms |\n", + "| employee_id_unique_check | demo_employee_list.employee_id | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.80 ms |\n", + "| employee_last_name_column_exists_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 8.32 ms |\n", + "| employee_last_name_logical_type_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 19.79 ms |\n", + "| employee_last_name_required_check | demo_employee_list.employee_last_name | demo_employee_list | PASSED | mustBe | 0 | 0 | 10.76 ms |\n", + "| phone_number_column_exists_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 13.25 ms |\n", + "| phone_number_logical_type_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 17.65 ms |\n", + "| phone_number_unique_check | demo_employee_list.phone_number | demo_employee_list | PASSED | mustBe | 0 | 0 | 11.12 ms |\n", "+-------------------------------------------------------+------------------------------------------+-------------------------------------------+--------+-------------------+----------+--------+----------------+\n", - "Total Execution: 1399.97 ms\n", + "Total Execution: 1353.15 ms\n", "\n", "=== Failed Checks and Rows (up to 5 row(s) per failed check) ===\n", "\n", From 9c6127cdca62fed6b82da29f0191e7286725e3a7 Mon Sep 17 00:00:00 2001 From: james_teo Date: Tue, 28 Apr 2026 17:20:36 +0800 Subject: [PATCH 3/4] Update usage examples --- .../vowl_demo_HDB_results_check_results.csv | 48 +- examples/vowl_demo_HDB_results_summary.json | 266 +- ...demo_multi_table_results_check_results.csv | 175 +- ...vowl_demo_multi_table_results_summary.json | 947 ++++-- examples/vowl_usage_patterns_demo.ipynb | 3013 ++++++++++++----- 5 files changed, 3182 insertions(+), 1267 deletions(-) diff --git a/examples/vowl_demo_HDB_results_check_results.csv b/examples/vowl_demo_HDB_results_check_results.csv index f717f12..7daeac4 100644 --- a/examples/vowl_demo_HDB_results_check_results.csv +++ b/examples/vowl_demo_HDB_results_check_results.csv @@ -1,27 +1,21 @@ -"check_name","target","schema_name","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms","name","query","mustBe","metric","mustBeBetween","tags","arguments" -"month_column_exists_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Column 'month' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",15.560625004582107,"month_column_exists_check","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"month_logical_type_check","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['hdb_resale_prices']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",14.568041005986743,"month_logical_type_check","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND CAST(""month"" AS VARCHAR) IS NULL","0",,,, -"town_column_exists_check","hdb_resale_prices.town","hdb_resale_prices","sql","sql","conformity","Column 'town' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference",,"True",7.94391700765118,"town_column_exists_check","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"flat_type_column_exists_check","hdb_resale_prices.flat_type","hdb_resale_prices","sql","sql","conformity","Column 'flat_type' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference",,"True",6.528957994305529,"flat_type_column_exists_check","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"block_column_exists_check","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","Column 'block' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference",,"True",7.354166998993605,"block_column_exists_check","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"street_name_column_exists_check","hdb_resale_prices.street_name","hdb_resale_prices","sql","sql","conformity","Column 'street_name' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference",,"True",7.871333000366576,"street_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"storey_range_column_exists_check","hdb_resale_prices.storey_range","hdb_resale_prices","sql","sql","conformity","Column 'storey_range' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference",,"True",7.073707995004952,"storey_range_column_exists_check","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"floor_area_sqm_column_exists_check","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","conformity","Column 'floor_area_sqm' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference",,"True",7.138625005609356,"floor_area_sqm_column_exists_check","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"flat_model_column_exists_check","hdb_resale_prices.flat_model","hdb_resale_prices","sql","sql","conformity","Column 'flat_model' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference",,"True",7.1229590103030205,"flat_model_column_exists_check","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"lease_commence_date_column_exists_check","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Column 'lease_commence_date' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference",,"True",8.973290998255834,"lease_commence_date_column_exists_check","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"remaining_lease_column_exists_check","hdb_resale_prices.remaining_lease","hdb_resale_prices","sql","sql","conformity","Column 'remaining_lease' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference",,"True",6.421375001082197,"remaining_lease_column_exists_check","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"resale_price_column_exists_check","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Column 'resale_price' must exist in 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference",,"True",7.602665995364077,"resale_price_column_exists_check","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","0",,,, -"hdb_resale_prices_rowCount","hdb_resale_prices","hdb_resale_prices","sql","library","completeness","Total row count for 'hdb_resale_prices'","PASSED",,"mustBeBetween","201879","[0, 30000000]",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices""","['hdb_resale_prices']","$.schema[0].quality[0]","RowCountCheckReference",,"True",11.940374999539927,,,,"rowCount","[0, 30000000]",, -"Month","hdb_resale_prices.month","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['hdb_resale_prices']","$.schema[0].properties[0].quality[0]","SQLColumnCheckReference","string","False",67.6931250054622,"Month","SELECT COUNT(*) -FROM ""hdb_resale_prices"" -WHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';","0",,,"['SG-DRM v5.0']", -"town_nullValues","hdb_resale_prices.town","hdb_resale_prices","sql","library","completeness","Count of NULL values in 'town' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE ""town"" IS NULL","['hdb_resale_prices']","$.schema[0].properties[1].quality[0]","NullValuesCheckReference",,"True",18.517291988246143,,,"0","nullValues",,, -"flat_type_invalidValues","hdb_resale_prices.flat_type","hdb_resale_prices","sql","library","conformity","Count of invalid values in 'flat_type' of 'hdb_resale_prices'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""flat_type"" IS NULL AND NOT TRY_CAST(""flat_type"" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')","['hdb_resale_prices']","$.schema[0].properties[2].quality[0]","InvalidValuesCheckReference",,"True",30.758624998270534,,,"0","invalidValues",,,"{'validValues': ['1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION']}" -"AddressBlockHouseNumber","hdb_resale_prices.block","hdb_resale_prices","sql","sql","conformity","House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')","['hdb_resale_prices']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference",,"False",28.13729199988302,"AddressBlockHouseNumber","SELECT COUNT(*) -FROM ""hdb_resale_prices"" -WHERE CAST(block AS TEXT) !~ '^[A-Za-z0-9]{1,10}$';","0",,,"['SG-DRM v5.0']", -"floor_area_must_be_less_than_200","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","sql","consistency","Validates that floor area must be less than 200","FAILED","error","mustBe","12","0",12,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200","['hdb_resale_prices']","$.schema[0].properties[6].quality[0]","SQLColumnCheckReference",,"False",23.854999992181547,"floor_area_must_be_less_than_200","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE floor_area_sqm >= 200","0",,,"['User-Defined Quality Rule']", -"Year","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')","['hdb_resale_prices']","$.schema[0].properties[8].quality[0]","SQLColumnCheckReference",,"False",26.971458006300963,"Year","SELECT COUNT(*) -FROM ""hdb_resale_prices"" -WHERE CAST(lease_commence_date AS TEXT) !~ '^[0-9]{4}$';","0",,,"['SG-DRM v5.0']", -"resale_price_must_not_exceed_2m","hdb_resale_prices.resale_price","hdb_resale_prices","sql","sql","conformity","Resale price must not be more than 2 million SGD","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000","['hdb_resale_prices']","$.schema[0].properties[10].quality[0]","SQLColumnCheckReference",,"False",20.45279199955985,"resale_price_must_not_exceed_2m","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE resale_price > 2000000","0",,,"['Business Metadata Enrichment']", +"check_name","target","schema_name","engine","status","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms" +"month_column_exists_check","hdb_resale_prices.month","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",9.511541997198947 +"month_logical_type_check","hdb_resale_prices.month","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['hdb_resale_prices']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",15.60741598950699 +"town_column_exists_check","hdb_resale_prices.town","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""town"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference",,"True",7.495708996430039 +"flat_type_column_exists_check","hdb_resale_prices.flat_type","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_type"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference",,"True",7.0631249982398 +"block_column_exists_check","hdb_resale_prices.block","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""block"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference",,"True",8.010416000615805 +"street_name_column_exists_check","hdb_resale_prices.street_name","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""street_name"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference",,"True",6.567249991348945 +"storey_range_column_exists_check","hdb_resale_prices.storey_range","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""storey_range"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference",,"True",7.874708011513576 +"floor_area_sqm_column_exists_check","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""floor_area_sqm"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference",,"True",7.521874998928979 +"flat_model_column_exists_check","hdb_resale_prices.flat_model","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""flat_model"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference",,"True",7.332666995353065 +"lease_commence_date_column_exists_check","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""lease_commence_date"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference",,"True",8.957708007073961 +"remaining_lease_column_exists_check","hdb_resale_prices.remaining_lease","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""remaining_lease"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference",,"True",7.871249996242113 +"resale_price_column_exists_check","hdb_resale_prices.resale_price","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""resale_price"" FROM ""hdb_resale_prices"" LIMIT 0) AS _vowl_column_exists","['hdb_resale_prices']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference",,"True",8.052582998061553 +"hdb_resale_prices_rowCount","hdb_resale_prices","hdb_resale_prices","sql","PASSED","mustBeBetween","201879","[0, 30000000]",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices""","['hdb_resale_prices']","$.schema[0].quality[0]","RowCountCheckReference",,"True",10.504459001822397 +"Month","hdb_resale_prices.month","hdb_resale_prices","sql","FAILED","mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['hdb_resale_prices']","$.schema[0].properties[0].quality[0]","SQLColumnCheckReference","string","False",31.510332992183976 +"town_nullValues","hdb_resale_prices.town","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE ""town"" IS NULL","['hdb_resale_prices']","$.schema[0].properties[1].quality[0]","NullValuesCheckReference",,"True",17.8000419982709 +"flat_type_invalidValues","hdb_resale_prices.flat_type","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT ""flat_type"" IS NULL AND NOT TRY_CAST(""flat_type"" AS TEXT) IN ('1 ROOM', '2 ROOM', '3 ROOM', '4 ROOM', '5 ROOM', 'EXECUTIVE', 'MULTI-GENERATION')","['hdb_resale_prices']","$.schema[0].properties[2].quality[0]","InvalidValuesCheckReference",,"True",75.48116600082722 +"AddressBlockHouseNumber","hdb_resale_prices.block","hdb_resale_prices","sql","FAILED","mustBe","1","0",1,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(block AS TEXT), '^[A-Za-z0-9]{1,10}$')","['hdb_resale_prices']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference",,"False",62.309749991982244 +"floor_area_must_be_less_than_200","hdb_resale_prices.floor_area_sqm","hdb_resale_prices","sql","FAILED","mustBe","12","0",12,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(floor_area_sqm AS BIGINT) >= 200","['hdb_resale_prices']","$.schema[0].properties[6].quality[0]","SQLColumnCheckReference",,"False",65.37941700662486 +"Year","hdb_resale_prices.lease_commence_date","hdb_resale_prices","sql","FAILED","mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE NOT REGEXP_MATCHES(TRY_CAST(lease_commence_date AS TEXT), '^[0-9]{4}$')","['hdb_resale_prices']","$.schema[0].properties[8].quality[0]","SQLColumnCheckReference",,"False",27.749708999181166 +"resale_price_must_not_exceed_2m","hdb_resale_prices.resale_price","hdb_resale_prices","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""hdb_resale_prices"" WHERE TRY_CAST(resale_price AS BIGINT) > 2000000","['hdb_resale_prices']","$.schema[0].properties[10].quality[0]","SQLColumnCheckReference",,"False",25.50208300817758 diff --git a/examples/vowl_demo_HDB_results_summary.json b/examples/vowl_demo_HDB_results_summary.json index 2cc1bb0..26d74fa 100644 --- a/examples/vowl_demo_HDB_results_summary.json +++ b/examples/vowl_demo_HDB_results_summary.json @@ -14,7 +14,7 @@ "use_try_cast": true }, "failed_rows": 18, - "total_execution_time_ms": 332.48562400694937, + "total_execution_time_ms": 418.1032079795841, "success_rate": 80.0, "connection_results": { "hdb_resale_prices": { @@ -30,20 +30,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 15.560625004582107, + "execution_time_ms": 9.511541997198947, "check_path": "$.schema[0].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'month' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "month" }, "target": "hdb_resale_prices.month", "logical_type": "string", @@ -60,20 +66,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.568041005986743, + "execution_time_ms": 15.60741598950699, "check_path": "$.schema[0].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'month' must be valid string", "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE NOT \"month\" IS NULL AND CAST(\"month\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "hdb_resale_prices.month", "logical_type": "string", @@ -90,20 +102,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.94391700765118, + "execution_time_ms": 7.495708996430039, "check_path": "$.schema[0].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "town_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'town' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"town\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "town" }, "target": "hdb_resale_prices.town", "aggregation_type": "count", @@ -119,20 +137,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.528957994305529, + "execution_time_ms": 7.0631249982398, "check_path": "$.schema[0].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "flat_type_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'flat_type' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"flat_type\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "flat_type" }, "target": "hdb_resale_prices.flat_type", "aggregation_type": "count", @@ -148,20 +172,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.354166998993605, + "execution_time_ms": 8.010416000615805, "check_path": "$.schema[0].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "block_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'block' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"block\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "block" }, "target": "hdb_resale_prices.block", "aggregation_type": "count", @@ -177,20 +207,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.871333000366576, + "execution_time_ms": 6.567249991348945, "check_path": "$.schema[0].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "street_name_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'street_name' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"street_name\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "street_name" }, "target": "hdb_resale_prices.street_name", "aggregation_type": "count", @@ -206,20 +242,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.073707995004952, + "execution_time_ms": 7.874708011513576, "check_path": "$.schema[0].properties[5].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "storey_range_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'storey_range' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"storey_range\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "storey_range" }, "target": "hdb_resale_prices.storey_range", "aggregation_type": "count", @@ -235,20 +277,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.138625005609356, + "execution_time_ms": 7.521874998928979, "check_path": "$.schema[0].properties[6].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "floor_area_sqm_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'floor_area_sqm' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"floor_area_sqm\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "floor_area_sqm" }, "target": "hdb_resale_prices.floor_area_sqm", "aggregation_type": "count", @@ -264,20 +312,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.1229590103030205, + "execution_time_ms": 7.332666995353065, "check_path": "$.schema[0].properties[7].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "flat_model_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'flat_model' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"flat_model\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "flat_model" }, "target": "hdb_resale_prices.flat_model", "aggregation_type": "count", @@ -293,20 +347,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.973290998255834, + "execution_time_ms": 8.957708007073961, "check_path": "$.schema[0].properties[8].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "lease_commence_date_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'lease_commence_date' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"lease_commence_date\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "lease_commence_date" }, "target": "hdb_resale_prices.lease_commence_date", "aggregation_type": "count", @@ -322,20 +382,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.421375001082197, + "execution_time_ms": 7.871249996242113, "check_path": "$.schema[0].properties[9].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "remaining_lease_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'remaining_lease' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"remaining_lease\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "remaining_lease" }, "target": "hdb_resale_prices.remaining_lease", "aggregation_type": "count", @@ -351,20 +417,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.602665995364077, + "execution_time_ms": 8.052582998061553, "check_path": "$.schema[0].properties[10].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "resale_price_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'resale_price' must exist in 'hdb_resale_prices'", "query": "SELECT COUNT(*) FROM (SELECT \"resale_price\" FROM \"hdb_resale_prices\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "resale_price" }, "target": "hdb_resale_prices.resale_price", "aggregation_type": "count", @@ -383,14 +455,14 @@ ], "actual_value": 201879, "failed_rows_count": 0, - "execution_time_ms": 11.940374999539927, + "execution_time_ms": 10.504459001822397, "check_path": "$.schema[0].quality[0]", "check_ref_type": "RowCountCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBeBetween", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "rowCount", "mustBeBetween": [ @@ -398,8 +470,20 @@ 30000000 ], "dimension": "completeness", + "tags": [ + "vowl_generated_check" + ], "description": "Total row count for 'hdb_resale_prices'" }, + "contract_definition": { + "type": "library", + "metric": "rowCount", + "mustBeBetween": [ + 0, + 30000000 + ], + "dimension": "completeness" + }, "target": "hdb_resale_prices", "aggregation_type": "count", "tables_in_query": [ @@ -414,13 +498,24 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 67.6931250054622, + "execution_time_ms": 31.510332992183976, "check_path": "$.schema[0].properties[0].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "Month", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "Month", @@ -447,20 +542,29 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 18.517291988246143, + "execution_time_ms": 17.8000419982709, "check_path": "$.schema[0].properties[1].quality[0]", "check_ref_type": "NullValuesCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "nullValues", "mustBe": 0, "dimension": "completeness", + "tags": [ + "vowl_generated_check" + ], "description": "Count of NULL values in 'town' of 'hdb_resale_prices'" }, + "contract_definition": { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness" + }, "target": "hdb_resale_prices.town", "aggregation_type": "count", "tables_in_query": [ @@ -475,14 +579,14 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 30.758624998270534, + "execution_time_ms": 75.48116600082722, "check_path": "$.schema[0].properties[2].quality[0]", "check_ref_type": "InvalidValuesCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "invalidValues", "mustBe": 0, @@ -498,8 +602,28 @@ "MULTI-GENERATION" ] }, + "tags": [ + "vowl_generated_check" + ], "description": "Count of invalid values in 'flat_type' of 'hdb_resale_prices'" }, + "contract_definition": { + "type": "library", + "metric": "invalidValues", + "mustBe": 0, + "dimension": "conformity", + "arguments": { + "validValues": [ + "1 ROOM", + "2 ROOM", + "3 ROOM", + "4 ROOM", + "5 ROOM", + "EXECUTIVE", + "MULTI-GENERATION" + ] + } + }, "target": "hdb_resale_prices.flat_type", "aggregation_type": "count", "tables_in_query": [ @@ -514,13 +638,24 @@ "expected_value": 0, "actual_value": 1, "failed_rows_count": 1, - "execution_time_ms": 28.13729199988302, + "execution_time_ms": 62.309749991982244, "check_path": "$.schema[0].properties[3].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "AddressBlockHouseNumber", + "description": "House number component of a street address | Alphanumeric up to 10 characters (e.g. 12, 535, 358C)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(block AS TEXT) !~ '^[A-Za-z0-9]{1,10}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "AddressBlockHouseNumber", @@ -546,13 +681,25 @@ "expected_value": 0, "actual_value": 12, "failed_rows_count": 12, - "execution_time_ms": 23.854999992181547, + "execution_time_ms": 65.37941700662486, "check_path": "$.schema[0].properties[6].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "name": "floor_area_must_be_less_than_200", + "description": "Validates that floor area must be less than 200", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE floor_area_sqm >= 200", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, "contract_definition": { "name": "floor_area_must_be_less_than_200", "description": "Validates that floor area must be less than 200", @@ -579,13 +726,24 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 26.971458006300963, + "execution_time_ms": 27.749708999181166, "check_path": "$.schema[0].properties[8].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "Year", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM \"hdb_resale_prices\"\nWHERE CAST(lease_commence_date AS TEXT) !~ '^[0-9]{4}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "Year", @@ -611,13 +769,25 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 20.45279199955985, + "execution_time_ms": 25.50208300817758, "check_path": "$.schema[0].properties[10].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "hdb_resale_prices", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "name": "resale_price_must_not_exceed_2m", + "description": "Resale price must not be more than 2 million SGD", + "type": "sql", + "dimension": "conformity", + "severity": "error", + "query": "SELECT COUNT(*) FROM \"hdb_resale_prices\" WHERE resale_price > 2000000", + "mustBe": 0, + "tags": [ + "Business Metadata Enrichment" + ] + }, "contract_definition": { "name": "resale_price_must_not_exceed_2m", "description": "Resale price must not be more than 2 million SGD", diff --git a/examples/vowl_demo_multi_table_results_check_results.csv b/examples/vowl_demo_multi_table_results_check_results.csv index 769b4f4..f73ae8c 100644 --- a/examples/vowl_demo_multi_table_results_check_results.csv +++ b/examples/vowl_demo_multi_table_results_check_results.csv @@ -1,98 +1,77 @@ -"check_name","target","schema_name","engine","type","dimension","description","status","severity","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms","name","query","mustBe","tags","metric","mustBeGreaterThan","arguments" -"employee_id_column_exists_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",8.266415999969468,"employee_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_id_logical_type_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",10.77183299639728,"employee_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND CAST(""employee_id"" AS VARCHAR) IS NULL","0",,,, -"employee_id_logical_type_options_minLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",34.72320800938178,"employee_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) < 7","0",,,, -"employee_id_logical_type_options_maxLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",33.42779100057669,"employee_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) > 7","0",,,, -"employee_id_logical_type_options_pattern_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]{6}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]{6}$')","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",39.862334000645205,"employee_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT CAST(""employee_id"" AS VARCHAR) ~ '^e[0-9]{6}$'","0",,,, -"employee_id_required_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].required","RequiredCheckReference","string","True",7.818917001713999,"employee_id_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","0",,,, -"employee_id_unique_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[0].unique","UniqueCheckReference","string","True",8.558542001992464,"employee_id_unique_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","0",,,, -"employee_id_primary_key_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",6.696457989164628,"employee_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","0",,,, -"payroll_id_column_exists_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference","string","True",6.77716700010933,"payroll_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"payroll_id_logical_type_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND TRY_CAST(""payroll_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].logicalType","LogicalTypeCheckReference","string","True",12.050000004819594,"payroll_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND CAST(""payroll_id"" AS VARCHAR) IS NULL","0",,,, -"payroll_id_logical_type_options_minLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have minimum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) < 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",37.43920900160447,"payroll_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(CAST(""payroll_id"" AS VARCHAR)) < 36","0",,,, -"payroll_id_logical_type_options_maxLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must have maximum length of 36","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) > 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",36.900374994729646,"payroll_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(CAST(""payroll_id"" AS VARCHAR)) > 36","0",,,, -"payroll_id_logical_type_options_pattern_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","conformity","Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""payroll_id"" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",30.695709006977268,"payroll_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT CAST(""payroll_id"" AS VARCHAR) ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'","0",,,, -"payroll_id_required_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","completeness","Column 'payroll_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].required","RequiredCheckReference","string","True",6.23133301269263,"payroll_id_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","0",,,, -"payroll_id_unique_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Column 'payroll_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[1].unique","UniqueCheckReference","string","True",6.7342499969527125,"payroll_id_unique_check","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","0",,,, -"payroll_id_primary_key_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","sql","consistency","Primary key column 'payroll_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[1].primaryKey","PrimaryKeyCheckReference","string","True",5.641625000862405,"payroll_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","0",,,, -"month_column_exists_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference","string","True",6.300708002527244,"month_column_exists_check","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"month_logical_type_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Values in 'month' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].logicalType","LogicalTypeCheckReference","string","True",10.405374996480532,"month_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND CAST(""month"" AS VARCHAR) IS NULL","0",,,, -"month_logical_type_options_pattern_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Column 'month' must match pattern '^\d{4}-\d{2}$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-\d{2}$')","['demo_employee_payroll']","$.schema[0].properties[2].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",25.42725000239443,"month_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT CAST(""month"" AS VARCHAR) ~ '^\d{4}-\d{2}$'","0",,,, -"month_required_check","demo_employee_payroll.month","demo_employee_payroll","sql","sql","completeness","Column 'month' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].required","RequiredCheckReference","string","True",4.294291997211985,"month_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","0",,,, -"payroll_start_dt_column_exists_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_start_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference","string","True",4.076666999026202,"payroll_start_dt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"payroll_start_dt_logical_type_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_start_dt' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND TRY_CAST(""payroll_start_dt"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[3].logicalType","LogicalTypeCheckReference","string","True",6.958707992453128,"payroll_start_dt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND CAST(""payroll_start_dt"" AS VARCHAR) IS NULL","0",,,, -"payroll_end_dt_column_exists_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Column 'payroll_end_dt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference","date","True",4.2034159996546805,"payroll_end_dt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"payroll_end_dt_logical_type_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","Values in 'payroll_end_dt' must be valid date","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND TRY_CAST(""payroll_end_dt"" AS DATE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[4].logicalType","LogicalTypeCheckReference","date","True",7.174208003561944,"payroll_end_dt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND CAST(""payroll_end_dt"" AS DATE) IS NULL","0",,,, -"total_amt_column_exists_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Column 'total_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference","integer","True",4.167917009908706,"total_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"total_amt_logical_type_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt' must be valid integer","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (TRY_CAST(""total_amt"" AS DOUBLE) IS NULL OR TRY_CAST(""total_amt"" AS DOUBLE) <> TRY_CAST(""total_amt"" AS BIGINT))","['demo_employee_payroll']","$.schema[0].properties[5].logicalType","LogicalTypeCheckReference","integer","True",7.325166996452026,"total_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (CAST(""total_amt"" AS DOUBLE PRECISION) IS NULL OR CAST(""total_amt"" AS DOUBLE PRECISION) <> CAST(""total_amt"" AS BIGINT))","0",,,, -"employer_cpf_amt_column_exists_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference","number","True",4.097916011232883,"employer_cpf_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employer_cpf_amt_logical_type_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employer_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND TRY_CAST(""employer_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[6].logicalType","LogicalTypeCheckReference","number","True",8.272374994703569,"employer_cpf_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND CAST(""employer_cpf_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, -"total_amt_employee_column_exists_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Column 'total_amt_employee' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference","number","True",5.967875011265278,"total_amt_employee_column_exists_check","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"total_amt_employee_logical_type_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","conformity","Values in 'total_amt_employee' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND TRY_CAST(""total_amt_employee"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].logicalType","LogicalTypeCheckReference","number","True",39.624583994736895,"total_amt_employee_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND CAST(""total_amt_employee"" AS DOUBLE PRECISION) IS NULL","0",,,, -"total_amt_employee_required_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","sql","completeness","Column 'total_amt_employee' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].required","RequiredCheckReference","number","True",5.84237500152085,"total_amt_employee_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","0",,,, -"employee_cpf_amt_column_exists_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference","number","True",6.1369589966489,"employee_cpf_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_cpf_amt_logical_type_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_cpf_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND TRY_CAST(""employee_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].logicalType","LogicalTypeCheckReference","number","True",9.304834005888551,"employee_cpf_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND CAST(""employee_cpf_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, -"employee_cpf_amt_required_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_cpf_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].required","RequiredCheckReference","number","True",5.339750001439825,"employee_cpf_amt_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","0",,,, -"employee_gross_amt_column_exists_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Column 'employee_gross_amt' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference","number","True",8.089125010883436,"employee_gross_amt_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_gross_amt_logical_type_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","conformity","Values in 'employee_gross_amt' must be valid number","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND TRY_CAST(""employee_gross_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].logicalType","LogicalTypeCheckReference","number","True",12.091583994333632,"employee_gross_amt_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND CAST(""employee_gross_amt"" AS DOUBLE PRECISION) IS NULL","0",,,, -"employee_gross_amt_required_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","sql","completeness","Column 'employee_gross_amt' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].required","RequiredCheckReference","number","True",7.5386660028016195,"employee_gross_amt_required_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","0",,,, -"phone_number_column_exists_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference","string","True",8.80279199918732,"phone_number_column_exists_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","0",,,, -"phone_number_logical_type_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[10].logicalType","LogicalTypeCheckReference","string","True",14.585208002245054,"phone_number_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND CAST(""phone_number"" AS VARCHAR) IS NULL","0",,,, -"phone_number_logical_type_options_minLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have minimum length of 8","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) < 8","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",32.70300000440329,"phone_number_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(CAST(""phone_number"" AS VARCHAR)) < 8","0",,,, -"phone_number_logical_type_options_maxLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must have maximum length of 11","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) > 11","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",34.6164999937173,"phone_number_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(CAST(""phone_number"" AS VARCHAR)) > 11","0",,,, -"phone_number_logical_type_options_pattern_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","sql","conformity","Column 'phone_number' must match pattern '^\+65[0-9]{8}$'","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+65[0-9]{8}$')","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",38.351875002263114,"phone_number_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT CAST(""phone_number"" AS VARCHAR) ~ '^\+65[0-9]{8}$'","0",,,, -"total_amt_employee_equals_gross_plus_cpf","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)","['demo_employee_payroll']","$.schema[0].quality[0]","SQLTableCheckReference",,"False",23.633499993593432,"total_amt_employee_equals_gross_plus_cpf","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee != (employee_gross_amt + employee_cpf_amt)","0","['User-Defined Quality Rule']",,, -"demo_employee_payroll_rowCount","demo_employee_payroll","demo_employee_payroll","sql","library","completeness","Total row count for 'demo_employee_payroll'","PASSED",,"mustBeGreaterThan","3","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll""","['demo_employee_payroll']","$.schema[0].quality[3]","RowCountCheckReference",,"True",12.046417003148235,,,,,"rowCount","0", -"employee_id_nullValues","demo_employee_payroll.employee_id","demo_employee_payroll","sql","library","completeness","Count of NULL values in 'employee_id' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].quality[0]","NullValuesCheckReference","string","True",17.949457993381657,,,"0",,"nullValues",, -"Month","demo_employee_payroll.month","demo_employee_payroll","sql","sql","conformity","Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[0]","SQLColumnCheckReference","string","False",23.35858299920801,"Month","SELECT COUNT(*) -FROM demo_employee_payroll -WHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';","0","['SG-DRM v5.0']",,, -"month_invalidValues","demo_employee_payroll.month","demo_employee_payroll","sql","library","conformity","Count of invalid values in 'month' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[1]","InvalidValuesCheckReference","string","True",25.47229200717993,,,"0",,"invalidValues",,"{'pattern': '^\\d{4}-(0[1-9]|1[0-2])$'}" -"Date With Timezone","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference","string","False",23.1642079888843,"Date With Timezone","SELECT COUNT(*) -FROM demo_employee_payroll -WHERE CAST(payroll_start_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$' - OR LENGTH(CAST(payroll_start_dt AS TEXT)) > 22;","0","['SG-DRM v5.0']",,, -"Date With Timezone","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","sql","conformity","If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2","FAILED",,"mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[4].quality[0]","SQLColumnCheckReference","date","False",22.935790999326855,"Date With Timezone","SELECT COUNT(*) -FROM demo_employee_payroll -WHERE CAST(payroll_end_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$' - OR LENGTH(CAST(payroll_end_dt AS TEXT)) > 22;","0","['SG-DRM v5.0']",,, -"total_amt_equals_employer_cpf_plus_total_amt_employee","demo_employee_payroll.total_amt","demo_employee_payroll","sql","sql","consistency","Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency","PASSED","error","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","['demo_employee_payroll']","$.schema[0].properties[5].quality[0]","SQLColumnCheckReference","integer","False",17.85162500164006,"total_amt_equals_employer_cpf_plus_total_amt_employee","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt != (employer_cpf_amt + total_amt_employee) OR (total_amt IS NOT NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","0","['User-Defined Quality Rule']",,, -"total_amt_missingValues","demo_employee_payroll.total_amt","demo_employee_payroll","sql","library","completeness","Count of missing values in 'total_amt' of 'demo_employee_payroll'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt"" IS NULL OR TRY_CAST(""total_amt"" AS TEXT) IN ('')","['demo_employee_payroll']","$.schema[0].properties[5].quality[1]","MissingValuesCheckReference","integer","True",22.719083004631102,,,"0",,"missingValues",,"{'missingValues': [None, '']}" -"employee_id_column_exists_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[0].name","DeclaredColumnExistsCheckReference","string","True",5.673040999681689,"employee_id_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_id_logical_type_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Values in 'employee_id' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[0].logicalType","LogicalTypeCheckReference","string","True",8.802083000773564,"employee_id_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND CAST(""employee_id"" AS VARCHAR) IS NULL","0",,,, -"employee_id_logical_type_options_pattern_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must match pattern '^e[0-9]+$'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]+$')","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",27.994290998321958,"employee_id_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT CAST(""employee_id"" AS VARCHAR) ~ '^e[0-9]+$'","0",,,, -"employee_id_logical_type_options_minLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have minimum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",27.982749990769662,"employee_id_logical_type_options_minLength_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) < 7","0",,,, -"employee_id_logical_type_options_maxLength_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","conformity","Column 'employee_id' must have maximum length of 7","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",28.017333999741822,"employee_id_logical_type_options_maxLength_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(CAST(""employee_id"" AS VARCHAR)) > 7","0",,,, -"employee_id_required_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","completeness","Column 'employee_id' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","['demo_employee_list']","$.schema[1].properties[0].required","RequiredCheckReference","string","True",5.478375009261072,"employee_id_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","0",,,, -"employee_id_unique_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Column 'employee_id' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[0].unique","UniqueCheckReference","string","True",7.041749995551072,"employee_id_unique_check","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","0",,,, -"employee_id_primary_key_check","demo_employee_list.employee_id","demo_employee_list","sql","sql","consistency","Primary key column 'employee_id' must be unique and not null","PASSED",,"mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_list']","$.schema[1].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",5.410500001744367,"employee_id_primary_key_check","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","0",,,, -"employee_last_name_column_exists_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Column 'employee_last_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[1].name","DeclaredColumnExistsCheckReference","string","True",6.55170799291227,"employee_last_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_last_name_logical_type_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","Values in 'employee_last_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND TRY_CAST(""employee_last_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[1].logicalType","LogicalTypeCheckReference","string","True",10.189666994847357,"employee_last_name_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND CAST(""employee_last_name"" AS VARCHAR) IS NULL","0",,,, -"employee_last_name_required_check","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","completeness","Column 'employee_last_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[1].required","RequiredCheckReference","string","True",6.203083001310006,"employee_last_name_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","0",,,, -"employee_first_name_column_exists_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Column 'employee_first_name' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[2].name","DeclaredColumnExistsCheckReference","string","True",6.034209000063129,"employee_first_name_column_exists_check","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, -"employee_first_name_logical_type_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","Values in 'employee_first_name' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND TRY_CAST(""employee_first_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[2].logicalType","LogicalTypeCheckReference","string","True",10.555083004874177,"employee_first_name_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND CAST(""employee_first_name"" AS VARCHAR) IS NULL","0",,,, -"employee_first_name_required_check","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","completeness","Column 'employee_first_name' must not contain NULL values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[2].required","RequiredCheckReference","string","True",6.4019589917734265,"employee_first_name_required_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","0",,,, -"current_title_column_exists_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Column 'current_title' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[3].name","DeclaredColumnExistsCheckReference","string","True",6.396415992639959,"current_title_column_exists_check","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, -"current_title_logical_type_check","demo_employee_list.current_title","demo_employee_list","sql","sql","conformity","Values in 'current_title' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND TRY_CAST(""current_title"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[3].logicalType","LogicalTypeCheckReference","string","True",10.182790996623226,"current_title_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND CAST(""current_title"" AS VARCHAR) IS NULL","0",,,, -"phone_number_column_exists_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must exist in 'demo_employee_list'","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[4].name","DeclaredColumnExistsCheckReference","string","True",6.789667007979006,"phone_number_column_exists_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","0",,,, -"phone_number_logical_type_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Values in 'phone_number' must be valid string","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[4].logicalType","LogicalTypeCheckReference","string","True",10.848208010429516,"phone_number_logical_type_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND CAST(""phone_number"" AS VARCHAR) IS NULL","0",,,, -"phone_number_logical_type_options_pattern_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Column 'phone_number' must match pattern '^\+[0-9]+$'","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+[0-9]+$')","['demo_employee_list']","$.schema[1].properties[4].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",32.44387501035817,"phone_number_logical_type_options_pattern_check","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT CAST(""phone_number"" AS VARCHAR) ~ '^\+[0-9]+$'","0",,,, -"phone_number_unique_check","demo_employee_list.phone_number","demo_employee_list","sql","sql","consistency","Column 'phone_number' must contain unique values","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[4].unique","UniqueCheckReference","string","True",6.953083007829264,"phone_number_unique_check","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","0",,,, -"PersonName","demo_employee_list.employee_last_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[1].quality[0]","SQLColumnCheckReference","string","False",22.425041999667883,"PersonName","SELECT COUNT(*) -FROM demo_employee_list -WHERE employee_last_name IS NOT NULL - AND LENGTH(CAST(employee_last_name AS TEXT)) > 66;","0","['SG-DRM v5.0']",,, -"PersonName","demo_employee_list.employee_first_name","demo_employee_list","sql","sql","conformity","PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[2].quality[0]","SQLColumnCheckReference","string","False",18.33512500161305,"PersonName","SELECT COUNT(*) -FROM demo_employee_list -WHERE employee_first_name IS NOT NULL - AND LENGTH(CAST(employee_first_name AS TEXT)) > 66;","0","['SG-DRM v5.0']",,, -"TelephoneNumber","demo_employee_list.phone_number","demo_employee_list","sql","sql","conformity","Based on ITU T E.164 | e.g. 095236","PASSED",,"mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')","['demo_employee_list']","$.schema[1].properties[4].quality[0]","SQLColumnCheckReference","string","False",18.17870800732635,"TelephoneNumber","SELECT COUNT(*) -FROM demo_employee_list -WHERE CAST(phone_number AS TEXT) !~ '^[0-9]{1,12}$';","0","['SG-DRM v5.0']",,, -"employee_id_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.","FAILED",,"mustBe","1","0",1,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[1]","SQLTableCheckReference",,"False",62.502582994056866,"employee_id_exists_in_master_list","SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref - ON payroll.employee_id = ref.employee_id -WHERE ref.employee_id IS NULL","0","['Cross-Table Validation']",,, -"phone_number_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","sql","consistency","Ensures the phone numbers recorded in payroll matches an entry in the master employee list.","FAILED",,"mustBe","2","0",2,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[2]","SQLTableCheckReference",,"False",18.972791003761813,"phone_number_exists_in_master_list","SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref - ON payroll.phone_number = ref.phone_number -WHERE payroll.phone_number IS NOT NULL - AND ref.phone_number IS NULL","0","['Cross-Table Validation']",,, +"check_name","target","schema_name","engine","status","operator","actual_value","expected_value","failed_rows_count","aggregation_type","message","rendered_implementation","tables_in_query","check_path","check_ref_type","logical_type","is_generated","execution_time_ms" +"employee_id_column_exists_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[0].name","DeclaredColumnExistsCheckReference","string","True",7.536083998274989 +"employee_id_logical_type_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].logicalType","LogicalTypeCheckReference","string","True",11.358792005921714 +"employee_id_logical_type_options_minLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",33.94816699437797 +"employee_id_logical_type_options_maxLength_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",33.068207994801924 +"employee_id_logical_type_options_pattern_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]{6}$')","['demo_employee_payroll']","$.schema[0].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",29.56283300591167 +"employee_id_required_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].required","RequiredCheckReference","string","True",5.534250012715347 +"employee_id_unique_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[0].unique","UniqueCheckReference","string","True",6.91154100059066 +"employee_id_primary_key_check","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_payroll"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",6.103708001319319 +"payroll_id_column_exists_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[1].name","DeclaredColumnExistsCheckReference","string","True",5.827750006574206 +"payroll_id_logical_type_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND TRY_CAST(""payroll_id"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].logicalType","LogicalTypeCheckReference","string","True",9.204667003359646 +"payroll_id_logical_type_options_minLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) < 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",28.868458000943065 +"payroll_id_logical_type_options_maxLength_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND LENGTH(TRY_CAST(""payroll_id"" AS TEXT)) > 36","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",30.009207999682985 +"payroll_id_logical_type_options_pattern_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""payroll_id"" AS TEXT), '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$')","['demo_employee_payroll']","$.schema[0].properties[1].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",34.06583299511112 +"payroll_id_required_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[1].required","RequiredCheckReference","string","True",33.69458299130201 +"payroll_id_unique_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)","['demo_employee_payroll']","$.schema[0].properties[1].unique","UniqueCheckReference","string","True",11.5525409928523 +"payroll_id_primary_key_check","demo_employee_payroll.payroll_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""payroll_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""payroll_id"" FROM ""demo_employee_payroll"" WHERE NOT ""payroll_id"" IS NULL GROUP BY ""payroll_id"" HAVING COUNT(*) > 1)))","['demo_employee_payroll']","$.schema[0].properties[1].primaryKey","PrimaryKeyCheckReference","string","True",9.977291003451683 +"month_column_exists_check","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""month"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[2].name","DeclaredColumnExistsCheckReference","string","True",11.00429099460598 +"month_logical_type_check","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND TRY_CAST(""month"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].logicalType","LogicalTypeCheckReference","string","True",14.883666997775435 +"month_logical_type_options_pattern_check","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-\d{2}$')","['demo_employee_payroll']","$.schema[0].properties[2].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",50.78175000380725 +"month_required_check","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""month"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[2].required","RequiredCheckReference","string","True",14.663374997326173 +"payroll_start_dt_column_exists_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_start_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[3].name","DeclaredColumnExistsCheckReference","string","True",7.284459003130905 +"payroll_start_dt_logical_type_check","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_start_dt"" IS NULL AND TRY_CAST(""payroll_start_dt"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[3].logicalType","LogicalTypeCheckReference","string","True",8.672416995977983 +"payroll_end_dt_column_exists_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""payroll_end_dt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[4].name","DeclaredColumnExistsCheckReference","date","True",5.804250002256595 +"payroll_end_dt_logical_type_check","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""payroll_end_dt"" IS NULL AND TRY_CAST(""payroll_end_dt"" AS DATE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[4].logicalType","LogicalTypeCheckReference","date","True",9.189958000206389 +"total_amt_column_exists_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[5].name","DeclaredColumnExistsCheckReference","integer","True",4.887916991719976 +"total_amt_logical_type_check","demo_employee_payroll.total_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt"" IS NULL AND (TRY_CAST(""total_amt"" AS DOUBLE) IS NULL OR TRY_CAST(""total_amt"" AS DOUBLE) <> TRY_CAST(""total_amt"" AS BIGINT))","['demo_employee_payroll']","$.schema[0].properties[5].logicalType","LogicalTypeCheckReference","integer","True",39.02799999923445 +"employer_cpf_amt_column_exists_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employer_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[6].name","DeclaredColumnExistsCheckReference","number","True",5.7217919966205955 +"employer_cpf_amt_logical_type_check","demo_employee_payroll.employer_cpf_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employer_cpf_amt"" IS NULL AND TRY_CAST(""employer_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[6].logicalType","LogicalTypeCheckReference","number","True",9.278917001211084 +"total_amt_employee_column_exists_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""total_amt_employee"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[7].name","DeclaredColumnExistsCheckReference","number","True",12.201416000607423 +"total_amt_employee_logical_type_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""total_amt_employee"" IS NULL AND TRY_CAST(""total_amt_employee"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].logicalType","LogicalTypeCheckReference","number","True",11.42637500015553 +"total_amt_employee_required_check","demo_employee_payroll.total_amt_employee","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt_employee"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[7].required","RequiredCheckReference","number","True",6.754499991075136 +"employee_cpf_amt_column_exists_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_cpf_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[8].name","DeclaredColumnExistsCheckReference","number","True",7.811667004716583 +"employee_cpf_amt_logical_type_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_cpf_amt"" IS NULL AND TRY_CAST(""employee_cpf_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].logicalType","LogicalTypeCheckReference","number","True",10.776291994261555 +"employee_cpf_amt_required_check","demo_employee_payroll.employee_cpf_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_cpf_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[8].required","RequiredCheckReference","number","True",7.453291007550433 +"employee_gross_amt_column_exists_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_gross_amt"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[9].name","DeclaredColumnExistsCheckReference","number","True",6.992291993810795 +"employee_gross_amt_logical_type_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""employee_gross_amt"" IS NULL AND TRY_CAST(""employee_gross_amt"" AS DOUBLE) IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].logicalType","LogicalTypeCheckReference","number","True",11.974291992373765 +"employee_gross_amt_required_check","demo_employee_payroll.employee_gross_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_gross_amt"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[9].required","RequiredCheckReference","number","True",6.4171249978244305 +"phone_number_column_exists_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_payroll"" LIMIT 0) AS _vowl_column_exists","['demo_employee_payroll']","$.schema[0].properties[10].name","DeclaredColumnExistsCheckReference","string","True",9.263290994567797 +"phone_number_logical_type_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_payroll']","$.schema[0].properties[10].logicalType","LogicalTypeCheckReference","string","True",17.175583998323418 +"phone_number_logical_type_options_minLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) < 8","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",43.42120799992699 +"phone_number_logical_type_options_maxLength_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND LENGTH(TRY_CAST(""phone_number"" AS TEXT)) > 11","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",45.03570799715817 +"phone_number_logical_type_options_pattern_check","demo_employee_payroll.phone_number","demo_employee_payroll","sql","FAILED","mustBe","3","0",3,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+65[0-9]{8}$')","['demo_employee_payroll']","$.schema[0].properties[10].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",43.62716700416058 +"total_amt_employee_equals_gross_plus_cpf","demo_employee_payroll","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)","['demo_employee_payroll']","$.schema[0].quality[0]","SQLTableCheckReference",,"False",21.877167004277 +"demo_employee_payroll_rowCount","demo_employee_payroll","demo_employee_payroll","sql","PASSED","mustBeGreaterThan","3","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll""","['demo_employee_payroll']","$.schema[0].quality[3]","RowCountCheckReference",,"True",42.15633300191257 +"employee_id_nullValues","demo_employee_payroll.employee_id","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""employee_id"" IS NULL","['demo_employee_payroll']","$.schema[0].properties[0].quality[0]","NullValuesCheckReference","string","True",27.207833001739345 +"Month","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(month AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[0]","SQLColumnCheckReference","string","False",32.52749999228399 +"month_invalidValues","demo_employee_payroll.month","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE NOT ""month"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""month"" AS TEXT), '^\d{4}-(0[1-9]|1[0-2])$')","['demo_employee_payroll']","$.schema[0].properties[2].quality[1]","InvalidValuesCheckReference","string","True",26.39941700908821 +"Date With Timezone","demo_employee_payroll.payroll_start_dt","demo_employee_payroll","sql","FAILED","mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_start_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_start_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[3].quality[0]","SQLColumnCheckReference","string","False",21.720541000831872 +"Date With Timezone","demo_employee_payroll.payroll_end_dt","demo_employee_payroll","sql","FAILED","mustBe","3","0",3,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE NOT REGEXP_MATCHES(TRY_CAST(payroll_end_dt AS TEXT), '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$') OR LENGTH(TRY_CAST(payroll_end_dt AS TEXT)) > 22","['demo_employee_payroll']","$.schema[0].properties[4].quality[0]","SQLColumnCheckReference","date","False",21.855708007933572 +"total_amt_equals_employer_cpf_plus_total_amt_employee","demo_employee_payroll.total_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt <> (employer_cpf_amt + total_amt_employee) OR (NOT total_amt IS NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))","['demo_employee_payroll']","$.schema[0].properties[5].quality[0]","SQLColumnCheckReference","integer","False",18.529374996433035 +"total_amt_missingValues","demo_employee_payroll.total_amt","demo_employee_payroll","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_payroll"" WHERE ""total_amt"" IS NULL OR TRY_CAST(""total_amt"" AS TEXT) IN ('')","['demo_employee_payroll']","$.schema[0].properties[5].quality[1]","MissingValuesCheckReference","integer","True",33.28112499730196 +"employee_id_column_exists_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[0].name","DeclaredColumnExistsCheckReference","string","True",10.498125004232861 +"employee_id_logical_type_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND TRY_CAST(""employee_id"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[0].logicalType","LogicalTypeCheckReference","string","True",20.03816599608399 +"employee_id_logical_type_options_pattern_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""employee_id"" AS TEXT), '^e[0-9]+$')","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",42.80774999642745 +"employee_id_logical_type_options_minLength_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) < 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.minLength","LogicalTypeOptionsCheckReference","string","True",48.060291999718174 +"employee_id_logical_type_options_maxLength_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL AND LENGTH(TRY_CAST(""employee_id"" AS TEXT)) > 7","['demo_employee_list']","$.schema[1].properties[0].logicalTypeOptions.maxLength","LogicalTypeOptionsCheckReference","string","True",34.422624987200834 +"employee_id_required_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL","['demo_employee_list']","$.schema[1].properties[0].required","RequiredCheckReference","string","True",8.225458004744723 +"employee_id_unique_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[0].unique","UniqueCheckReference","string","True",8.616082996013574 +"employee_id_primary_key_check","demo_employee_list.employee_id","demo_employee_list","sql","PASSED","mustBe","0","0",0,"custom","","SELECT ((SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_id"" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT ""employee_id"" FROM ""demo_employee_list"" WHERE NOT ""employee_id"" IS NULL GROUP BY ""employee_id"" HAVING COUNT(*) > 1)))","['demo_employee_list']","$.schema[1].properties[0].primaryKey","PrimaryKeyCheckReference","string","True",6.8277919926913455 +"employee_last_name_column_exists_check","demo_employee_list.employee_last_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_last_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[1].name","DeclaredColumnExistsCheckReference","string","True",17.24595800624229 +"employee_last_name_logical_type_check","demo_employee_list.employee_last_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_last_name"" IS NULL AND TRY_CAST(""employee_last_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[1].logicalType","LogicalTypeCheckReference","string","True",14.31154199235607 +"employee_last_name_required_check","demo_employee_list.employee_last_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_last_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[1].required","RequiredCheckReference","string","True",6.7928750067949295 +"employee_first_name_column_exists_check","demo_employee_list.employee_first_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""employee_first_name"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[2].name","DeclaredColumnExistsCheckReference","string","True",7.681333998334594 +"employee_first_name_logical_type_check","demo_employee_list.employee_first_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""employee_first_name"" IS NULL AND TRY_CAST(""employee_first_name"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[2].logicalType","LogicalTypeCheckReference","string","True",11.540167004568502 +"employee_first_name_required_check","demo_employee_list.employee_first_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE ""employee_first_name"" IS NULL","['demo_employee_list']","$.schema[1].properties[2].required","RequiredCheckReference","string","True",7.362165997619741 +"current_title_column_exists_check","demo_employee_list.current_title","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""current_title"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[3].name","DeclaredColumnExistsCheckReference","string","True",7.876791001763195 +"current_title_logical_type_check","demo_employee_list.current_title","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""current_title"" IS NULL AND TRY_CAST(""current_title"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[3].logicalType","LogicalTypeCheckReference","string","True",12.056583000230603 +"phone_number_column_exists_check","demo_employee_list.phone_number","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" LIMIT 0) AS _vowl_column_exists","['demo_employee_list']","$.schema[1].properties[4].name","DeclaredColumnExistsCheckReference","string","True",13.484708993928507 +"phone_number_logical_type_check","demo_employee_list.phone_number","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND TRY_CAST(""phone_number"" AS TEXT) IS NULL","['demo_employee_list']","$.schema[1].properties[4].logicalType","LogicalTypeCheckReference","string","True",11.268125002970919 +"phone_number_logical_type_options_pattern_check","demo_employee_list.phone_number","demo_employee_list","sql","FAILED","mustBe","2","0",2,"count","","SELECT COUNT(*) FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL AND NOT REGEXP_MATCHES(TRY_CAST(""phone_number"" AS TEXT), '^\+[0-9]+$')","['demo_employee_list']","$.schema[1].properties[4].logicalTypeOptions.pattern","LogicalTypeOptionsCheckReference","string","True",32.583833002718166 +"phone_number_unique_check","demo_employee_list.phone_number","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM (SELECT ""phone_number"" FROM ""demo_employee_list"" WHERE NOT ""phone_number"" IS NULL GROUP BY ""phone_number"" HAVING COUNT(*) > 1)","['demo_employee_list']","$.schema[1].properties[4].unique","UniqueCheckReference","string","True",6.587625000975095 +"PersonName","demo_employee_list.employee_last_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_last_name IS NULL AND LENGTH(TRY_CAST(employee_last_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[1].quality[0]","SQLColumnCheckReference","string","False",21.21066600375343 +"PersonName","demo_employee_list.employee_first_name","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT employee_first_name IS NULL AND LENGTH(TRY_CAST(employee_first_name AS TEXT)) > 66","['demo_employee_list']","$.schema[1].properties[2].quality[0]","SQLColumnCheckReference","string","False",25.908499999786727 +"TelephoneNumber","demo_employee_list.phone_number","demo_employee_list","sql","PASSED","mustBe","0","0",0,"count","","SELECT COUNT(*) FROM demo_employee_list WHERE NOT REGEXP_MATCHES(TRY_CAST(phone_number AS TEXT), '^[0-9]{1,12}$')","['demo_employee_list']","$.schema[1].properties[4].quality[0]","SQLColumnCheckReference","string","False",23.390207992633805 +"employee_id_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","FAILED","mustBe","1","0",1,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.employee_id = ref.employee_id WHERE ref.employee_id IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[1]","SQLTableCheckReference",,"False",121.92870900616981 +"phone_number_exists_in_master_list","demo_employee_payroll","demo_employee_payroll","sql","FAILED","mustBe","2","0",2,"count","","SELECT COUNT(*) FROM demo_employee_payroll AS payroll LEFT JOIN demo_employee_list AS ref ON payroll.phone_number = ref.phone_number WHERE NOT payroll.phone_number IS NULL AND ref.phone_number IS NULL","['demo_employee_list', 'demo_employee_payroll']","$.schema[0].quality[2]","SQLTableCheckReference",,"False",29.520874988520518 diff --git a/examples/vowl_demo_multi_table_results_summary.json b/examples/vowl_demo_multi_table_results_summary.json index 055c097..6d78d8f 100644 --- a/examples/vowl_demo_multi_table_results_summary.json +++ b/examples/vowl_demo_multi_table_results_summary.json @@ -15,7 +15,7 @@ "use_try_cast": true }, "failed_rows": 14, - "total_execution_time_ms": 1179.7893720504362, + "total_execution_time_ms": 1514.5568709558574, "success_rate": 92.10526315789474, "connection_results": { "demo_employee_payroll": { @@ -35,20 +35,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.266415999969468, + "execution_time_ms": 7.536083998274989, "check_path": "$.schema[0].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_id" }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -65,20 +71,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.77183299639728, + "execution_time_ms": 11.358792005921714, "check_path": "$.schema[0].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_id' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND CAST(\"employee_id\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -95,20 +107,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 34.72320800938178, + "execution_time_ms": 33.94816699437797, "check_path": "$.schema[0].properties[0].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_minLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must have minimum length of 7", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) < 7", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 7 }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -125,20 +143,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 33.42779100057669, + "execution_time_ms": 33.068207994801924, "check_path": "$.schema[0].properties[0].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_maxLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must have maximum length of 7", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) > 7", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 7 }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -155,20 +179,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 39.862334000645205, + "execution_time_ms": 29.56283300591167, "check_path": "$.schema[0].properties[0].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must match pattern '^e[0-9]{6}$'", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL AND NOT CAST(\"employee_id\" AS VARCHAR) ~ '^e[0-9]{6}$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^e[0-9]{6}$" }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -185,20 +215,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.818917001713999, + "execution_time_ms": 5.534250012715347, "check_path": "$.schema[0].properties[0].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_id' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -215,20 +251,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.558542001992464, + "execution_time_ms": 6.91154100059066, "check_path": "$.schema[0].properties[0].unique", "check_ref_type": "UniqueCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_unique_check", "type": "sql", "dimension": "consistency", "description": "Column 'employee_id' must contain unique values", "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -245,20 +287,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.696457989164628, + "execution_time_ms": 6.103708001319319, "check_path": "$.schema[0].properties[0].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_primary_key_check", "type": "sql", "dimension": "consistency", "description": "Primary key column 'employee_id' must be unique and not null", "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_payroll\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", @@ -275,20 +323,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.77716700010933, + "execution_time_ms": 5.827750006574206, "check_path": "$.schema[0].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_id' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "payroll_id" }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -305,20 +359,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 12.050000004819594, + "execution_time_ms": 9.204667003359646, "check_path": "$.schema[0].properties[1].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'payroll_id' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND CAST(\"payroll_id\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -335,20 +395,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 37.43920900160447, + "execution_time_ms": 28.868458000943065, "check_path": "$.schema[0].properties[1].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_logical_type_options_minLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_id' must have minimum length of 36", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(CAST(\"payroll_id\" AS VARCHAR)) < 36", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 36 }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -365,20 +431,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 36.900374994729646, + "execution_time_ms": 30.009207999682985, "check_path": "$.schema[0].properties[1].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_logical_type_options_maxLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_id' must have maximum length of 36", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND LENGTH(CAST(\"payroll_id\" AS VARCHAR)) > 36", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 36 }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -395,20 +467,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 30.695709006977268, + "execution_time_ms": 34.06583299511112, "check_path": "$.schema[0].properties[1].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_id' must match pattern '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL AND NOT CAST(\"payroll_id\" AS VARCHAR) ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -425,20 +503,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.23133301269263, + "execution_time_ms": 33.69458299130201, "check_path": "$.schema[0].properties[1].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'payroll_id' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -455,20 +539,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.7342499969527125, + "execution_time_ms": 11.5525409928523, "check_path": "$.schema[0].properties[1].unique", "check_ref_type": "UniqueCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_unique_check", "type": "sql", "dimension": "consistency", "description": "Column 'payroll_id' must contain unique values", "query": "SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -485,20 +575,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.641625000862405, + "execution_time_ms": 9.977291003451683, "check_path": "$.schema[0].properties[1].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_id_primary_key_check", "type": "sql", "dimension": "consistency", "description": "Primary key column 'payroll_id' must be unique and not null", "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"payroll_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"payroll_id\" FROM \"demo_employee_payroll\" WHERE NOT \"payroll_id\" IS NULL GROUP BY \"payroll_id\" HAVING COUNT(*) > 1)))", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.payroll_id", "logical_type": "string", @@ -515,20 +611,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.300708002527244, + "execution_time_ms": 11.00429099460598, "check_path": "$.schema[0].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'month' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"month\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "month" }, "target": "demo_employee_payroll.month", "logical_type": "string", @@ -545,20 +647,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.405374996480532, + "execution_time_ms": 14.883666997775435, "check_path": "$.schema[0].properties[2].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'month' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND CAST(\"month\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_payroll.month", "logical_type": "string", @@ -575,20 +683,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.42725000239443, + "execution_time_ms": 50.78175000380725, "check_path": "$.schema[0].properties[2].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'month' must match pattern '^\\d{4}-\\d{2}$'", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"month\" IS NULL AND NOT CAST(\"month\" AS VARCHAR) ~ '^\\d{4}-\\d{2}$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^\\d{4}-\\d{2}$" }, "target": "demo_employee_payroll.month", "logical_type": "string", @@ -605,20 +719,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.294291997211985, + "execution_time_ms": 14.663374997326173, "check_path": "$.schema[0].properties[2].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "month_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'month' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"month\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.month", "logical_type": "string", @@ -635,20 +755,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.076666999026202, + "execution_time_ms": 7.284459003130905, "check_path": "$.schema[0].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_start_dt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_start_dt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"payroll_start_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "payroll_start_dt" }, "target": "demo_employee_payroll.payroll_start_dt", "logical_type": "string", @@ -665,20 +791,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.958707992453128, + "execution_time_ms": 8.672416995977983, "check_path": "$.schema[0].properties[3].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_start_dt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'payroll_start_dt' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_start_dt\" IS NULL AND CAST(\"payroll_start_dt\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_payroll.payroll_start_dt", "logical_type": "string", @@ -695,20 +827,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.2034159996546805, + "execution_time_ms": 5.804250002256595, "check_path": "$.schema[0].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_end_dt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'payroll_end_dt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"payroll_end_dt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "payroll_end_dt" }, "target": "demo_employee_payroll.payroll_end_dt", "logical_type": "date", @@ -725,20 +863,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.174208003561944, + "execution_time_ms": 9.189958000206389, "check_path": "$.schema[0].properties[4].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "payroll_end_dt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'payroll_end_dt' must be valid date", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"payroll_end_dt\" IS NULL AND CAST(\"payroll_end_dt\" AS DATE) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "date" }, "target": "demo_employee_payroll.payroll_end_dt", "logical_type": "date", @@ -755,20 +899,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.167917009908706, + "execution_time_ms": 4.887916991719976, "check_path": "$.schema[0].properties[5].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'total_amt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"total_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "total_amt" }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", @@ -785,20 +935,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.325166996452026, + "execution_time_ms": 39.02799999923445, "check_path": "$.schema[0].properties[5].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'total_amt' must be valid integer", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt\" IS NULL AND (CAST(\"total_amt\" AS DOUBLE PRECISION) IS NULL OR CAST(\"total_amt\" AS DOUBLE PRECISION) <> CAST(\"total_amt\" AS BIGINT))", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "integer" }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", @@ -815,20 +971,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 4.097916011232883, + "execution_time_ms": 5.7217919966205955, "check_path": "$.schema[0].properties[6].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employer_cpf_amt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employer_cpf_amt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"employer_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employer_cpf_amt" }, "target": "demo_employee_payroll.employer_cpf_amt", "logical_type": "number", @@ -845,20 +1007,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.272374994703569, + "execution_time_ms": 9.278917001211084, "check_path": "$.schema[0].properties[6].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employer_cpf_amt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employer_cpf_amt' must be valid number", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employer_cpf_amt\" IS NULL AND CAST(\"employer_cpf_amt\" AS DOUBLE PRECISION) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "number" }, "target": "demo_employee_payroll.employer_cpf_amt", "logical_type": "number", @@ -875,20 +1043,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.967875011265278, + "execution_time_ms": 12.201416000607423, "check_path": "$.schema[0].properties[7].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_employee_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'total_amt_employee' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"total_amt_employee\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "total_amt_employee" }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", @@ -905,20 +1079,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 39.624583994736895, + "execution_time_ms": 11.42637500015553, "check_path": "$.schema[0].properties[7].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_employee_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'total_amt_employee' must be valid number", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"total_amt_employee\" IS NULL AND CAST(\"total_amt_employee\" AS DOUBLE PRECISION) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "number" }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", @@ -935,20 +1115,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.84237500152085, + "execution_time_ms": 6.754499991075136, "check_path": "$.schema[0].properties[7].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_employee_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'total_amt_employee' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"total_amt_employee\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.total_amt_employee", "logical_type": "number", @@ -965,20 +1151,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.1369589966489, + "execution_time_ms": 7.811667004716583, "check_path": "$.schema[0].properties[8].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_cpf_amt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_cpf_amt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_cpf_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_cpf_amt" }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", @@ -995,20 +1187,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 9.304834005888551, + "execution_time_ms": 10.776291994261555, "check_path": "$.schema[0].properties[8].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_cpf_amt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_cpf_amt' must be valid number", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_cpf_amt\" IS NULL AND CAST(\"employee_cpf_amt\" AS DOUBLE PRECISION) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "number" }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", @@ -1025,20 +1223,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.339750001439825, + "execution_time_ms": 7.453291007550433, "check_path": "$.schema[0].properties[8].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_cpf_amt_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_cpf_amt' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_cpf_amt\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.employee_cpf_amt", "logical_type": "number", @@ -1055,20 +1259,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.089125010883436, + "execution_time_ms": 6.992291993810795, "check_path": "$.schema[0].properties[9].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_gross_amt_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_gross_amt' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_gross_amt\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_gross_amt" }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", @@ -1085,20 +1295,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 12.091583994333632, + "execution_time_ms": 11.974291992373765, "check_path": "$.schema[0].properties[9].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_gross_amt_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_gross_amt' must be valid number", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"employee_gross_amt\" IS NULL AND CAST(\"employee_gross_amt\" AS DOUBLE PRECISION) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "number" }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", @@ -1115,20 +1331,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.5386660028016195, + "execution_time_ms": 6.4171249978244305, "check_path": "$.schema[0].properties[9].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_gross_amt_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_gross_amt' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE \"employee_gross_amt\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_payroll.employee_gross_amt", "logical_type": "number", @@ -1145,20 +1367,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.80279199918732, + "execution_time_ms": 9.263290994567797, "check_path": "$.schema[0].properties[10].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must exist in 'demo_employee_payroll'", "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_payroll\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "phone_number" }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", @@ -1175,20 +1403,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 14.585208002245054, + "execution_time_ms": 17.175583998323418, "check_path": "$.schema[0].properties[10].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'phone_number' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND CAST(\"phone_number\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", @@ -1205,20 +1439,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 32.70300000440329, + "execution_time_ms": 43.42120799992699, "check_path": "$.schema[0].properties[10].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_options_minLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must have minimum length of 8", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(CAST(\"phone_number\" AS VARCHAR)) < 8", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 8 }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", @@ -1235,20 +1475,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 34.6164999937173, + "execution_time_ms": 45.03570799715817, "check_path": "$.schema[0].properties[10].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_options_maxLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must have maximum length of 11", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND LENGTH(CAST(\"phone_number\" AS VARCHAR)) > 11", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 11 }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", @@ -1265,20 +1511,26 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 38.351875002263114, + "execution_time_ms": 43.62716700416058, "check_path": "$.schema[0].properties[10].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must match pattern '^\\+65[0-9]{8}$'", "query": "SELECT COUNT(*) FROM \"demo_employee_payroll\" WHERE NOT \"phone_number\" IS NULL AND NOT CAST(\"phone_number\" AS VARCHAR) ~ '^\\+65[0-9]{8}$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^\\+65[0-9]{8}$" }, "target": "demo_employee_payroll.phone_number", "logical_type": "string", @@ -1295,14 +1547,14 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 23.633499993593432, + "execution_time_ms": 21.877167004277, "check_path": "$.schema[0].quality[0]", "check_ref_type": "SQLTableCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "total_amt_employee_equals_gross_plus_cpf", "description": "Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency", "type": "sql", @@ -1314,9 +1566,21 @@ "User-Defined Quality Rule" ] }, - "target": "demo_employee_payroll", - "aggregation_type": "count", - "tables_in_query": [ + "contract_definition": { + "name": "total_amt_employee_equals_gross_plus_cpf", + "description": "Validates that total_amt_employee equals the sum of employee_gross_amt and employee_cpf_amt for consistency", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee != (employee_gross_amt + employee_cpf_amt)", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, + "target": "demo_employee_payroll", + "aggregation_type": "count", + "tables_in_query": [ "demo_employee_payroll" ], "rendered_implementation": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt_employee <> (employee_gross_amt + employee_cpf_amt)" @@ -1328,20 +1592,29 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 0, - "execution_time_ms": 12.046417003148235, + "execution_time_ms": 42.15633300191257, "check_path": "$.schema[0].quality[3]", "check_ref_type": "RowCountCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBeGreaterThan", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "rowCount", "mustBeGreaterThan": 0, "dimension": "completeness", + "tags": [ + "vowl_generated_check" + ], "description": "Total row count for 'demo_employee_payroll'" }, + "contract_definition": { + "type": "library", + "metric": "rowCount", + "mustBeGreaterThan": 0, + "dimension": "completeness" + }, "target": "demo_employee_payroll", "aggregation_type": "count", "tables_in_query": [ @@ -1356,20 +1629,29 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 17.949457993381657, + "execution_time_ms": 27.207833001739345, "check_path": "$.schema[0].properties[0].quality[0]", "check_ref_type": "NullValuesCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "nullValues", "mustBe": 0, "dimension": "completeness", + "tags": [ + "vowl_generated_check" + ], "description": "Count of NULL values in 'employee_id' of 'demo_employee_payroll'" }, + "contract_definition": { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness" + }, "target": "demo_employee_payroll.employee_id", "logical_type": "string", "aggregation_type": "count", @@ -1385,13 +1667,24 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 23.35858299920801, + "execution_time_ms": 32.52749999228399, "check_path": "$.schema[0].properties[2].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "Month", + "description": "Based on ISO 8601, assumed to be in UTC +8 | YYYY-MM", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(month AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "Month", @@ -1418,14 +1711,14 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 25.47229200717993, + "execution_time_ms": 26.39941700908821, "check_path": "$.schema[0].properties[2].quality[1]", "check_ref_type": "InvalidValuesCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "invalidValues", "mustBe": 0, @@ -1433,8 +1726,20 @@ "arguments": { "pattern": "^\\d{4}-(0[1-9]|1[0-2])$" }, + "tags": [ + "vowl_generated_check" + ], "description": "Count of invalid values in 'month' of 'demo_employee_payroll'" }, + "contract_definition": { + "type": "library", + "metric": "invalidValues", + "mustBe": 0, + "dimension": "conformity", + "arguments": { + "pattern": "^\\d{4}-(0[1-9]|1[0-2])$" + } + }, "target": "demo_employee_payroll.month", "logical_type": "string", "aggregation_type": "count", @@ -1450,13 +1755,24 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 23.1642079888843, + "execution_time_ms": 21.720541000831872, "check_path": "$.schema[0].properties[3].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "Date With Timezone", + "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(payroll_start_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$'\n OR LENGTH(CAST(payroll_start_dt AS TEXT)) > 22;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "Date With Timezone", @@ -1483,13 +1799,24 @@ "expected_value": 0, "actual_value": 3, "failed_rows_count": 3, - "execution_time_ms": 22.935790999326855, + "execution_time_ms": 21.855708007933572, "check_path": "$.schema[0].properties[4].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "Date With Timezone", + "description": "If not in UTC +8 | YYYY-MM-DD[T]hh:mm+hh:mm e.g. 2015-01-01T12:00+02:00 for UTC +2", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_payroll\nWHERE CAST(payroll_end_dt AS TEXT) !~ '^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9][+-]([01][0-9]|2[0-3]):[0-5][0-9]$'\n OR LENGTH(CAST(payroll_end_dt AS TEXT)) > 22;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "Date With Timezone", @@ -1516,13 +1843,25 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 17.85162500164006, + "execution_time_ms": 18.529374996433035, "check_path": "$.schema[0].properties[5].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "name": "total_amt_equals_employer_cpf_plus_total_amt_employee", + "description": "Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency", + "type": "sql", + "dimension": "consistency", + "severity": "error", + "query": "SELECT COUNT(*) FROM demo_employee_payroll WHERE total_amt != (employer_cpf_amt + total_amt_employee) OR (total_amt IS NOT NULL AND (employer_cpf_amt IS NULL OR total_amt_employee IS NULL))", + "mustBe": 0, + "tags": [ + "User-Defined Quality Rule" + ] + }, "contract_definition": { "name": "total_amt_equals_employer_cpf_plus_total_amt_employee", "description": "Validates that total_amt equals the sum of employer_cpf_amt and total_amt_employee for consistency", @@ -1550,14 +1889,14 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 22.719083004631102, + "execution_time_ms": 33.28112499730196, "check_path": "$.schema[0].properties[5].quality[1]", "check_ref_type": "MissingValuesCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "type": "library", "metric": "missingValues", "mustBe": 0, @@ -1568,8 +1907,23 @@ "" ] }, + "tags": [ + "vowl_generated_check" + ], "description": "Count of missing values in 'total_amt' of 'demo_employee_payroll'" }, + "contract_definition": { + "type": "library", + "metric": "missingValues", + "mustBe": 0, + "dimension": "completeness", + "arguments": { + "missingValues": [ + null, + "" + ] + } + }, "target": "demo_employee_payroll.total_amt", "logical_type": "integer", "aggregation_type": "count", @@ -1585,20 +1939,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.673040999681689, + "execution_time_ms": 10.498125004232861, "check_path": "$.schema[1].properties[0].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must exist in 'demo_employee_list'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_id" }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1615,20 +1975,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 8.802083000773564, + "execution_time_ms": 20.03816599608399, "check_path": "$.schema[1].properties[0].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_id' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND CAST(\"employee_id\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1645,20 +2011,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 27.994290998321958, + "execution_time_ms": 42.80774999642745, "check_path": "$.schema[1].properties[0].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must match pattern '^e[0-9]+$'", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND NOT CAST(\"employee_id\" AS VARCHAR) ~ '^e[0-9]+$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^e[0-9]+$" }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1675,20 +2047,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 27.982749990769662, + "execution_time_ms": 48.060291999718174, "check_path": "$.schema[1].properties[0].logicalTypeOptions.minLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_minLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must have minimum length of 7", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) < 7", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 7 }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1705,20 +2083,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 28.017333999741822, + "execution_time_ms": 34.422624987200834, "check_path": "$.schema[1].properties[0].logicalTypeOptions.maxLength", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_logical_type_options_maxLength_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_id' must have maximum length of 7", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL AND LENGTH(CAST(\"employee_id\" AS VARCHAR)) > 7", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": 7 }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1735,20 +2119,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.478375009261072, + "execution_time_ms": 8.225458004744723, "check_path": "$.schema[1].properties[0].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_id' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1765,20 +2155,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 7.041749995551072, + "execution_time_ms": 8.616082996013574, "check_path": "$.schema[1].properties[0].unique", "check_ref_type": "UniqueCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_unique_check", "type": "sql", "dimension": "consistency", "description": "Column 'employee_id' must contain unique values", "query": "SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1795,20 +2191,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 5.410500001744367, + "execution_time_ms": 6.8277919926913455, "check_path": "$.schema[1].properties[0].primaryKey", "check_ref_type": "PrimaryKeyCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_id_primary_key_check", "type": "sql", "dimension": "consistency", "description": "Primary key column 'employee_id' must be unique and not null", "query": "SELECT ((SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_id\" IS NULL)) + ((SELECT COUNT(*) FROM (SELECT \"employee_id\" FROM \"demo_employee_list\" WHERE NOT \"employee_id\" IS NULL GROUP BY \"employee_id\" HAVING COUNT(*) > 1)))", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.employee_id", "logical_type": "string", @@ -1825,20 +2227,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.55170799291227, + "execution_time_ms": 17.24595800624229, "check_path": "$.schema[1].properties[1].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_last_name_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_last_name' must exist in 'demo_employee_list'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_last_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_last_name" }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", @@ -1855,20 +2263,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.189666994847357, + "execution_time_ms": 14.31154199235607, "check_path": "$.schema[1].properties[1].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_last_name_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_last_name' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_last_name\" IS NULL AND CAST(\"employee_last_name\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", @@ -1885,20 +2299,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.203083001310006, + "execution_time_ms": 6.7928750067949295, "check_path": "$.schema[1].properties[1].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_last_name_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_last_name' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_last_name\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.employee_last_name", "logical_type": "string", @@ -1915,20 +2335,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.034209000063129, + "execution_time_ms": 7.681333998334594, "check_path": "$.schema[1].properties[2].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_first_name_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'employee_first_name' must exist in 'demo_employee_list'", "query": "SELECT COUNT(*) FROM (SELECT \"employee_first_name\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "employee_first_name" }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", @@ -1945,20 +2371,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.555083004874177, + "execution_time_ms": 11.540167004568502, "check_path": "$.schema[1].properties[2].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_first_name_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'employee_first_name' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"employee_first_name\" IS NULL AND CAST(\"employee_first_name\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", @@ -1975,20 +2407,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.4019589917734265, + "execution_time_ms": 7.362165997619741, "check_path": "$.schema[1].properties[2].required", "check_ref_type": "RequiredCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "employee_first_name_required_check", "type": "sql", "dimension": "completeness", "description": "Column 'employee_first_name' must not contain NULL values", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE \"employee_first_name\" IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.employee_first_name", "logical_type": "string", @@ -2005,20 +2443,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.396415992639959, + "execution_time_ms": 7.876791001763195, "check_path": "$.schema[1].properties[3].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "current_title_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'current_title' must exist in 'demo_employee_list'", "query": "SELECT COUNT(*) FROM (SELECT \"current_title\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "current_title" }, "target": "demo_employee_list.current_title", "logical_type": "string", @@ -2035,20 +2479,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.182790996623226, + "execution_time_ms": 12.056583000230603, "check_path": "$.schema[1].properties[3].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "current_title_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'current_title' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"current_title\" IS NULL AND CAST(\"current_title\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_list.current_title", "logical_type": "string", @@ -2065,20 +2515,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.789667007979006, + "execution_time_ms": 13.484708993928507, "check_path": "$.schema[1].properties[4].name", "check_ref_type": "DeclaredColumnExistsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_column_exists_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must exist in 'demo_employee_list'", "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" LIMIT 0) AS _vowl_column_exists", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "phone_number" }, "target": "demo_employee_list.phone_number", "logical_type": "string", @@ -2095,20 +2551,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 10.848208010429516, + "execution_time_ms": 11.268125002970919, "check_path": "$.schema[1].properties[4].logicalType", "check_ref_type": "LogicalTypeCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_check", "type": "sql", "dimension": "conformity", "description": "Values in 'phone_number' must be valid string", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND CAST(\"phone_number\" AS VARCHAR) IS NULL", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "string" }, "target": "demo_employee_list.phone_number", "logical_type": "string", @@ -2125,20 +2587,26 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 32.44387501035817, + "execution_time_ms": 32.583833002718166, "check_path": "$.schema[1].properties[4].logicalTypeOptions.pattern", "check_ref_type": "LogicalTypeOptionsCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_logical_type_options_pattern_check", "type": "sql", "dimension": "conformity", "description": "Column 'phone_number' must match pattern '^\\+[0-9]+$'", "query": "SELECT COUNT(*) FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL AND NOT CAST(\"phone_number\" AS VARCHAR) ~ '^\\+[0-9]+$'", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": "^\\+[0-9]+$" }, "target": "demo_employee_list.phone_number", "logical_type": "string", @@ -2155,20 +2623,26 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 6.953083007829264, + "execution_time_ms": 6.587625000975095, "check_path": "$.schema[1].properties[4].unique", "check_ref_type": "UniqueCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": true, "engine": "sql", - "contract_definition": { + "check_definition": { "name": "phone_number_unique_check", "type": "sql", "dimension": "consistency", "description": "Column 'phone_number' must contain unique values", "query": "SELECT COUNT(*) FROM (SELECT \"phone_number\" FROM \"demo_employee_list\" WHERE NOT \"phone_number\" IS NULL GROUP BY \"phone_number\" HAVING COUNT(*) > 1)", - "mustBe": 0 + "mustBe": 0, + "tags": [ + "vowl_generated_check" + ] + }, + "contract_definition": { + "value": true }, "target": "demo_employee_list.phone_number", "logical_type": "string", @@ -2185,13 +2659,24 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 22.425041999667883, + "execution_time_ms": 21.21066600375343, "check_path": "$.schema[1].properties[1].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "PersonName", + "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE employee_last_name IS NOT NULL\n AND LENGTH(CAST(employee_last_name AS TEXT)) > 66;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "PersonName", @@ -2218,13 +2703,24 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 18.33512500161305, + "execution_time_ms": 25.908499999786727, "check_path": "$.schema[1].properties[2].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "PersonName", + "description": "PersonName may include various name types for UIN/FIN holders including legal names, abbreviated names, Hanyu Pinyin names, alias names, married names, and changed names | X(66)", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE employee_first_name IS NOT NULL\n AND LENGTH(CAST(employee_first_name AS TEXT)) > 66;", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "PersonName", @@ -2251,13 +2747,24 @@ "expected_value": 0, "actual_value": 0, "failed_rows_count": 0, - "execution_time_ms": 18.17870800732635, + "execution_time_ms": 23.390207992633805, "check_path": "$.schema[1].properties[4].quality[0]", "check_ref_type": "SQLColumnCheckReference", "schema_name": "demo_employee_list", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "type": "sql", + "name": "TelephoneNumber", + "description": "Based on ITU T E.164 | e.g. 095236", + "mustBe": 0, + "query": "SELECT COUNT(*)\nFROM demo_employee_list\nWHERE CAST(phone_number AS TEXT) !~ '^[0-9]{1,12}$';", + "tags": [ + "SG-DRM v5.0" + ], + "dimension": "conformity" + }, "contract_definition": { "type": "sql", "name": "TelephoneNumber", @@ -2284,13 +2791,24 @@ "expected_value": 0, "actual_value": 1, "failed_rows_count": 1, - "execution_time_ms": 62.502582994056866, + "execution_time_ms": 121.92870900616981, "check_path": "$.schema[0].quality[1]", "check_ref_type": "SQLTableCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "name": "employee_id_exists_in_master_list", + "description": "Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.", + "type": "sql", + "dimension": "consistency", + "query": "SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref \n ON payroll.employee_id = ref.employee_id\nWHERE ref.employee_id IS NULL", + "mustBe": 0, + "tags": [ + "Cross-Table Validation" + ] + }, "contract_definition": { "name": "employee_id_exists_in_master_list", "description": "Referential integrity check: Ensures all employee IDs in the payroll table exist in the reference employee list.", @@ -2317,13 +2835,24 @@ "expected_value": 0, "actual_value": 2, "failed_rows_count": 2, - "execution_time_ms": 18.972791003761813, + "execution_time_ms": 29.520874988520518, "check_path": "$.schema[0].quality[2]", "check_ref_type": "SQLTableCheckReference", "schema_name": "demo_employee_payroll", "operator": "mustBe", "is_generated": false, "engine": "sql", + "check_definition": { + "name": "phone_number_exists_in_master_list", + "description": "Ensures the phone numbers recorded in payroll matches an entry in the master employee list.", + "type": "sql", + "dimension": "consistency", + "query": "SELECT COUNT(*) FROM demo_employee_payroll payroll LEFT JOIN demo_employee_list ref \n ON payroll.phone_number = ref.phone_number\nWHERE payroll.phone_number IS NOT NULL \n AND ref.phone_number IS NULL", + "mustBe": 0, + "tags": [ + "Cross-Table Validation" + ] + }, "contract_definition": { "name": "phone_number_exists_in_master_list", "description": "Ensures the phone numbers recorded in payroll matches an entry in the master employee list.", diff --git a/examples/vowl_usage_patterns_demo.ipynb b/examples/vowl_usage_patterns_demo.ipynb index 4dff11a..13be359 100644 --- a/examples/vowl_usage_patterns_demo.ipynb +++ b/examples/vowl_usage_patterns_demo.ipynb @@ -125,12 +125,153 @@ "name": "stderr", "output_type": "stream", "text": [ - "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_78146/1521035067.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "/var/folders/tg/0hbw1fbs5q1_5kdtp36klrtw0000gn/T/ipykernel_25503/1521035067.py:4: DtypeWarning: Columns (0: lease_commence_date) have mixed types. Specify dtype option on import or set low_memory=False.\n", " df = pd.read_csv(HDB_CSV).fillna(\"\")\n" ] }, { "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "month", + "rawType": "str", + "type": "string" + }, + { + "name": "town", + "rawType": "str", + "type": "string" + }, + { + "name": "flat_type", + "rawType": "str", + "type": "string" + }, + { + "name": "block", + "rawType": "str", + "type": "string" + }, + { + "name": "street_name", + "rawType": "str", + "type": "string" + }, + { + "name": "storey_range", + "rawType": "str", + "type": "string" + }, + { + "name": "floor_area_sqm", + "rawType": "float64", + "type": "float" + }, + { + "name": "flat_model", + "rawType": "str", + "type": "string" + }, + { + "name": "lease_commence_date", + "rawType": "object", + "type": "string" + }, + { + "name": "remaining_lease", + "rawType": "str", + "type": "string" + }, + { + "name": "resale_price", + "rawType": "float64", + "type": "float" + } + ], + "ref": "ec1f9970-c371-41d3-836c-6ff2f09b92c8", + "rows": [ + [ + "0", + "2017-01", + "ANG MO KIO", + "2 ROOM", + "406", + "ANG MO KIO AVE 10", + "10 TO 12", + "44.0", + "Improved", + "1979", + "61 years 04 months", + "232000.0" + ], + [ + "1", + "2017-01", + "ANG MO KIO", + "3 ROOM", + "108", + "ANG MO KIO AVE 4", + "01 TO 03", + "67.0", + "New Generation", + "1978", + "60 years 07 months", + "250000.0" + ], + [ + "2", + "2017-01", + "ANG MO KIO", + "3 ROOM", + "602", + "ANG MO KIO AVE 5", + "01 TO 03", + "67.0", + "New Generation", + "1980", + "62 years 05 months", + "262000.0" + ], + [ + "3", + "2017-01", + "ANG MO KIO", + "3 ROOM", + "465", + "ANG MO KIO AVE 10", + "04 TO 06", + "68.0", + "New Generation", + "1980", + "62 years 01 month", + "265000.0" + ], + [ + "4", + "2017-01", + "ANG MO KIO", + "3 ROOM", + "601", + "ANG MO KIO AVE 5", + "01 TO 03", + "67.0", + "New Generation", + "1980", + "62 years 05 months", + "265000.0" + ] + ], + "shape": { + "columns": 11, + "rows": 5 + } + }, "text/html": [ "
\n", "