From 0944a133fcd59cccd09718e0424cc7cc22a1e6fc Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 16:41:25 +0300 Subject: [PATCH] Backport scientific Decimal params to v0.7.3 --- .formatter.exs | 3 +- CHANGELOG.md | 2 +- lib/ch/query.ex | 11 ++- mix.exs | 3 +- mix.lock | 1 + test/ch/decimal_param_test.exs | 125 +++++++++++++++++++++++++++++++++ test/ch/query_string_test.exs | 16 ----- 7 files changed, 139 insertions(+), 22 deletions(-) create mode 100644 test/ch/decimal_param_test.exs diff --git a/.formatter.exs b/.formatter.exs index 8eecf50..d9c81cd 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench}/**/*.{ex,exs}"] + inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench}/**/*.{ex,exs}"], + import_deps: [:stream_data] ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 50614db..d66cdc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.7.3 (2026-05-06) -- Bound Decimal query parameter rendering +- use scientific decimals rendering in params https://github.com/plausible/ch/pull/333 ## 0.7.2 (2026-05-05) diff --git a/lib/ch/query.ex b/lib/ch/query.ex index c810e55..00731ff 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -332,7 +332,7 @@ defimpl DBConnection.Query, for: Ch.Query do defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) defp encode_param(nil), do: "\\N" - defp encode_param(%Decimal{} = d), do: decimal_to_string(d) + defp encode_param(%Decimal{} = d), do: decimal_to_string!(d) defp encode_param(%Date{} = date), do: Date.to_iso8601(date) defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) defp encode_param(%Time{} = time), do: Time.to_iso8601(time) @@ -402,8 +402,6 @@ defimpl DBConnection.Query, for: Ch.Query do [encode_array_param(k), ?:, encode_array_param(v)] end - defp decimal_to_string(decimal), do: Decimal.to_string(decimal) - defp escape_param([{pattern, replacement} | escapes], param) do param = String.replace(param, pattern, replacement) escape_param(escapes, param) @@ -411,6 +409,13 @@ defimpl DBConnection.Query, for: Ch.Query do defp escape_param([], param), do: param + @compile inline: [decimal_to_string!: 1] + defp decimal_to_string!(%Decimal{coef: coef}) when coef in [:NaN, :inf] do + raise ArgumentError, "ClickHouse Decimal values must be finite" + end + + defp decimal_to_string!(d), do: Decimal.to_string(d, :scientific) + @spec headers(Keyword.t()) :: Mint.Types.headers() defp headers(opts), do: Keyword.get(opts, :headers, []) end diff --git a/mix.exs b/mix.exs index 3fec8d2..69ac7b4 100644 --- a/mix.exs +++ b/mix.exs @@ -46,7 +46,8 @@ defmodule Ch.MixProject do {:benchee, "~> 1.0", only: [:bench]}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :docs}, - {:tz, "~> 0.28.1", only: [:test]} + {:tz, "~> 0.28.1", only: [:test]}, + {:stream_data, "~> 1.3", only: :test} ] end diff --git a/mix.lock b/mix.lock index 1cf2d83..bf24df8 100644 --- a/mix.lock +++ b/mix.lock @@ -16,6 +16,7 @@ "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, + "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, "tz": {:hex, :tz, "0.28.1", "717f5ffddfd1e475e2a233e221dc0b4b76c35c4b3650b060c8e3ba29dd6632e9", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.6", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "bfdca1aa1902643c6c43b77c1fb0cb3d744fd2f09a8a98405468afdee0848c8a"}, } diff --git a/test/ch/decimal_param_test.exs b/test/ch/decimal_param_test.exs new file mode 100644 index 0000000..3406fac --- /dev/null +++ b/test/ch/decimal_param_test.exs @@ -0,0 +1,125 @@ +defmodule Ch.DecimalParamTest do + use ExUnit.Case, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}], + async: true + + use ExUnitProperties + + import Ch.Test, only: [parameterize_query: 4, parameterize_query!: 4] + + setup ctx do + {:ok, conn} = Ch.start_link() + {:ok, conn: conn, query_options: ctx[:query_options] || []} + end + + test "decimal parameter boundaries", ctx do + max_integer = String.duplicate("9", 76) + max_scale = "0." <> String.duplicate("0", 75) <> "1" + + assert_decimal_param(ctx, Decimal.new("1.23"), "Decimal(76, 2)", Decimal.new("1.23")) + assert_decimal_param(ctx, Decimal.new("-1.23"), "Decimal(76, 2)", Decimal.new("-1.23")) + + assert_decimal_param( + ctx, + Decimal.new(max_integer), + "Decimal(76, 0)", + Decimal.new(max_integer) + ) + + assert_decimal_param(ctx, Decimal.new(1, 1, -76), "Decimal(76, 76)", Decimal.new(max_scale)) + end + + test "compact exponent Decimal params are not expanded before request", ctx do + encoded = encoded_decimal_param(ctx.query_options, Decimal.new("1e1000000")) + + assert encoded =~ "1E+1000000" + assert byte_size(encoded) < 300 + end + + test "decimal parameters reject over-limit values", ctx do + assert decimal_error(ctx, Decimal.new(1, 1, 76), "Decimal(76, 0)") =~ + "Decimal value is too big: 1 digits were read: '1'e76. Expected to read decimal with scale 0 and precision 76: value 1E+76 cannot be parsed as Decimal(76, 0) for query parameter 'd'" + + assert decimal_error(ctx, Decimal.new(String.duplicate("9", 77)), "Decimal(76, 0)") =~ + "Too many digits (77 > 76) in decimal value: value 99999999999999999999999999999999999999999999999999999999999999999999999999999 cannot be parsed as Decimal(76, 0) for query parameter 'd'." + + assert decimal_error(ctx, Decimal.new("1e1000000"), "Decimal(76, 0)") =~ + "Decimal value is too big: 1 digits were read: '1'e1000000. Expected to read decimal with scale 0 and precision 76: value 1E+1000000 cannot be parsed as Decimal(76, 0) for query parameter 'd'." + + assert_raise ArgumentError, "ClickHouse Decimal values must be finite", fn -> + decimal_error(ctx, Decimal.new("NaN"), "Decimal(76, 0)") + end + + assert_raise ArgumentError, "ClickHouse Decimal values must be finite", fn -> + decimal_error(ctx, Decimal.new("Infinity"), "Decimal(76, 0)") + end + end + + test "decimal parameters below declared scale round to zero", ctx do + assert_decimal_param(ctx, Decimal.new(1, 1, -77), "Decimal(76, 76)", Decimal.new("0E-76")) + end + + property "compact exponent Decimal integer params round-trip", ctx do + check all decimal <- compact_decimal_integer() do + assert_decimal_param(ctx, decimal, "Decimal(76, 0)", decimal) + end + end + + property "Decimal params with scale round-trip", ctx do + check all decimal <- decimal_with_scale() do + assert_decimal_param(ctx, decimal, "Decimal(76, 12)", decimal) + end + end + + defp assert_decimal_param(ctx, decimal, type, expected) do + assert %Ch.Result{rows: [[actual, ^type]]} = + parameterize_query!( + ctx, + "select {d:#{type}} as x, toTypeName(x)", + %{"d" => decimal}, + ctx.query_options + ) + + assert Decimal.compare(actual, expected) == :eq + end + + defp decimal_error(ctx, decimal, type) do + assert {:error, error} = + parameterize_query( + ctx, + "select {d:#{type}}", + %{"d" => decimal}, + ctx.query_options + ) + + Exception.message(error) + end + + defp compact_decimal_integer do + gen all sign <- member_of([1, -1]), + coef <- integer(1..9_999_999_999_999_999), + exp <- integer(0..50) do + Decimal.new(sign, coef, exp) + end + end + + defp decimal_with_scale do + gen all sign <- member_of([1, -1]), + coef <- integer(1..9_999_999_999_999), + exp <- integer(-12..20) do + Decimal.new(sign, coef, exp) + end + end + + defp encoded_decimal_param(query_options, decimal) do + query = Ch.Query.build("select {d:Decimal(76, 0)}", query_options) + + {query_params, _headers, body} = + DBConnection.Query.encode(query, %{"d" => decimal}, []) + + case query_params do + [{"param_d", value}] -> value + [] -> IO.iodata_to_binary(body) + end + end +end diff --git a/test/ch/query_string_test.exs b/test/ch/query_string_test.exs index 604bf29..398ab3e 100644 --- a/test/ch/query_string_test.exs +++ b/test/ch/query_string_test.exs @@ -31,20 +31,4 @@ defmodule Ch.QueryStringTest do ).rows == [[["abc", "123"]]] end - - test "decimal params are bounded", %{query_options: query_options} do - query = Ch.Query.build("select {d:Decimal(76, 0)}", query_options) - - {query_params, _headers, body} = - DBConnection.Query.encode(query, %{"d" => Decimal.new("1e1000000")}, []) - - encoded = - case query_params do - [{"param_d", value}] -> value - [] -> IO.iodata_to_binary(body) - end - - assert encoded =~ "1E+1000000" - assert byte_size(encoded) < 300 - end end