diff --git a/.formatter.exs b/.formatter.exs index 8eecf50..d9c81cd 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench}/**/*.{ex,exs}"] + inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench}/**/*.{ex,exs}"], + import_deps: [:stream_data] ] diff --git a/CHANGELOG.md b/CHANGELOG.md index efb2c33..2682a61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- use scientific decimals rendering in params https://github.com/plausible/ch/pull/333 + ## 0.8.1 (2026-05-05) - relax Decimal version requirement https://github.com/plausible/ch/pull/332 diff --git a/lib/ch/query.ex b/lib/ch/query.ex index 709c0ac..4fad7e8 100644 --- a/lib/ch/query.ex +++ b/lib/ch/query.ex @@ -332,7 +332,7 @@ defimpl DBConnection.Query, for: Ch.Query do defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) defp encode_param(nil), do: "\\N" - defp encode_param(%Decimal{} = d), do: Decimal.to_string(d, :normal) + defp encode_param(%Decimal{} = d), do: decimal_to_string!(d) defp encode_param(%Date{} = date), do: Date.to_iso8601(date) defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) defp encode_param(%Time{} = time), do: Time.to_iso8601(time) @@ -409,6 +409,13 @@ defimpl DBConnection.Query, for: Ch.Query do defp escape_param([], param), do: param + @compile inline: [decimal_to_string!: 1] + defp decimal_to_string!(%Decimal{coef: coef}) when coef in [:NaN, :inf] do + raise ArgumentError, "ClickHouse Decimal values must be finite" + end + + defp decimal_to_string!(d), do: Decimal.to_string(d, :scientific) + @spec headers(Keyword.t()) :: Mint.Types.headers() defp headers(opts), do: Keyword.get(opts, :headers, []) end diff --git a/mix.exs b/mix.exs index 85104f8..1069e78 100644 --- a/mix.exs +++ b/mix.exs @@ -61,7 +61,8 @@ defmodule Ch.MixProject do {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:ex_doc, ">= 0.0.0", only: :docs}, {:tz, "~> 0.28.1", only: [:dev, :test, :bench]}, - {:excoveralls, "~> 0.18.5", only: :test} + {:excoveralls, "~> 0.18.5", only: :test}, + {:stream_data, "~> 1.3", only: :test} ] end diff --git a/mix.lock b/mix.lock index dc9bf70..2421590 100644 --- a/mix.lock +++ b/mix.lock @@ -17,6 +17,7 @@ "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, + "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, "tz": {:hex, :tz, "0.28.1", "717f5ffddfd1e475e2a233e221dc0b4b76c35c4b3650b060c8e3ba29dd6632e9", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.6", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "bfdca1aa1902643c6c43b77c1fb0cb3d744fd2f09a8a98405468afdee0848c8a"}, } diff --git a/test/ch/decimal_param_test.exs b/test/ch/decimal_param_test.exs new file mode 100644 index 0000000..90edf93 --- /dev/null +++ b/test/ch/decimal_param_test.exs @@ -0,0 +1,125 @@ +defmodule Ch.DecimalParamTest do + use ExUnit.Case, + parameterize: [%{query_options: []}, %{query_options: [multipart: true]}], + async: true + + use ExUnitProperties + + import Ch.Test, only: [parameterize_query: 4, parameterize_query!: 4] + + setup ctx do + {:ok, conn} = Ch.start_link() + {:ok, conn: conn, query_options: ctx[:query_options] || []} + end + + test "decimal parameter boundaries", ctx do + max_integer = String.duplicate("9", 76) + max_scale = "0." <> String.duplicate("0", 75) <> "1" + + assert_decimal_param(ctx, Decimal.new("1.23"), "Decimal(76, 2)", Decimal.new("1.23")) + assert_decimal_param(ctx, Decimal.new("-1.23"), "Decimal(76, 2)", Decimal.new("-1.23")) + + assert_decimal_param( + ctx, + Decimal.new(max_integer), + "Decimal(76, 0)", + Decimal.new(max_integer) + ) + + assert_decimal_param(ctx, Decimal.new(1, 1, -76), "Decimal(76, 76)", Decimal.new(max_scale)) + end + + test "compact exponent Decimal params are not expanded before request", ctx do + encoded = encoded_decimal_param(ctx.query_options, Decimal.new("1e1000000")) + + assert encoded =~ "1E+1000000" + assert byte_size(encoded) < 300 + end + + test "decimal parameters reject over-limit values", ctx do + assert decimal_error(ctx, Decimal.new(1, 1, 76), "Decimal(76, 0)") =~ + "value 1E+76 cannot be parsed as Decimal(76, 0)" + + assert decimal_error(ctx, Decimal.new(String.duplicate("9", 77)), "Decimal(76, 0)") =~ + "value 99999999999999999999999999999999999999999999999999999999999999999999999999999 cannot be parsed as Decimal(76, 0)" + + assert decimal_error(ctx, Decimal.new("1e1000000"), "Decimal(76, 0)") =~ + "value 1E+1000000 cannot be parsed as Decimal(76, 0)" + + assert_raise ArgumentError, "ClickHouse Decimal values must be finite", fn -> + decimal_error(ctx, Decimal.new("NaN"), "Decimal(76, 0)") + end + + assert_raise ArgumentError, "ClickHouse Decimal values must be finite", fn -> + decimal_error(ctx, Decimal.new("Infinity"), "Decimal(76, 0)") + end + end + + test "decimal parameters below declared scale round to zero", ctx do + assert_decimal_param(ctx, Decimal.new(1, 1, -77), "Decimal(76, 76)", Decimal.new("0E-76")) + end + + property "compact exponent Decimal integer params round-trip", ctx do + check all decimal <- compact_decimal_integer() do + assert_decimal_param(ctx, decimal, "Decimal(76, 0)", decimal) + end + end + + property "Decimal params with scale round-trip", ctx do + check all decimal <- decimal_with_scale() do + assert_decimal_param(ctx, decimal, "Decimal(76, 12)", decimal) + end + end + + defp assert_decimal_param(ctx, decimal, type, expected) do + assert %Ch.Result{rows: [[actual, ^type]]} = + parameterize_query!( + ctx, + "select {d:#{type}} as x, toTypeName(x)", + %{"d" => decimal}, + ctx.query_options + ) + + assert Decimal.compare(actual, expected) == :eq + end + + defp decimal_error(ctx, decimal, type) do + assert {:error, error} = + parameterize_query( + ctx, + "select {d:#{type}}", + %{"d" => decimal}, + ctx.query_options + ) + + Exception.message(error) + end + + defp compact_decimal_integer do + gen all sign <- member_of([1, -1]), + coef <- integer(1..9_999_999_999_999_999), + exp <- integer(0..50) do + Decimal.new(sign, coef, exp) + end + end + + defp decimal_with_scale do + gen all sign <- member_of([1, -1]), + coef <- integer(1..9_999_999_999_999), + exp <- integer(-12..20) do + Decimal.new(sign, coef, exp) + end + end + + defp encoded_decimal_param(query_options, decimal) do + query = Ch.Query.build("select {d:Decimal(76, 0)}", query_options) + + {query_params, _headers, body} = + DBConnection.Query.encode(query, %{"d" => decimal}, []) + + case query_params do + [{"param_d", value}] -> value + [] -> IO.iodata_to_binary(body) + end + end +end