From 5c37b556369b7d7d32cd0efe460a518fe7315b68 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 9 Jul 2026 13:02:23 +0900 Subject: [PATCH 1/9] [SPARK-57999][SQL][PYTHON][DOCS] Document accepted input and return types for built-in functions ### What changes were proposed in this pull request? Documents, for every built-in function argument, which input types it accepts, and for the DataFrame APIs, what type the function returns, across the three public surfaces: SQL `@ExpressionDescription` arguments blocks, Scala `functions.scala` Scaladoc, and PySpark docstrings. Each argument names its canonical/expected type (e.g. "An expression that evaluates to an integer." for SQL, "A column that evaluates to an integer." for the DataFrame APIs), return types are documented on the DataFrame surfaces (e.g. "Returns a column that evaluates to a double." / "Returns a column of the same type as the input."), and constant-only arguments are marked "Must be a constant.". Documentation only; no functional or API change. ### Why are the changes needed? The SQL reference and the Scala/PySpark `functions` APIs previously had no per-argument type information for most built-in functions and no documented return type on the DataFrame APIs. ### Does this PR introduce any user-facing change? Yes, documentation only. The built-in function reference and the Scala/PySpark `functions` docstrings now state the accepted input type of each argument and the return type of each DataFrame function. ### How was this patch tested? The documented types were derived from and cross-checked against the analyzer's measured accept/reject behavior for every function and argument position (see the PR description for the generation-and-verification process). Existing documentation and expression tests (`ExpressionInfoSuite`, docs build) pass. --- python/pyspark/sql/functions/builtin.py | 1137 ++++++++++++++--- .../org/apache/spark/sql/functions.scala | 1080 +++++++++++----- .../sql/catalyst/expressions/Between.scala | 4 + .../sql/catalyst/expressions/TryEval.scala | 42 + .../expressions/aggregate/AnyValue.scala | 7 + .../aggregate/ApproxTopKAggregates.scala | 16 + .../expressions/aggregate/Average.scala | 10 + .../aggregate/CentralMomentAgg.scala | 30 + .../catalyst/expressions/aggregate/Corr.scala | 7 + .../expressions/aggregate/CountIf.scala | 5 + .../expressions/aggregate/Covariance.scala | 14 + .../expressions/aggregate/First.scala | 7 + .../catalyst/expressions/aggregate/Last.scala | 7 + .../expressions/aggregate/Measure.scala | 5 + .../catalyst/expressions/aggregate/Mode.scala | 8 + .../catalyst/expressions/aggregate/Sum.scala | 10 + .../aggregate/bitwiseAggregates.scala | 15 + .../aggregate/boolAggregates.scala | 10 + .../expressions/aggregate/collect.scala | 2 + .../aggregate/datasketchesAggregates.scala | 7 + .../expressions/aggregate/kllAggregates.scala | 12 + .../aggregate/linearRegression.scala | 63 + .../expressions/aggregate/percentiles.scala | 5 + .../aggregate/thetasketchesAggregates.scala | 8 + .../aggregate/tupleIntersectionAgg.scala | 16 + .../aggregate/tupleSketchAgg.scala | 20 + .../expressions/aggregate/tupleUnionAgg.scala | 15 + .../sql/catalyst/expressions/arithmetic.scala | 57 + .../expressions/bitmapExpressions.scala | 15 + .../expressions/bitwiseExpressions.scala | 38 + .../expressions/collationExpressions.scala | 1 + .../expressions/collectionOperations.scala | 171 +++ .../expressions/complexTypeCreator.scala | 16 + .../expressions/datetimeExpressions.scala | 221 ++++ .../spark/sql/catalyst/expressions/hash.scala | 22 + .../expressions/higherOrderFunctions.scala | 55 + .../expressions/intervalExpressions.scala | 8 + .../expressions/jsonExpressions.scala | 15 + .../catalyst/expressions/kllExpressions.scala | 93 ++ .../expressions/maskExpressions.scala | 5 + .../expressions/mathExpressions.scala | 203 +++ .../spark/sql/catalyst/expressions/misc.scala | 36 + .../expressions/nullExpressions.scala | 24 + .../expressions/numberFormatExpressions.scala | 21 + .../sql/catalyst/expressions/predicates.scala | 26 + .../expressions/randomExpressions.scala | 26 + .../expressions/regexpExpressions.scala | 28 + .../expressions/st/stExpressions.scala | 3 + .../expressions/stringExpressions.scala | 267 ++++ .../expressions/timeExpressions.scala | 30 + .../catalyst/expressions/urlExpressions.scala | 23 +- .../variant/variantExpressions.scala | 10 + .../sql/catalyst/expressions/xml/xpath.scala | 56 + 53 files changed, 3576 insertions(+), 456 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 1e348cfd42874..1b1641d1998cf 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -517,11 +517,13 @@ def sqrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -553,7 +555,9 @@ def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -634,6 +638,7 @@ def try_avg(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -691,9 +696,11 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a double, decimal, or interval. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a double or decimal. Examples -------- @@ -754,9 +761,11 @@ def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a numeric. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a numeric. Examples -------- @@ -803,9 +812,11 @@ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - multiplicand + multiplicand. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name - multiplier + multiplier. + A column that evaluates to a numeric or interval. Examples -------- @@ -867,7 +878,9 @@ def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -948,6 +961,7 @@ def try_sum(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -1017,11 +1031,13 @@ def abs(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the absolute value on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the absolute value of the input. + Returns a column of the same type as the input. Examples -------- @@ -1083,8 +1099,10 @@ def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column of any type. deterministic : bool, optional if there are multiple equally-frequent results then return the lowest (defaults to false). + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -1358,13 +1376,16 @@ def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values to be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be maximized. This could be the column instance or the column name as string. + A column of any type. k : int, optional If specified, returns an array of up to `k` values associated with the top `k` maximum ordering values, sorted in descending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1469,13 +1490,16 @@ def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values that will be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be minimized. This could be the column instance or the column name as string. + A column of any type. k : int, optional If specified, returns an array of up to `k` values associated with the bottom `k` minimum ordering values, sorted in ascending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1638,6 +1662,7 @@ def sum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1702,6 +1727,7 @@ def avg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1756,6 +1782,7 @@ def mean(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1800,6 +1827,7 @@ def median(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric, interval, or time. Returns ------- @@ -1939,8 +1967,10 @@ def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2094,8 +2124,10 @@ def string_agg( ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2286,11 +2318,13 @@ def acos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2341,11 +2375,13 @@ def acosh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse hyperbolic cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse hyperbolic cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2392,11 +2428,13 @@ def asin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse sine of `col`, as if computed by `java.lang.Math.asin()` + Returns a column that evaluates to a double. Examples -------- @@ -2444,11 +2482,13 @@ def asinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2495,11 +2535,13 @@ def atan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse tangent of `col`, as if computed by `java.lang.Math.atan()` + Returns a column that evaluates to a double. Examples -------- @@ -2546,11 +2588,13 @@ def atanh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2599,11 +2643,13 @@ def cbrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2650,8 +2696,10 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a double, decimal, or long. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2659,6 +2707,7 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2708,8 +2757,10 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a double, decimal, or long. scale : :class:`~pyspark.sql.Column` or int An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2717,6 +2768,7 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2765,12 +2817,14 @@ def cos(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosine of the angle, as if computed by `java.lang.Math.cos()`. + Returns a column that evaluates to a double. Examples -------- @@ -2817,12 +2871,14 @@ def cosh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()` + Returns a column that evaluates to a double. Examples -------- @@ -2869,11 +2925,13 @@ def cot(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cotangent of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2921,11 +2979,13 @@ def csc(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosecant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2992,11 +3052,13 @@ def exp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential of the given value. + Returns a column that evaluates to a double. Examples -------- @@ -3045,11 +3107,13 @@ def expm1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential less one. + Returns a column that evaluates to a double. Examples -------- @@ -3098,8 +3162,10 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the floor on. + A column that evaluates to a double, decimal, or long. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -3108,6 +3174,7 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` nearest integer that is less than or equal to given value. + Returns a column that evaluates to a long or decimal. Examples -------- @@ -3153,11 +3220,13 @@ def log10(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logarithm of the given value in Base 10. + Returns a column that evaluates to a double. Examples -------- @@ -3206,11 +3275,13 @@ def log1p(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate natural logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of the "given value plus one". + Returns a column that evaluates to a double. Examples -------- @@ -3245,11 +3316,13 @@ def negative(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate negative value for. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` negative value. + Returns a column of the same type as the input. Examples -------- @@ -3300,11 +3373,13 @@ def positive(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input value column. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` value. + Returns a column of the same type as the input. Examples -------- @@ -3337,11 +3412,13 @@ def rint(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -3376,12 +3453,14 @@ def sec(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Angle in radians + Angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` Secant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -3428,11 +3507,13 @@ def signum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3470,11 +3551,13 @@ def sign(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3512,11 +3595,13 @@ def sin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` sine of the angle, as if computed by `java.lang.Math.sin()` + Returns a column that evaluates to a double. Examples -------- @@ -3564,12 +3649,14 @@ def sinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh()` + Returns a column that evaluates to a double. Examples -------- @@ -3615,12 +3702,14 @@ def tan(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` tangent of the given value, as if computed by `java.lang.Math.tan()` + Returns a column that evaluates to a double. Examples -------- @@ -3667,13 +3756,15 @@ def tanh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic tangent of the given value as if computed by `java.lang.Math.tanh()` + Returns a column that evaluates to a double. Examples -------- @@ -3810,12 +3901,14 @@ def bit_count(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral or boolean. Returns ------- :class:`~pyspark.sql.Column` the number of bits that are set in the argument expr as an unsigned 64-bit integer, or NULL if the argument is NULL. + Returns a column that evaluates to an integer. See Also -------- @@ -3853,13 +3946,16 @@ def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -3912,13 +4008,16 @@ def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -4271,6 +4370,7 @@ def stddev(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. See Also -------- @@ -4310,6 +4410,7 @@ def std(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4353,6 +4454,7 @@ def stddev_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4394,6 +4496,7 @@ def stddev_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4434,6 +4537,7 @@ def variance(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4478,6 +4582,7 @@ def var_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4518,6 +4623,7 @@ def var_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -4556,8 +4662,10 @@ def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4646,8 +4754,10 @@ def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4736,8 +4846,10 @@ def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. See Also -------- @@ -4827,8 +4939,10 @@ def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. Returns ------- @@ -4917,8 +5031,10 @@ def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. Returns ------- @@ -5007,8 +5123,10 @@ def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. Returns ------- @@ -5097,8 +5215,10 @@ def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. Returns ------- @@ -5187,8 +5307,10 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. See Also -------- @@ -5277,8 +5399,10 @@ def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a double. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a double. Returns ------- @@ -5366,6 +5490,7 @@ def every(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. See Also -------- @@ -5422,6 +5547,7 @@ def bool_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. Returns ------- @@ -5475,6 +5601,7 @@ def some(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5531,6 +5658,7 @@ def bool_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5579,6 +5707,7 @@ def bit_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5652,6 +5781,7 @@ def bit_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5725,6 +5855,7 @@ def bit_xor(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5801,6 +5932,7 @@ def skewness(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. See Also -------- @@ -5842,6 +5974,7 @@ def kurtosis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- @@ -6096,12 +6229,14 @@ def degrees(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in degrees, as if computed by `java.lang.Math.toDegrees()` + Returns a column that evaluates to a double. Examples -------- @@ -6135,12 +6270,14 @@ def radians(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in degrees + angle in degrees. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in radians, as if computed by `java.lang.Math.toRadians()` + Returns a column that evaluates to a double. Examples -------- @@ -6174,9 +6311,11 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] Parameters ---------- col1 : :class:`~pyspark.sql.Column`, column name or float - coordinate on y-axis + coordinate on y-axis. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float - coordinate on x-axis + coordinate on x-axis. + A column that evaluates to a double. Returns ------- @@ -6186,6 +6325,7 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] in polar coordinates that corresponds to the point (`x`, `y`) in Cartesian coordinates, as if computed by `java.lang.Math.atan2()` + Returns a column that evaluates to a double. Examples -------- @@ -6214,13 +6354,16 @@ def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] ---------- col1 : :class:`~pyspark.sql.Column`, column name or float a leg. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float b leg. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` length of the hypotenuse. + Returns a column that evaluates to a double. Examples -------- @@ -6249,13 +6392,16 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) ---------- col1 : :class:`~pyspark.sql.Column`, column name or float the base number. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float the exponent number. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the base rased to the power the argument. + Returns a column that evaluates to a double. Examples -------- @@ -6287,14 +6433,17 @@ def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", Parameters ---------- dividend : :class:`~pyspark.sql.Column`, column name or float - the column that contains dividend, or the specified dividend value + the column that contains dividend, or the specified dividend value. + A column that evaluates to a numeric. divisor : :class:`~pyspark.sql.Column`, column name or float - the column that contains divisor, or the specified divisor value + the column that contains divisor, or the specified divisor value. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` positive value of dividend mod divisor. + Returns a column of the same type as the input. Notes ----- @@ -6343,18 +6492,23 @@ def width_bucket( Parameters ---------- v : :class:`~pyspark.sql.Column` or column name - value to compute a bucket number in the histogram + value to compute a bucket number in the histogram. + A column that evaluates to a double or interval. min : :class:`~pyspark.sql.Column` or column name - minimum value of the histogram + minimum value of the histogram. + A column that evaluates to a double or interval. max : :class:`~pyspark.sql.Column` or column name - maximum value of the histogram + maximum value of the histogram. + A column that evaluates to a double or interval. numBucket : :class:`~pyspark.sql.Column`, column name or int - the number of buckets + the number of buckets. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` the bucket number into which the value would fall after being evaluated + Returns a column that evaluates to a long. Examples -------- @@ -6809,6 +6963,7 @@ def coalesce(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` value of the first column that is not null. + Returns a column of the same type as the input. Examples -------- @@ -6858,8 +7013,10 @@ def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate correlation. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate correlation. + A column that evaluates to a double. Returns ------- @@ -6896,8 +7053,10 @@ def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a double. Returns ------- @@ -6938,8 +7097,10 @@ def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a double. Returns ------- @@ -7088,8 +7249,10 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch first value for. + A column of any type. ignorenulls : bool if first value is null then look for first non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7354,11 +7517,13 @@ def isnan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` True if value is NaN and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7393,6 +7558,7 @@ def isnull(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` True if value is null and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7430,8 +7596,10 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch last value for. + A column of any type. ignorenulls : bool if last value is null then look for non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7533,13 +7701,16 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to check. + A column that evaluates to a double or float. col2 : :class:`~pyspark.sql.Column` or column name second column to return if first is NaN. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` value from first column or second if first is NaN . + Returns a column that evaluates to a double. Examples -------- @@ -7772,11 +7943,13 @@ def rand(seed: Optional[int] = None) -> Column: ---------- seed : int, optional Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7834,11 +8007,13 @@ def randn(seed: Optional[int] = None) -> Column: ---------- seed : int (default: None) Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7890,8 +8065,10 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7900,6 +8077,7 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -7946,8 +8124,10 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7956,6 +8136,7 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -8016,13 +8197,16 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column that evaluates to an integer. Examples -------- @@ -8071,13 +8255,16 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted values. + Returns a column that evaluates to an integer. Examples -------- @@ -8126,13 +8313,16 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column that evaluates to an integer. Examples -------- @@ -8254,6 +8444,7 @@ def struct( ------- :class:`~pyspark.sql.Column` a struct type column of given columns. + Returns a column that evaluates to a struct. See Also -------- @@ -8330,6 +8521,7 @@ def greatest(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` greatest value. + Returns a column of the same type as the input. See Also -------- @@ -8374,6 +8566,7 @@ def least(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` least value. + Returns a column of the same type as the input. See Also -------- @@ -8420,6 +8613,7 @@ def when(condition: Column, value: Any) -> Column: ------- :class:`~pyspark.sql.Column` column representing when expression. + Returns a column of the same type as the input. See Also -------- @@ -8486,14 +8680,17 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non Parameters ---------- arg1 : :class:`~pyspark.sql.Column`, str or float - base number or actual number (in this case base is `e`) + base number or actual number (in this case base is `e`). + A column that evaluates to a double. arg2 : :class:`~pyspark.sql.Column`, str or float, optional number to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8560,11 +8757,13 @@ def ln(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8605,11 +8804,13 @@ def log2(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. Examples -------- @@ -8647,15 +8848,19 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to convert base for. + A column that evaluates to a string. fromBase: int from base number. + A column that evaluates to an integer. toBase: int to base number. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a string. Examples -------- @@ -8691,11 +8896,13 @@ def factorial(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate factorial for. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` factorial of given value. + Returns a column that evaluates to a long. Examples -------- @@ -8990,8 +9197,10 @@ def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9036,8 +9245,10 @@ def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9089,8 +9300,10 @@ def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9142,6 +9355,7 @@ def count_if(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a boolean. Returns ------- @@ -9604,8 +9818,10 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to format. + A column that evaluates to a timestamp or time. format: literal string format to use to represent datetime values. + A column that evaluates to a string. See Also -------- @@ -9620,6 +9836,7 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ------- :class:`~pyspark.sql.Column` string value representing formatted datetime. + Returns a column that evaluates to a string. Examples -------- @@ -9696,11 +9913,13 @@ def year(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` year part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9787,11 +10006,13 @@ def quarter(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` quarter of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9878,11 +10099,13 @@ def month(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` month part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9971,11 +10194,13 @@ def dayofweek(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the week for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10057,6 +10282,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. See Also -------- @@ -10069,6 +10295,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10140,11 +10367,13 @@ def day(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10236,11 +10465,13 @@ def dayofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the year for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10326,11 +10557,13 @@ def hour(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` hour part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10408,6 +10641,7 @@ def minute(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. See Also -------- @@ -10425,6 +10659,7 @@ def minute(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` minutes part of the timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10490,11 +10725,13 @@ def second(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` `seconds` part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10571,11 +10808,13 @@ def weekofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` `week` of the year for given date as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10654,11 +10893,13 @@ def weekday(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + Returns a column that evaluates to an integer. See Also -------- @@ -10738,11 +10979,13 @@ def monthname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of month name for date/timestamp (Jan, Feb, Mar...) + Returns a column that evaluates to a string. See Also -------- @@ -10819,11 +11062,13 @@ def dayname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of day name for date/timestamp (Mon, Tue, Wed...) + Returns a column that evaluates to a string. See Also -------- @@ -10907,6 +11152,7 @@ def extract(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -10962,6 +11208,7 @@ def date_part(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -11017,6 +11264,7 @@ def datepart(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -11066,16 +11314,20 @@ def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") Parameters ---------- year : :class:`~pyspark.sql.Column` or column name - The year to build the date + The year to build the date. + A column that evaluates to an integer. month : :class:`~pyspark.sql.Column` or column name - The month to build the date + The month to build the date. + A column that evaluates to an integer. day : :class:`~pyspark.sql.Column` or column name - The day to build the date + The day to build the date. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date built from given parts. + Returns a column that evaluates to a date. See Also -------- @@ -11112,14 +11364,17 @@ def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11172,14 +11427,17 @@ def dateadd(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11235,14 +11493,17 @@ def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days before the given date to calculate. Accepts negative value as well to calculate forward in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date before/after given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11295,13 +11556,16 @@ def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11343,13 +11607,16 @@ def date_diff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11392,11 +11659,13 @@ def date_from_unix_date(days: "ColumnOrName") -> Column: ---------- days : :class:`~pyspark.sql.Column` or column name the target column to work on. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the date from the number of days since 1970-01-01. + Returns a column that evaluates to a date. See Also -------- @@ -11434,14 +11703,17 @@ def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Col ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. months : :class:`~pyspark.sql.Column` or column name or int how many months after the given date to calculate. Accepts negative value as well to calculate backwards. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of months. + Returns a column that evaluates to a date. See Also -------- @@ -11496,15 +11768,19 @@ def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool ---------- date1 : :class:`~pyspark.sql.Column` or column name first date column. + A column that evaluates to a timestamp. date2 : :class:`~pyspark.sql.Column` or column name second date column. + A column that evaluates to a timestamp. roundOff : bool, optional whether to round (to 8 digits) the final value or not (default: True). + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` number of months between two dates. + Returns a column that evaluates to a double. Examples -------- @@ -11556,13 +11832,16 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, timestamp, or timestamp_ntz. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11613,13 +11892,16 @@ def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, timestamp, or timestamp_ntz. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11675,11 +11957,13 @@ def unix_date(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the number of days since 1970-01-01. + Returns a column that evaluates to an integer. See Also -------- @@ -11718,11 +12002,13 @@ def unix_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of microseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11762,11 +12048,13 @@ def unix_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of milliseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11857,11 +12145,13 @@ def unix_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of seconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11914,13 +12204,16 @@ def to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Col ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a string. See Also -------- @@ -11983,6 +12276,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -11990,6 +12284,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ------- :class:`~pyspark.sql.Column` timestamp value as :class:`pyspark.sql.types.TimestampType` type. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12056,13 +12351,16 @@ def try_to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a string. See Also -------- @@ -12123,6 +12421,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -12518,15 +12817,18 @@ def trunc(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a date. format : literal string 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - Other options are: 'week', 'quarter' + Other options are: 'week', 'quarter'. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` truncated date. + Returns a column that evaluates to a date. See Also -------- @@ -12573,14 +12875,17 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column: 'month', 'mon', 'mm' to truncate by month, 'day', 'dd' to truncate by day, Other options are: - 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter' + 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'. + A column that evaluates to a string. timestamp : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` truncated timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12625,14 +12930,17 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. dayOfWeek : literal string day of the week, case-insensitive, accepts: - "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column of computed results. + Returns a column that evaluates to a date. Examples -------- @@ -12671,11 +12979,13 @@ def last_day(date: "ColumnOrName") -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` last day of the month. + Returns a column that evaluates to a date. Examples -------- @@ -12716,13 +13026,16 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss" ---------- timestamp : :class:`~pyspark.sql.Column` or column name column of unix time values. + A column that evaluates to a long. format : literal string, optional - format to use to convert to (default: yyyy-MM-dd HH:mm:ss) + format to use to convert to (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` formatted timestamp as string. + Returns a column that evaluates to a string. See Also -------- @@ -12777,13 +13090,16 @@ def unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name, optional timestamps of string values. + A column that evaluates to a string, date, timestamp, or timestamp_ntz. format : literal string, optional alternative format to use for converting (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` unix time as long integer. + Returns a column that evaluates to a long. Examples -------- @@ -12854,7 +13170,8 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12862,6 +13179,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12870,6 +13188,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col ------- :class:`~pyspark.sql.Column` timestamp value represented in given timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12923,7 +13242,8 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12931,6 +13251,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4.0 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12939,6 +13260,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum ------- :class:`~pyspark.sql.Column` timestamp value represented in UTC timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12983,11 +13305,13 @@ def timestamp_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13027,15 +13351,19 @@ def time_diff(unit: "ColumnOrName", start: "ColumnOrName", end: "ColumnOrName") unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or column name A starting time. + A column that evaluates to a time. end : :class:`~pyspark.sql.Column` or column name An ending time. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` The difference between two times, in the specified units. + Returns a column that evaluates to a long. See Also -------- @@ -13070,13 +13398,16 @@ def time_trunc(unit: "ColumnOrName", time: "ColumnOrName") -> Column: unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. time : :class:`~pyspark.sql.Column` or column name A time to truncate. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` A time truncated to the specified unit. + Returns a column that evaluates to a time. See Also -------- @@ -13110,11 +13441,13 @@ def timestamp_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13152,11 +13485,13 @@ def timestamp_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13195,6 +13530,7 @@ def timestamp_nanos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name a column of ``BIGINT`` or ``DECIMAL`` nanosecond values since the UTC epoch. + A column that evaluates to an integral or decimal. Returns ------- @@ -13639,6 +13975,7 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str]) ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a struct. See Also -------- @@ -13698,8 +14035,10 @@ def to_unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, timestamp, or timestamp_ntz. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. + A column that evaluates to a string. See Also -------- @@ -13771,6 +14110,7 @@ def to_timestamp_ltz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. @@ -13842,8 +14182,10 @@ def to_timestamp_ntz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, timestamp, or timestamp_ntz. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. + A column that evaluates to a string. See Also -------- @@ -14133,6 +14475,7 @@ def crc32(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- @@ -14140,6 +14483,7 @@ def crc32(col: "ColumnOrName") -> Column: the column for computed results. .. versionadded:: 1.5.0 + Returns a column that evaluates to a long. Examples -------- @@ -14168,11 +14512,13 @@ def md5(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. Examples -------- @@ -14201,11 +14547,13 @@ def sha1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14241,14 +14589,17 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. numBits : int the desired bit length of the result, which must have a value of 224, 256, 384, 512, or 0 (which is equivalent to 256). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14298,6 +14649,7 @@ def hash(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as int column. + Returns a column that evaluates to an integer. See Also -------- @@ -14350,6 +14702,7 @@ def xxhash64(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as long column. + Returns a column that evaluates to a long. See Also -------- @@ -14397,14 +14750,17 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column name or column that represents the input column to test + column name or column that represents the input column to test. + A column that evaluates to a boolean. errMsg : :class:`~pyspark.sql.Column` or literal string, optional - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` `null` if the input column is `true` otherwise throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14468,12 +14824,14 @@ def raise_error(errMsg: Union[Column, str]) -> Column: Parameters ---------- errMsg : :class:`~pyspark.sql.Column` or literal string - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14517,11 +14875,13 @@ def upper(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` upper case values. + Returns a column that evaluates to a string. See Also -------- @@ -14558,11 +14918,13 @@ def lower(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` lower case values. + Returns a column that evaluates to a string. See Also -------- @@ -14599,11 +14961,13 @@ def ascii(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` numeric value. + Returns a column that evaluates to an integer. Examples -------- @@ -14635,11 +14999,13 @@ def base64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` BASE64 encoding of string value. + Returns a column that evaluates to a string. See Also -------- @@ -14675,11 +15041,13 @@ def unbase64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` encoded string value. + Returns a column that evaluates to a binary. See Also -------- @@ -14715,8 +15083,10 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14724,6 +15094,7 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` left trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14790,8 +15161,10 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14799,6 +15172,7 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` right trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14865,8 +15239,10 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14874,6 +15250,7 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` trimmed values from both sides. + Returns a column that evaluates to a string. See Also -------- @@ -14941,6 +15318,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ---------- sep : literal string words separator. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name list of columns to work on. @@ -14948,6 +15326,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` string of concatenated words. + Returns a column that evaluates to a string. See Also -------- @@ -15028,13 +15407,16 @@ def encode(col: "ColumnOrName", charset: str) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. charset : literal string charset to use to encode. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a binary. See Also -------- @@ -15067,11 +15449,13 @@ def is_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` whether the input string is a valid UTF-8 string. + Returns a column that evaluates to a boolean. See Also -------- @@ -15104,11 +15488,13 @@ def make_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the valid UTF-8 version of the given input string. + Returns a column that evaluates to a string. See Also -------- @@ -15140,11 +15526,13 @@ def validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, error otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15176,11 +15564,13 @@ def try_validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, null otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15215,14 +15605,17 @@ def format_number(col: "ColumnOrName", d: int) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - the column name of the numeric value to be formatted + the column name of the numeric value to be formatted. + A column that evaluates to a numeric. d : int - the N decimal places + the N decimal places. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. Examples -------- @@ -15253,7 +15646,8 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: Parameters ---------- format : literal string - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -15261,6 +15655,7 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. See Also -------- @@ -15313,21 +15708,26 @@ def instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. substr : :class:`~pyspark.sql.Column` or literal string substring to look for. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `substr` now accepts column. start : int or :class:`~pyspark.sql.Column`, optional Starting position (1-based, can be negative for backward search). If not specified, defaults to 1. + A column that evaluates to an integer. occurrence : int or :class:`~pyspark.sql.Column`, optional Which occurrence to locate (must be > 0). Defaults to 1. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` location of the substring as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -15416,19 +15816,24 @@ def overlay( Parameters ---------- src : :class:`~pyspark.sql.Column` or column name - the string that will be replaced + the string that will be replaced. + A column that evaluates to a string or binary. replace : :class:`~pyspark.sql.Column` or column name - the substitution string + the substitution string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int - the starting position in src + the starting position in src. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name or int, optional the number of bytes to replace in src - string by 'replace' defaults to -1, which represents the length of the 'replace' string + string by 'replace' defaults to -1, which represents the length of the 'replace' string. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` string with replaced values. + Returns a column that evaluates to a string. Examples -------- @@ -15516,16 +15921,20 @@ def sentences( Parameters ---------- string : :class:`~pyspark.sql.Column` or column name - a string to be split + a string to be split. + A column that evaluates to a string. language : :class:`~pyspark.sql.Column` or column name, optional - a language of the locale + a language of the locale. + A column that evaluates to a string. country : :class:`~pyspark.sql.Column` or column name, optional - a country of the locale + a country of the locale. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` arrays of split sentences. + Returns a column that evaluates to an array. See Also -------- @@ -15589,14 +15998,17 @@ def substring( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int starting position in str. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pos` now accepts column and column name. len : :class:`~pyspark.sql.Column` or column name or int length of chars. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `len` now accepts column and column name. @@ -15605,6 +16017,7 @@ def substring( ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -15694,15 +16107,19 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. delim : literal string delimiter of values. + A column that evaluates to a string. count : int number of occurrences. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column that evaluates to a string. See Also -------- @@ -15752,11 +16169,14 @@ def levenshtein( ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. threshold : int, optional if set when the levenshtein distance of the two given strings - less than or equal to a given threshold then return result distance, or -1 + less than or equal to a given threshold then return result distance, or -1. + A column that evaluates to an integer. .. versionadded:: 3.5.0 @@ -15764,6 +16184,7 @@ def levenshtein( ------- :class:`~pyspark.sql.Column` Levenshtein distance as integer value. + Returns a column that evaluates to an integer. Examples -------- @@ -15805,13 +16226,16 @@ def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Colu ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` Jaro-Winkler similarity as a double value. + Returns a column that evaluates to a double. Examples -------- @@ -15840,16 +16264,20 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: Parameters ---------- substr : literal string - a string + a string. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or column name - a Column of :class:`pyspark.sql.types.StringType` + a Column of :class:`pyspark.sql.types.StringType`. + A column that evaluates to a string. pos : int, optional - start position (zero based) + start position (zero based). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` position of the substring. + Returns a column that evaluates to an integer. Notes ----- @@ -15907,14 +16335,17 @@ def lpad( ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15923,6 +16354,7 @@ def lpad( ------- :class:`~pyspark.sql.Column` left padded result. + Returns a column that evaluates to a string. See Also -------- @@ -15977,14 +16409,17 @@ def rpad( ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15993,6 +16428,7 @@ def rpad( ------- :class:`~pyspark.sql.Column` right padded result. + Returns a column that evaluates to a string. See Also -------- @@ -16043,8 +16479,10 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. n : :class:`~pyspark.sql.Column` or column name or int number of times to repeat value. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `n` now accepts column and column name. @@ -16053,6 +16491,7 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ------- :class:`~pyspark.sql.Column` string with repeated values. + Returns a column that evaluates to a string. Examples -------- @@ -16108,10 +16547,12 @@ def split( Parameters ---------- str : :class:`~pyspark.sql.Column` or column name - a string expression to split + a string expression to split. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or literal string a string representing a regular expression. The regex string should be a Java regular expression. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `pattern` now accepts column. Does not accept column name since string type remain @@ -16120,6 +16561,7 @@ def split( limit : :class:`~pyspark.sql.Column` or column name or int an integer which controls the number of times `pattern` is applied. + A column that evaluates to an integer. * ``limit > 0``: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last @@ -16134,6 +16576,7 @@ def split( ------- :class:`~pyspark.sql.Column` array of separated strings. + Returns a column that evaluates to an array. See Also -------- @@ -16207,13 +16650,16 @@ def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16260,13 +16706,16 @@ def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16313,13 +16762,16 @@ def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16368,13 +16820,16 @@ def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = Non ---------- length : :class:`~pyspark.sql.Column` or int Number of characters in the string to generate. + A column that evaluates to an integer. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random string with the specified length. + Returns a column that evaluates to a string. See Also -------- @@ -16421,13 +16876,16 @@ def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the number of times that a Java regex pattern is matched in the string. + Returns a column that evaluates to an integer. Examples -------- @@ -16478,15 +16936,19 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. pattern : str regex pattern to apply. + A column that evaluates to a string. idx : int matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` matched value specified by `idx` group id. + Returns a column that evaluates to a string. See Also -------- @@ -16539,15 +17001,19 @@ def regexp_extract_all( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` all strings in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an array. See Also -------- @@ -16617,18 +17083,23 @@ def regexp_replace( Parameters ---------- string : :class:`~pyspark.sql.Column` or str - column name or column containing the string value + column name or column containing the string value. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str - column object or str containing the regexp pattern + column object or str containing the regexp pattern. + A column that evaluates to a string. replacement : :class:`~pyspark.sql.Column` or str - column object or str containing the replacement + column object or str containing the replacement. + A column that evaluates to a string. position : :class:`~pyspark.sql.Column` or int, optional position to start replacement. The first position is 1. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` string with all substrings replaced. + Returns a column that evaluates to a string. Examples -------- @@ -16697,13 +17168,16 @@ def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the first substring that matches a Java regex within the string `str`. + Returns a column that evaluates to a string. Examples -------- @@ -16766,16 +17240,20 @@ def regexp_instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the position of the first substring in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an integer. Examples -------- @@ -16841,11 +17319,13 @@ def initcap(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string with all first letters are uppercase in each word. + Returns a column that evaluates to a string. Examples -------- @@ -16875,11 +17355,13 @@ def soundex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` SoundEx encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -16909,11 +17391,13 @@ def bin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` binary representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16952,6 +17436,7 @@ def hex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long, binary, or string. See Also -------- @@ -16961,6 +17446,7 @@ def hex(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hexadecimal representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16990,6 +17476,7 @@ def unhex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. See Also -------- @@ -16999,6 +17486,7 @@ def unhex(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` string representation of given hexadecimal value. + Returns a column that evaluates to a binary. Examples -------- @@ -17032,15 +17520,19 @@ def uniform( ---------- min : :class:`~pyspark.sql.Column`, int, or float Minimum value in the range. + A column that evaluates to a numeric. Must be a constant. max : :class:`~pyspark.sql.Column`, int, or float Maximum value in the range. + A column that evaluates to a numeric. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random number within the specified range. + Returns a column of the same type as the input. See Also -------- @@ -17093,11 +17585,13 @@ def length(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` length of the value. + Returns a column that evaluates to an integer. See Also -------- @@ -17130,12 +17624,14 @@ def octet_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Byte length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17165,12 +17661,14 @@ def bit_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Bit length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17202,17 +17700,21 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: Parameters ---------- srcCol : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string. matching : str matching characters. + A column that evaluates to a string. replace : str characters for replacement. If this is shorter than `matching` string then those chars that don't have replacement will be dropped. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` replaced value. + Returns a column that evaluates to a string. Examples -------- @@ -17247,8 +17749,10 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17309,9 +17813,11 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a decimal, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17355,9 +17861,11 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a decimal, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17397,8 +17905,10 @@ def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17426,11 +17936,14 @@ def replace( ---------- src : :class:`~pyspark.sql.Column` or str A column of string to be replaced. + A column that evaluates to a string. search : :class:`~pyspark.sql.Column` or str A column of string, If `search` is not found in `str`, `str` is returned unchanged. + A column that evaluates to a string. replace : :class:`~pyspark.sql.Column` or str, optional A column of string, If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + A column that evaluates to a string. Examples -------- @@ -17462,10 +17975,13 @@ def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnO ---------- src : :class:`~pyspark.sql.Column` or column name A column of string to be split. + A column that evaluates to a string. delimiter : :class:`~pyspark.sql.Column` or column name A column of string, the delimiter used for split. + A column that evaluates to a string. partNum : :class:`~pyspark.sql.Column` or column name A column of string, requested part of the split (1-based). + A column that evaluates to an integer. See Also -------- @@ -17507,15 +18023,19 @@ def substr( ---------- str : :class:`~pyspark.sql.Column` or column name A column of string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name A column of string, the substring of `str` that starts at `pos`. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name, optional A column of string, the substring of `str` is of length `len`. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -17563,15 +18083,19 @@ def try_parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17679,15 +18203,19 @@ def parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17777,7 +18305,8 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: Parameters ---------- format : :class:`~pyspark.sql.Column` or str - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -17815,11 +18344,13 @@ def url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17893,11 +18424,13 @@ def try_url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17938,11 +18471,13 @@ def url_encode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a string to be URL-encoded. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the URL-encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -18018,10 +18553,13 @@ def position( ---------- substr : :class:`~pyspark.sql.Column` or str A column of string, substring. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or str A column of string. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or str, optional A column of string, start position. + A column that evaluates to an integer. Examples -------- @@ -18062,9 +18600,11 @@ def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. suffix : :class:`~pyspark.sql.Column` or str - A column of string, the suffix. + The suffix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18100,9 +18640,11 @@ def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. prefix : :class:`~pyspark.sql.Column` or str - A column of string, the prefix. + The prefix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18138,6 +18680,7 @@ def char(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a long. Examples -------- @@ -18163,8 +18706,10 @@ def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. Examples -------- @@ -18195,6 +18740,7 @@ def char_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18227,6 +18773,7 @@ def character_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18258,6 +18805,7 @@ def chr(n: "ColumnOrName") -> Column: ---------- n : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a long. Examples -------- @@ -18293,8 +18841,10 @@ def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18351,8 +18901,10 @@ def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18402,9 +18954,11 @@ def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or str - The input column or strings to check, may be NULL. + The input to check; may be NULL. + A column that evaluates to a string or binary. right : :class:`~pyspark.sql.Column` or str - The input column or strings to find, may be NULL. + The value to find; may be NULL. + A column that evaluates to a string or binary. Examples -------- @@ -18468,8 +19022,10 @@ def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str The given string to be found. + A column that evaluates to a string. str_array : :class:`~pyspark.sql.Column` or str The comma-delimited list. + A column that evaluates to a string. Examples -------- @@ -18495,6 +19051,7 @@ def like( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18506,6 +19063,7 @@ def like( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18545,6 +19103,7 @@ def ilike( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18556,6 +19115,7 @@ def ilike( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18591,6 +19151,7 @@ def lcase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18622,6 +19183,7 @@ def ucase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18654,8 +19216,10 @@ def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or str Input column or strings, the leftmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18678,8 +19242,10 @@ def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. len : :class:`~pyspark.sql.Column` or str Input column or strings, the rightmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18708,18 +19274,24 @@ def mask( ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. upperChar: :class:`~pyspark.sql.Column` or str, optional character to replace upper-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. lowerChar: :class:`~pyspark.sql.Column` or str, optional character to replace lower-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. digitChar: :class:`~pyspark.sql.Column` or str, optional character to replace digit characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. otherChar: :class:`~pyspark.sql.Column` or str, optional character to replace all other characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` + Returns a column that evaluates to a string. Examples -------- @@ -18780,11 +19352,13 @@ def collation(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Target string column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` collation name of a given expression. + Returns a column that evaluates to a string. Examples -------- @@ -18810,11 +19384,13 @@ def quote(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to be quoted. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` quoted string + Returns a column that evaluates to a string. Examples -------- @@ -18942,13 +19518,16 @@ def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing a set of keys. All elements should not be null. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing a set of values. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A column of map type. + Returns a column that evaluates to a map. Notes ----- @@ -19028,6 +19607,7 @@ def array( :class:`~pyspark.sql.Column` A new Column of array type, where each value is an array containing the corresponding values from the input columns. + Returns a column that evaluates to an array. Examples -------- @@ -19118,14 +19698,17 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The target column containing the arrays. + A column that evaluates to an array. value : The value or column to check for in the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding array from the input column contains the specified value. + Returns a column that evaluates to a boolean. Examples -------- @@ -19205,6 +19788,7 @@ def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding arrays from the input columns contain any common elements. + Returns a column that evaluates to a boolean. Examples -------- @@ -19277,17 +19861,21 @@ def slice( ---------- x : :class:`~pyspark.sql.Column` or str Input array column or column name to be sliced. + A column that evaluates to an array. start : :class:`~pyspark.sql.Column`, str, or int The start index for the slice operation. If negative, starts the index from the end of the array. + A column that evaluates to an integer. length : :class:`~pyspark.sql.Column`, str, or int The length of the slice, representing number of elements in the resulting array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new Column object of Array type, where each value is a slice of the corresponding list from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -19354,16 +19942,20 @@ def array_join( ---------- col : :class:`~pyspark.sql.Column` or str The input column containing the arrays to be joined. + A column that evaluates to an array. delimiter : str The string to be used as the delimiter when joining the array elements. + A column that evaluates to a string. null_replacement : str, optional The string to replace null values within the array. If not set, null values are ignored. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of string type, where each value is the result of joining the corresponding array from the input column. + Returns a column that evaluates to a string. Examples -------- @@ -19461,6 +20053,7 @@ def concat(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` concatenated values. Type of the `Column` depends on input columns' type. + Returns a column of the same type as the input. See Also -------- @@ -19548,8 +20141,10 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to an array. value : Any value or a :class:`~pyspark.sql.Column` expression to look for. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `value` now also accepts a Column type. @@ -19558,6 +20153,7 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ------- :class:`~pyspark.sql.Column` position of the value in the given array if found and 0 otherwise. + Returns a column that evaluates to a long. Examples -------- @@ -19653,14 +20249,17 @@ def element_at(col: "ColumnOrName", extraction: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` value at given position. + Returns a column of the element type of the input array. Notes ----- @@ -19749,9 +20348,16 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. + + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column of the element type of the input array. Notes ----- @@ -19848,13 +20454,16 @@ def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column containing the array. + A column that evaluates to an array. index : :class:`~pyspark.sql.Column` or str or int Index to check for in the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` Value at the given position. + Returns a column of the element type of the input array. Notes ----- @@ -19941,14 +20550,17 @@ def array_prepend(col: "ColumnOrName", value: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` an array with the given value prepended. + Returns a column that evaluates to an array. Examples -------- @@ -20031,9 +20643,11 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. element : - element or a :class:`~pyspark.sql.Column` expression to be removed from the array + element or a :class:`~pyspark.sql.Column` expression to be removed from the array. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `element` now also accepts a Column type. @@ -20042,6 +20656,7 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the given value from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20132,12 +20747,14 @@ def array_distinct(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array of unique values from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20217,17 +20834,21 @@ def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: An Parameters ---------- arr : :class:`~pyspark.sql.Column` or str - name of column containing an array + name of column containing an array. + A column that evaluates to an array. pos : :class:`~pyspark.sql.Column` or str or int name of Numeric type column indicating position of insertion - (starting at index 1, negative position is a start from the back of the array) + (starting at index 1, negative position is a start from the back of the array). + A column that evaluates to an integer. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` an array of values, including the new specified value + Returns a column that evaluates to an array. Notes ----- @@ -20313,13 +20934,16 @@ def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the intersection of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20406,13 +21030,16 @@ def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the union of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20499,13 +21126,16 @@ def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the elements present in col1 but not in col2. + Returns a column that evaluates to an array. Notes ----- @@ -20587,12 +21217,14 @@ def array_compact(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the null values from the input column. + Returns a column that evaluates to an array. Notes ----- @@ -20678,13 +21310,16 @@ def array_append(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column containing the array. + A column that evaluates to an array. value : A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new array column with `value` appended to the original array. + Returns a column that evaluates to an array. Notes ----- @@ -20779,6 +21414,7 @@ def explode(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` One row per array item or map key value. + Returns a column of the element type of the input array. See Also -------- @@ -21093,6 +21729,7 @@ def explode_outer(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` one row per array item or map key value. + Returns a column of the element type of the input array. See Also -------- @@ -21275,14 +21912,17 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - string column in json format + string column in json format. + A column that evaluates to a string. path : str - path to the json object to extract + path to the json object to extract. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string representation of given JSON object value. + Returns a column that evaluates to a string. Examples -------- @@ -21355,6 +21995,7 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column: ------- :class:`~pyspark.sql.Column` a new row for each given field value from json object + Returns a column that evaluates to a string. Examples -------- @@ -21508,12 +22149,14 @@ def try_parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21546,6 +22189,7 @@ def to_variant_object( ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21588,12 +22232,14 @@ def parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21952,6 +22598,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or str name of column containing a struct, an array, a map, or a variant object. + A column that evaluates to a struct, array, map, or variant. options : dict, optional options to control converting. accepts the same options as the JSON datasource. See `Data Source Option `_ @@ -21965,6 +22612,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ------- :class:`~pyspark.sql.Column` JSON object as string column. + Returns a column that evaluates to a string. Examples -------- @@ -22125,11 +22773,13 @@ def json_array_length(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` length of json array. + Returns a column that evaluates to an integer. Examples -------- @@ -22153,11 +22803,13 @@ def json_object_keys(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` all the keys of the outermost JSON object. + Returns a column that evaluates to an array. Examples -------- @@ -22620,12 +23272,14 @@ def size(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22650,11 +23304,13 @@ def array_min(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the minimum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22735,11 +23391,13 @@ def array_max(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the maximum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22818,11 +23476,13 @@ def array_size(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the size of each array. + Returns a column that evaluates to an integer. Examples -------- @@ -22900,11 +23560,13 @@ def cardinality(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22940,14 +23602,17 @@ def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column or expression. + A column that evaluates to an array. asc : bool, optional Whether to sort in ascending or descending order. If `asc` is True (default), then the sorting is in ascending order. If False, then in descending order. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` Sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23034,7 +23699,8 @@ def array_sort( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. comparator : callable, optional A binary ``(Column, Column) -> Column: ...``. The comparator will take two @@ -23046,6 +23712,7 @@ def array_sort( ------- :class:`~pyspark.sql.Column` sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23079,6 +23746,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or expression to be shuffled. + A column that evaluates to an array. seed : :class:`~pyspark.sql.Column` or int, optional Seed value for the random generator. @@ -23088,6 +23756,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A new column that contains an array of elements in random order. + Returns a column that evaluates to an array. Notes ----- @@ -23164,12 +23833,14 @@ def reverse(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be reversed. + A column that evaluates to a string, binary, or array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a reversed string, a binary value with bytes in reverse order, or an array with elements in reverse order. + Returns a column of the same type as the input. Examples -------- @@ -23665,14 +24336,17 @@ def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Colu ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be repeated. + A column of any type. count : :class:`~pyspark.sql.Column` or str or int The name of the column, an expression, or an integer that represents the number of times to repeat the element. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an array of repeated elements. + Returns a column that evaluates to an array. Examples -------- @@ -23746,11 +24420,13 @@ def arrays_zip(*cols: "ColumnOrName") -> Column: ---------- cols : :class:`~pyspark.sql.Column` or str Columns of arrays to be merged. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` Merged array of entries. + Returns a column that evaluates to an array. Examples -------- @@ -23931,6 +24607,7 @@ def sequence( ------- :class:`~pyspark.sql.Column` A new column that contains an array of sequence values. + Returns a column that evaluates to an array. Examples -------- @@ -24218,7 +24895,8 @@ def transform( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function a function that is applied to each element of the input array. Can take one of the following forms: @@ -24236,6 +24914,7 @@ def transform( ------- :class:`~pyspark.sql.Column` a new array of transformed elements. + Returns a column that evaluates to an array. Examples -------- @@ -24273,7 +24952,8 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24286,6 +24966,7 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "any" element of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24314,7 +24995,8 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24327,6 +25009,7 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "all" elements of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24370,7 +25053,8 @@ def filter( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function A function that returns the Boolean expression. Can take one of the following forms: @@ -24389,6 +25073,7 @@ def filter( :class:`~pyspark.sql.Column` filtered array of elements where given function evaluated to True when passed as an argument. + Returns a column that evaluates to an array. Examples -------- @@ -24436,12 +25121,14 @@ def aggregate( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``initialValue`` + of the same type as ``initialValue``. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. @@ -24509,15 +25196,19 @@ def reduce( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``zero`` + of the same type as ``zero``. + A column that evaluates to an array. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. + A column of any type. Returns ------- @@ -24579,9 +25270,11 @@ def zip_with( Parameters ---------- left : :class:`~pyspark.sql.Column` or str - name of the first column or expression + name of the first column or expression. + A column that evaluates to an array. right : :class:`~pyspark.sql.Column` or str - name of the second column or expression + name of the second column or expression. + A column that evaluates to an array. f : function a binary function ``(x1: Column, x2: Column) -> Column...`` Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24593,6 +25286,7 @@ def zip_with( ------- :class:`~pyspark.sql.Column` array of calculated values derived by applying given function to each pair of arguments. + Returns a column that evaluates to an array. Examples -------- @@ -24855,15 +25549,19 @@ def str_to_map( ---------- text : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. pairDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split pairs. Default is comma (,). + A column that evaluates to a string. keyValueDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split key/value. Default is colon (:). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of map type where each string in the original column is converted into a map. + Returns a column that evaluates to a map. Examples -------- @@ -25129,15 +25827,19 @@ def convert_timezone( sourceTz : :class:`~pyspark.sql.Column`, optional The time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + A column that evaluates to a string. targetTz : :class:`~pyspark.sql.Column` The time zone to which the input timestamp should be converted. + A column that evaluates to a string. sourceTs : :class:`~pyspark.sql.Column` or column name A timestamp without time zone. + A column that evaluates to a timestamp_ntz or timestamp. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp for converted time zone. + Returns a column that evaluates to a timestamp_ntz. See Also -------- @@ -25199,17 +25901,22 @@ def make_dt_interval( ---------- days : :class:`~pyspark.sql.Column` or column name, optional The number of days, positive or negative. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The number of hours, positive or negative. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The number of minutes, positive or negative. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a DayTimeIntervalType duration. + Returns a column that evaluates to an interval. See Also -------- @@ -25300,6 +26007,7 @@ def try_make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25317,6 +26025,7 @@ def try_make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25467,6 +26176,7 @@ def make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25484,6 +26194,7 @@ def make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25616,15 +26327,19 @@ def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrNam ---------- hour : :class:`~pyspark.sql.Column` or column name The hour to represent, from 0 to 23. + A column that evaluates to an integer. minute : :class:`~pyspark.sql.Column` or column name The minute to represent, from 0 to 59. + A column that evaluates to an integer. second : :class:`~pyspark.sql.Column` or column name The second to represent, from 0 to 59.999999. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A column representing the created time. + Returns a column that evaluates to a time. Examples -------- @@ -25651,6 +26366,7 @@ def time_from_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Seconds since midnight (0 to 86399.999999). + A column that evaluates to a numeric. Examples -------- @@ -25677,6 +26393,7 @@ def time_from_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Milliseconds since midnight (0 to 86399999). + A column that evaluates to an integral. Examples -------- @@ -25703,6 +26420,7 @@ def time_from_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Microseconds since midnight (0 to 86399999999). + A column that evaluates to an integral. Examples -------- @@ -25872,22 +26590,27 @@ def make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -25895,6 +26618,7 @@ def make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -25910,6 +26634,7 @@ def make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26097,22 +26822,27 @@ def try_make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26120,6 +26850,7 @@ def try_make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -26135,6 +26866,7 @@ def try_make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26294,20 +27026,26 @@ def make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or str - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or str - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or str - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or str - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or str - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or str The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or str, optional The time zone identifier. For example, CET, UTC and etc. @@ -26315,6 +27053,7 @@ def make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26391,20 +27130,26 @@ def try_make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC and etc. @@ -26412,6 +27157,7 @@ def try_make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26528,22 +27274,27 @@ def make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26551,6 +27302,7 @@ def make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26564,6 +27316,7 @@ def make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time. + Returns a column that evaluates to a timestamp_ntz. See Also -------- @@ -26683,22 +27436,27 @@ def try_make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26706,6 +27464,7 @@ def try_make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26719,6 +27478,7 @@ def try_make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time, or NULL in case of an error. + Returns a column that evaluates to a timestamp_ntz. See Also -------- @@ -26804,14 +27564,17 @@ def make_ym_interval( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative + The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative + The number of months, positive or negative. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a year-month interval. + Returns a column that evaluates to an interval. See Also -------- @@ -26972,6 +27735,7 @@ def st_geogfromwkb(wkb: "ColumnOrName") -> Column: ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOGRAPHY value. + A column that evaluates to a binary. Examples -------- @@ -26995,8 +27759,10 @@ def st_geomfromwkb( ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOMETRY value. + A column that evaluates to a binary. srid : :class:`~pyspark.sql.Column` or int, optional The optional SRID value of the geometry. Default is 0. + A column that evaluates to an integer. Examples -------- @@ -27276,8 +28042,10 @@ def hll_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an integer, long, string, or binary. lgConfigK : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of K, where K is the number of buckets or slots for the HllSketch + The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. + A column that evaluates to an integer. Returns ------- @@ -27482,9 +28250,11 @@ def theta_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an array, binary, double, float, integer, long, or string. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries, where nominal entries is the size of the sketch - (must be between 4 and 26, defaults to 12) + (must be between 4 and 26, defaults to 12). + A column that evaluates to an integer. Returns ------- @@ -27634,9 +28404,11 @@ def tuple_sketch_agg_double( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing double summary values + The column containing double summary values. + A column that evaluates to a double. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27688,9 +28460,11 @@ def tuple_sketch_agg_integer( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing integer summary values + The column containing integer summary values. + A column that evaluates to an integer. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27741,7 +28515,8 @@ def tuple_union_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27794,7 +28569,8 @@ def tuple_union_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27846,7 +28622,8 @@ def tuple_intersection_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27896,7 +28673,8 @@ def tuple_intersection_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27947,7 +28725,8 @@ def kll_sketch_agg_bigint( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing bigint values to aggregate + The column containing bigint values to aggregate. + A column that evaluates to a byte, integer, long, or short. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28025,7 +28804,8 @@ def kll_sketch_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing double values to aggregate + The column containing double values to aggregate. + A column that evaluates to a float or double. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28191,12 +28971,14 @@ def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28221,12 +29003,14 @@ def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28251,12 +29035,14 @@ def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28281,12 +29067,14 @@ def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28314,12 +29102,14 @@ def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28347,12 +29137,14 @@ def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28380,14 +29172,17 @@ def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL bigint sketch + The first KLL bigint sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL bigint sketch + The second KLL bigint sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28412,14 +29207,17 @@ def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Colum Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL float sketch + The first KLL float sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL float sketch + The second KLL float sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28444,14 +29242,17 @@ def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL double sketch + The first KLL double sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL double sketch + The second KLL double sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28477,14 +29278,17 @@ def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a long. Examples -------- @@ -28513,14 +29317,17 @@ def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a float. Examples -------- @@ -28549,14 +29356,17 @@ def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a double. Examples -------- @@ -28585,14 +29395,17 @@ def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a long or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28621,14 +29434,17 @@ def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a float or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28657,14 +29473,17 @@ def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -29807,7 +30626,9 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any type. Examples -------- @@ -29842,7 +30663,9 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any type. Examples -------- @@ -29877,6 +30700,7 @@ def nullifzero(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column of any type. Examples -------- @@ -30031,26 +30855,33 @@ def aes_encrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to encrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to encrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. iv : :class:`~pyspark.sql.Column` or column name, optional Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or "". 16-byte array for CBC mode. 12-byte array for GCM mode. + A column that evaluates to a binary. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an encrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30170,23 +31001,29 @@ def aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30294,23 +31131,29 @@ def try_aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value or a NULL value. + Returns a column that evaluates to a binary. See Also -------- @@ -30423,6 +31266,7 @@ def sha(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a binary. See Also -------- @@ -30760,6 +31604,7 @@ def bitmap_bit_position(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30793,6 +31638,7 @@ def bitmap_bucket_number(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30827,6 +31673,7 @@ def bitmap_construct_agg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column will most likely be bitmap_bit_position(). + A column that evaluates to a long. See Also -------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index c8b269a63de5e..48e332c5afddd 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -877,14 +877,14 @@ object functions { /** * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * - * @param hour - * the hour to represent, from 0 to 23 - * @param minute - * the minute to represent, from 0 to 59 - * @param second - * the second to represent, from 0 to 59.999999 + * @param hour the hour to represent, from 0 to 23. A column that evaluates to an integer. + * @param minute the minute to represent, from 0 to 59. A column that evaluates to an integer. + * @param second the second to represent, from 0 to 59.999999. A column that evaluates to a + * decimal. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a time. */ def make_time(hour: Column, minute: Column, second: Column): Column = { Column.fn("make_time", hour, minute, second) @@ -2880,6 +2880,8 @@ object functions { * * @group map_funcs * @since 2.4 + * @return + * Returns a column that evaluates to a map. */ def map_from_arrays(keys: Column, values: Column): Column = Column.fn("map_from_arrays", keys, values) @@ -2890,6 +2892,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = Column.fn("str_to_map", text, pairDelim, keyValueDelim) @@ -2953,6 +2957,8 @@ object functions { * * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnan(e: Column): Column = e.isNaN @@ -2961,6 +2967,8 @@ object functions { * * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnull(e: Column): Column = e.isNull @@ -3012,6 +3020,8 @@ object functions { * * @group conditional_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) @@ -3043,6 +3053,8 @@ object functions { * * @group predicate_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def not(e: Column): Column = !e @@ -3055,6 +3067,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(seed: Long): Column = Column.fn("rand", lit(seed)) @@ -3079,6 +3093,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(seed: Long): Column = Column.fn("randn", lit(seed)) @@ -3101,6 +3117,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column): Column = randstr(length, lit(SparkClassUtils.random.nextLong)) @@ -3131,6 +3149,8 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(e: Column): Column = Column.fn("sqrt", e) @@ -3148,6 +3168,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) @@ -3165,6 +3187,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) @@ -3173,6 +3197,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) @@ -3182,6 +3208,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) @@ -3191,6 +3219,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) @@ -3245,6 +3275,8 @@ object functions { * * @group conditional_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def when(condition: Column, value: Any): Column = Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) @@ -3272,6 +3304,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_count(e: Column): Column = Column.fn("bit_count", e) @@ -3281,6 +3315,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos) @@ -3290,6 +3326,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) @@ -3314,12 +3352,15 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def abs(e: Column): Column = Column.fn("abs", e) /** * @return * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3338,6 +3379,7 @@ object functions { /** * @return * inverse hyperbolic cosine of `e` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3356,6 +3398,7 @@ object functions { /** * @return * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3374,6 +3417,7 @@ object functions { /** * @return * inverse hyperbolic sine of `e` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3392,6 +3436,7 @@ object functions { /** * @return * inverse tangent of `e` as if computed by `java.lang.Math.atan` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3408,14 +3453,13 @@ object functions { def atan(columnName: String): Column = atan(Column(columnName)) /** - * @param y - * coordinate on y-axis - * @param x - * coordinate on x-axis + * @param y coordinate on y-axis. A column that evaluates to a double. + * @param x coordinate on x-axis. A column that evaluates to a double. * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by * `java.lang.Math.atan2` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3531,6 +3575,7 @@ object functions { /** * @return * inverse hyperbolic tangent of `e` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3552,6 +3597,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(e: Column): Column = Column.fn("bin", e) @@ -3569,6 +3616,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(e: Column): Column = Column.fn("cbrt", e) @@ -3585,6 +3634,8 @@ object functions { * * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) @@ -3609,6 +3660,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) @@ -3625,15 +3678,17 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def conv(num: Column, fromBase: Int, toBase: Int): Column = Column.fn("conv", num, lit(fromBase), lit(toBase)) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * cosine of the angle, as if computed by `java.lang.Math.cos` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3652,10 +3707,10 @@ object functions { def cos(columnName: String): Column = cos(Column(columnName)) /** - * @param e - * hyperbolic angle + * @param e hyperbolic angle. A column that evaluates to a double. * @return * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3674,10 +3729,10 @@ object functions { def cosh(columnName: String): Column = cosh(Column(columnName)) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * cotangent of the angle + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3685,10 +3740,10 @@ object functions { def cot(e: Column): Column = Column.fn("cot", e) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * cosecant of the angle + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3708,6 +3763,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(e: Column): Column = Column.fn("exp", e) @@ -3724,6 +3781,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(e: Column): Column = Column.fn("expm1", e) @@ -3740,6 +3799,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def factorial(e: Column): Column = Column.fn("factorial", e) @@ -3748,6 +3809,8 @@ object functions { * * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) @@ -3794,6 +3857,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def hex(column: Column): Column = Column.fn("hex", column) @@ -3803,6 +3868,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unhex(column: Column): Column = Column.fn("unhex", column) @@ -3811,6 +3878,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) @@ -3898,6 +3967,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def ln(e: Column): Column = Column.fn("ln", e) @@ -3906,6 +3977,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(e: Column): Column = ln(e) @@ -3938,6 +4011,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(e: Column): Column = Column.fn("log10", e) @@ -3954,6 +4029,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(e: Column): Column = Column.fn("log1p", e) @@ -3970,6 +4047,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(expr: Column): Column = Column.fn("log2", expr) @@ -3986,6 +4065,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def negative(e: Column): Column = Column.fn("negative", e) @@ -4002,6 +4083,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def positive(e: Column): Column = Column.fn("positive", e) @@ -4010,6 +4093,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Column): Column = Column.fn("power", l, r) @@ -4074,6 +4159,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def power(l: Column, r: Column): Column = Column.fn("power", l, r) @@ -4082,6 +4169,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) @@ -4091,6 +4180,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(e: Column): Column = Column.fn("rint", e) @@ -4108,6 +4199,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column): Column = round(e, 0) @@ -4134,6 +4227,8 @@ object functions { * * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column): Column = bround(e, 0) @@ -4156,10 +4251,10 @@ object functions { def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * secant of the angle + * Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4182,6 +4277,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an integer. */ def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) @@ -4201,6 +4298,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an integer. */ def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) @@ -4220,6 +4319,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an integer. */ def shiftrightunsigned(e: Column, numBits: Int): Column = Column.fn("shiftrightunsigned", e, lit(numBits)) @@ -4229,6 +4330,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def sign(e: Column): Column = Column.fn("sign", e) @@ -4237,6 +4340,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(e: Column): Column = Column.fn("signum", e) @@ -4249,10 +4354,10 @@ object functions { def signum(columnName: String): Column = signum(Column(columnName)) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * sine of the angle, as if computed by `java.lang.Math.sin` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4271,10 +4376,10 @@ object functions { def sin(columnName: String): Column = sin(Column(columnName)) /** - * @param e - * hyperbolic angle + * @param e hyperbolic angle. A column that evaluates to a double. * @return * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4293,10 +4398,10 @@ object functions { def sinh(columnName: String): Column = sinh(Column(columnName)) /** - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * tangent of the given value, as if computed by `java.lang.Math.tan` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4315,10 +4420,10 @@ object functions { def tan(columnName: String): Column = tan(Column(columnName)) /** - * @param e - * hyperbolic angle + * @param e hyperbolic angle. A column that evaluates to a double. * @return * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4354,10 +4459,10 @@ object functions { * Converts an angle measured in radians to an approximately equivalent angle measured in * degrees. * - * @param e - * angle in radians + * @param e angle in radians. A column that evaluates to a double. * @return * angle in degrees, as if computed by `java.lang.Math.toDegrees` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4396,10 +4501,10 @@ object functions { * Converts an angle measured in degrees to an approximately equivalent angle measured in * radians. * - * @param e - * angle in degrees + * @param e angle in degrees. A column that evaluates to a double. * @return * angle in radians, as if computed by `java.lang.Math.toRadians` + * Returns a column that evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4425,16 +4530,14 @@ object functions { * evaluated. Note that input arguments must follow conditions listed below; otherwise, the * method will return null. * - * @param v - * value to compute a bucket number in the histogram - * @param min - * minimum value of the histogram - * @param max - * maximum value of the histogram - * @param numBucket - * the number of buckets + * @param v value to compute a bucket number in the histogram. A column that evaluates to a double + * or interval. + * @param min minimum value of the histogram. A column that evaluates to a double or interval. + * @param max maximum value of the histogram. A column that evaluates to a double or interval. + * @param numBucket the number of buckets. A column that evaluates to a long. * @return * the bucket number into which the value would fall after being evaluated + * Returns a column that evaluates to a long. * @group math_funcs * @since 3.5.0 */ @@ -4491,6 +4594,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def md5(e: Column): Column = Column.fn("md5", e) @@ -4500,6 +4605,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha1(e: Column): Column = Column.fn("sha1", e) @@ -4507,13 +4614,13 @@ object functions { * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a * hex string. * - * @param e - * column to compute SHA-2 on. - * @param numBits - * one of 224, 256, 384, or 512. + * @param e column to compute SHA-2 on. A column that evaluates to a binary. + * @param numBits one of 224, 256, 384, or 512. A column that evaluates to an integer. * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha2(e: Column, numBits: Int): Column = { require( @@ -4528,6 +4635,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def crc32(e: Column): Column = Column.fn("crc32", e) @@ -4555,6 +4664,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column): Column = Column.fn("assert_true", c) @@ -4571,6 +4682,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def raise_error(c: Column): Column = Column.fn("raise_error", c) @@ -4596,6 +4709,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) @@ -4618,26 +4733,24 @@ object functions { * for encryption, the identical AAD value must be provided for decryption. The default mode is * GCM. * - * @param input - * The binary value to encrypt. - * @param key - * The passphrase to use to encrypt the data. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. - * @param iv - * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * @param input The binary value to encrypt. A column that evaluates to a binary. + * @param key The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode Specifies which block cipher mode should be used to encrypt messages. Valid modes: + * ECB, GCM, CBC. A column that evaluates to a string. + * @param padding Specifies how to pad messages whose length is not a multiple of the block size. + * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and + * PKCS for CBC. A column that evaluates to a string. + * @param iv Optional initialization vector. Only supported for CBC and GCM modes. Valid values: + * None or "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates + * to a binary. + * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt( input: Column, @@ -4706,23 +4819,21 @@ object functions { * only supported for GCM. If provided for encryption, the identical AAD value must be provided * for decryption. The default mode is GCM. * - * @param input - * The binary value to decrypt. - * @param key - * The passphrase to use to decrypt the data. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * @param input The binary value to decrypt. A column that evaluates to a binary. + * @param key The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode Specifies which block cipher mode should be used to decrypt messages. Valid modes: + * ECB, GCM, CBC. A column that evaluates to a string. + * @param padding Specifies how to pad messages whose length is not a multiple of the block size. + * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and + * PKCS for CBC. A column that evaluates to a string. + * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt( input: Column, @@ -4772,23 +4883,21 @@ object functions { * This is a special version of `aes_decrypt` that performs the same operation, but returns a * NULL value instead of raising an error if the decryption cannot be performed. * - * @param input - * The binary value to decrypt. - * @param key - * The passphrase to use to decrypt the data. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * @param input The binary value to decrypt. A column that evaluates to a binary. + * @param key The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode Specifies which block cipher mode should be used to decrypt messages. Valid modes: + * ECB, GCM, CBC. A column that evaluates to a string. + * @param padding Specifies how to pad messages whose length is not a multiple of the block size. + * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and + * PKCS for CBC. A column that evaluates to a string. + * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt( input: Column, @@ -4839,6 +4948,8 @@ object functions { * * @group hash_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha(col: Column): Column = Column.fn("sha", col) @@ -4900,6 +5011,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def typeof(col: Column): Column = Column.fn("typeof", col) @@ -4922,6 +5035,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column): Column = uniform(min, max, lit(SparkClassUtils.random.nextLong)) @@ -4945,6 +5060,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(seed: Column): Column = Column.fn("random", seed) @@ -4962,6 +5079,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bit_position(col: Column): Column = Column.fn("bitmap_bit_position", col) @@ -4971,6 +5090,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bucket_number(col: Column): Column = Column.fn("bitmap_bucket_number", col) @@ -5021,6 +5142,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def ascii(e: Column): Column = Column.fn("ascii", e) @@ -5030,6 +5153,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def base64(e: Column): Column = Column.fn("base64", e) @@ -5038,6 +5163,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_length(e: Column): Column = Column.fn("bit_length", e) @@ -5073,6 +5200,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def encode(value: Column, charset: String): Column = Column.fn("encode", value, lit(charset)) @@ -5082,6 +5211,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_utf8(str: Column): Column = Column.fn("is_valid_utf8", str) @@ -5092,6 +5223,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def make_valid_utf8(str: Column): Column = Column.fn("make_valid_utf8", str) @@ -5102,6 +5235,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def validate_utf8(str: Column): Column = Column.fn("validate_utf8", str) @@ -5111,6 +5246,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_validate_utf8(str: Column): Column = Column.fn("try_validate_utf8", str) @@ -5124,6 +5261,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) @@ -5145,6 +5284,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def initcap(e: Column): Column = Column.fn("initcap", e) @@ -5158,6 +5299,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: String): Column = instr(str, lit(substring)) @@ -5253,6 +5396,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def length(e: Column): Column = Column.fn("length", e) @@ -5263,6 +5408,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def len(e: Column): Column = Column.fn("len", e) @@ -5271,6 +5418,8 @@ object functions { * * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def lower(e: Column): Column = Column.fn("lower", e) @@ -5279,6 +5428,7 @@ object functions { * to a given threshold. * @return * result distance, or -1 + * Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -5297,6 +5447,8 @@ object functions { * double between 0.0 (no similarity) and 1.0 (identical). * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a double. */ def jaro_winkler_similarity(l: Column, r: Column): Column = Column.fn("jaro_winkler_similarity", l, r) @@ -5310,6 +5462,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) @@ -5332,6 +5486,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) @@ -5358,6 +5514,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column): Column = Column.fn("ltrim", e) @@ -5380,6 +5538,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def octet_length(e: Column): Column = Column.fn("octet_length", e) @@ -5396,6 +5556,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collation(e: Column): Column = Column.fn("collation", e) @@ -5404,6 +5566,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) @@ -5412,6 +5576,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) @@ -5420,6 +5586,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) @@ -5429,6 +5597,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) @@ -5440,6 +5610,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) @@ -5450,6 +5622,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column): Column = Column.fn("regexp_extract_all", str, regexp) @@ -5469,6 +5643,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String): Column = regexp_replace(e, lit(pattern), lit(replacement)) @@ -5508,6 +5684,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) @@ -5518,6 +5696,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) @@ -5538,6 +5718,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unbase64(e: Column): Column = Column.fn("unbase64", e) @@ -5547,6 +5729,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) @@ -5573,6 +5757,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) @@ -5589,6 +5775,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column): Column = Column.fn("rtrim", e) @@ -5611,20 +5799,22 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def soundex(e: Column): Column = Column.fn("soundex", e) /** * Splits str around matches of the given pattern. * - * @param str - * a string expression to split - * @param pattern - * a string representing a regular expression. The regex string should be a Java regular - * expression. + * @param str a string expression to split + * @param pattern a string representing a regular expression. The regex string should be a Java + * regular expression. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) @@ -5693,6 +5883,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Int, len: Int): Column = Column.fn("substring", str, lit(pos), lit(len)) @@ -5717,6 +5909,8 @@ object functions { * right) is returned. substring_index performs a case-sensitive match when searching for delim. * * @group string_funcs + * @return + * Returns a column that evaluates to a string. */ def substring_index(str: Column, delim: String, count: Int): Column = Column.fn("substring_index", str, lit(delim), lit(count)) @@ -5727,6 +5921,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = Column.fn("overlay", src, replace, pos, len) @@ -5745,6 +5941,8 @@ object functions { * Splits a string into arrays of sentences, where each sentence is an array of words. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column, country: Column): Column = Column.fn("sentences", string, language, country) @@ -5773,6 +5971,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def translate(src: Column, matchingString: String, replaceString: String): Column = Column.fn("translate", src, lit(matchingString), lit(replaceString)) @@ -5782,6 +5982,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column): Column = Column.fn("trim", e) @@ -5804,6 +6006,8 @@ object functions { * * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def upper(e: Column): Column = Column.fn("upper", e) @@ -5815,6 +6019,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) @@ -5906,22 +6112,24 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) /** * Replaces all occurrences of `search` with `replace`. * - * @param src - * A column of string to be replaced - * @param search - * A column of string, If `search` is not found in `str`, `str` is returned unchanged. - * @param replace - * A column of string, If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. + * @param src A column of string to be replaced. A column that evaluates to a string. + * @param search A column of string, If `search` is not found in `str`, `str` is returned + * unchanged. A column that evaluates to a string. + * @param replace A column of string, If `replace` is not specified or is an empty string, nothing + * replaces the string that is removed from `str`. A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column, replace: Column): Column = Column.fn("replace", src, search, replace) @@ -5947,6 +6155,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def split_part(str: Column, delimiter: Column, partNum: Column): Column = Column.fn("split_part", str, delimiter, partNum) @@ -5957,6 +6167,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column, len: Column): Column = Column.fn("substr", str, pos, len) @@ -5975,6 +6187,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("try_parse_url", url, partToExtract, key) @@ -5993,6 +6207,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("parse_url", url, partToExtract, key) @@ -6022,6 +6238,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_decode(str: Column): Column = Column.fn("url_decode", str) @@ -6031,6 +6249,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) @@ -6040,6 +6260,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_encode(str: Column): Column = Column.fn("url_encode", str) @@ -6049,6 +6271,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column, start: Column): Column = Column.fn("position", substr, str, start) @@ -6070,6 +6294,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def endswith(str: Column, suffix: Column): Column = Column.fn("endswith", str, suffix) @@ -6081,6 +6307,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def startswith(str: Column, prefix: Column): Column = Column.fn("startswith", str, prefix) @@ -6091,6 +6319,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def char(n: Column): Column = Column.fn("char", n) @@ -6099,6 +6329,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column): Column = Column.fn("btrim", str) @@ -6116,6 +6348,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) @@ -6135,6 +6369,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) @@ -6144,6 +6380,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def char_length(str: Column): Column = Column.fn("char_length", str) @@ -6153,6 +6391,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def character_length(str: Column): Column = Column.fn("character_length", str) @@ -6162,6 +6402,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def chr(n: Column): Column = Column.fn("chr", n) @@ -6172,6 +6414,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) @@ -6194,6 +6438,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) @@ -6203,6 +6449,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("like", str, pattern, escapeChar) @@ -6222,6 +6470,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("ilike", str, pattern, escapeChar) @@ -6240,6 +6490,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def lcase(str: Column): Column = Column.fn("lcase", str) @@ -6248,6 +6500,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def ucase(str: Column): Column = Column.fn("ucase", str) @@ -6257,6 +6511,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def left(str: Column, len: Column): Column = Column.fn("left", str, len) @@ -6266,6 +6522,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def right(str: Column, len: Column): Column = Column.fn("right", str, len) @@ -6275,6 +6533,8 @@ object functions { * * @group string_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def quote(str: Column): Column = Column.fn("quote", str) @@ -7353,6 +7613,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_bigint(e: Column): Column = Column.fn("kll_sketch_to_string_bigint", e) @@ -7362,6 +7624,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_float(e: Column): Column = Column.fn("kll_sketch_to_string_float", e) @@ -7371,6 +7635,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_double(e: Column): Column = Column.fn("kll_sketch_to_string_double", e) @@ -7380,6 +7646,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_bigint(e: Column): Column = Column.fn("kll_sketch_get_n_bigint", e) @@ -7389,6 +7657,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_float(e: Column): Column = Column.fn("kll_sketch_get_n_float", e) @@ -7398,6 +7668,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_double(e: Column): Column = Column.fn("kll_sketch_get_n_double", e) @@ -7407,6 +7679,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_bigint(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_bigint", left, right) @@ -7416,6 +7690,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_float(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_float", left, right) @@ -7425,6 +7701,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_double(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_double", left, right) @@ -7435,6 +7713,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) @@ -7445,6 +7725,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a float. */ def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_float", sketch, rank) @@ -7455,6 +7737,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_double", sketch, rank) @@ -7465,6 +7749,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) @@ -7475,6 +7761,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_float", sketch, quantile) @@ -7485,6 +7773,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_double", sketch, quantile) @@ -7496,13 +7786,14 @@ object functions { /** * Returns the date that is `numMonths` after `startDate`. * - * @param startDate - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months + * @param startDate A date, timestamp or string. If a string, the data must be in a format that + * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a date. + * @param numMonths The number of months to add to `startDate`, can be negative to subtract + * months. A column that evaluates to an integer. * @return * A date, or null if `startDate` was a string that could not be cast to a date + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7585,13 +7876,14 @@ object functions { * See Datetime * Patterns for valid date and time format patterns * - * @param dateExpr - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993` + * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that can + * be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a timestamp or time. + * @param format A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that + * evaluates to a string. * @return * A string, or null if `dateExpr` was a string that could not be cast to a timestamp + * Returns a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a * specialized implementation. @@ -7606,13 +7898,14 @@ object functions { /** * Returns the date that is `days` days after `start` * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param days - * The number of days to add to `start`, can be negative to subtract days + * @param start A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param days The number of days to add to `start`, can be negative to subtract days. A column + * that evaluates to an integer, short, or byte. * @return * A date, or null if `start` was a string that could not be cast to a date + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7636,13 +7929,14 @@ object functions { /** * Returns the date that is `days` days after `start` * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param days - * A column of the number of days to add to `start`, can be negative to subtract days + * @param start A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param days A column of the number of days to add to `start`, can be negative to subtract days. + * A column that evaluates to an integer, short, or byte. * @return * A date, or null if `start` was a string that could not be cast to a date + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.5.0 */ @@ -7651,13 +7945,14 @@ object functions { /** * Returns the date that is `days` days before `start` * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param days - * The number of days to subtract from `start`, can be negative to add days + * @param start A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param days The number of days to subtract from `start`, can be negative to add days. A column + * that evaluates to an integer, short, or byte. * @return * A date, or null if `start` was a string that could not be cast to a date + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7688,15 +7983,16 @@ object functions { * // returns 1 * }}} * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param end A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param start A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a * date. Negative if `end` is before `start` + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7711,15 +8007,16 @@ object functions { * // returns 1 * }}} * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param end A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param start A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a * date. Negative if `end` is before `start` + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7730,6 +8027,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) @@ -7737,6 +8036,7 @@ object functions { * Extracts the year as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7746,6 +8046,7 @@ object functions { * Extracts the quarter as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7755,6 +8056,7 @@ object functions { * Extracts the month as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7765,6 +8067,7 @@ object functions { * for a Sunday through to 7 for a Saturday * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 */ @@ -7774,6 +8077,7 @@ object functions { * Extracts the day of the month as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7783,6 +8087,7 @@ object functions { * Extracts the day of the month as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7792,6 +8097,7 @@ object functions { * Extracts the day of the year as an integer from a given date/timestamp/string. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7803,6 +8109,7 @@ object functions { * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7811,12 +8118,11 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field - * selects which part of the source should be extracted. - * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * @param field selects which part of the source should be extracted. + * @param source a date/timestamp or interval column from where `field` should be extracted. * @return * a part of the date/timestamp or interval source + * Returns a column that evaluates to a double. * @group datetime_funcs * @since 3.5.0 */ @@ -7827,13 +8133,12 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `extract`. - * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * @param field selects which part of the source should be extracted, and supported string values + * are as same as the fields of the equivalent function `extract`. + * @param source a date/timestamp or interval column from where `field` should be extracted. * @return * a part of the date/timestamp or interval source + * Returns a column that evaluates to a double. * @group datetime_funcs * @since 3.5.0 */ @@ -7844,13 +8149,12 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `EXTRACT`. - * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * @param field selects which part of the source should be extracted, and supported string values + * are as same as the fields of the equivalent function `EXTRACT`. + * @param source a date/timestamp or interval column from where `field` should be extracted. * @return * a part of the date/timestamp or interval source + * Returns a column that evaluates to a double. * @group datetime_funcs * @since 3.5.0 */ @@ -7862,11 +8166,12 @@ object functions { * Returns the last day of the month which the given date belongs to. For example, input * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. * - * @param e - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param e A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. * @return * A date, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7878,6 +8183,7 @@ object functions { * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7888,12 +8194,15 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def weekday(e: Column): Column = Column.fn("weekday", e) /** * @return * A date created from year, month and day fields. + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.3.0 */ @@ -7914,15 +8223,14 @@ object functions { * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 * }}} * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param end A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param start A date, timestamp or string. If a string, the data must be in a format that can + * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` * @return * A double, or null if either `end` or `start` were strings that could not be cast to a * timestamp. Negative if `end` is before `start` + * Returns a column that evaluates to a double. * @group datetime_funcs * @since 1.5.0 */ @@ -7945,14 +8253,15 @@ object functions { * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first * Sunday after 2015-07-27. * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + * @param date A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param dayOfWeek Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", + * "Sun". A column that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` * was an invalid value + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7986,6 +8295,7 @@ object functions { * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return * An integer, or null if the input was a string that could not be cast to a timestamp + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7999,6 +8309,7 @@ object functions { * * @return * An integer, or null if the input was a string that could not be cast to a date + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8009,11 +8320,11 @@ object functions { * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd * HH:mm:ss format. * - * @param ut - * A number of a type that is castable to a long, such as string or integer. Can be negative - * for timestamps before the unix epoch + * @param ut A number of a type that is castable to a long, such as string or integer. Can be + * negative for timestamps before the unix epoch * @return * A string, or null if the input was a string that could not be cast to a long + * Returns a column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8050,6 +8361,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_timestamp(): Column = unix_timestamp(current_timestamp()) @@ -8090,10 +8403,10 @@ object functions { /** * Parses a string value to a time value. * - * @param str - * A string to be parsed to time. + * @param str A string to be parsed to time. * @return * A time, or raises an error if the input is malformed. + * Returns a column that evaluates to a string. * * @group datetime_funcs * @since 4.1.0 @@ -8125,11 +8438,12 @@ object functions { /** * Converts to a timestamp by casting rules to `TimestampType`. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param s A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a string, date, timestamp, timestamp_ntz, or numeric. * @return * A timestamp, or null if the input was a string that could not be cast to a timestamp + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8157,10 +8471,10 @@ object functions { /** * Parses a string value to a time value. * - * @param str - * A string to be parsed to time. + * @param str A string to be parsed to time. * @return * A time, or null if the input is malformed. + * Returns a column that evaluates to a string. * * @group datetime_funcs * @since 4.1.0 @@ -8196,6 +8510,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column, format: Column): Column = Column.fn("try_to_timestamp", s, format) @@ -8215,6 +8531,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def to_date(e: Column): Column = Column.fn("to_date", e) @@ -8243,6 +8561,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column): Column = Column.fn("try_to_date", e) @@ -8260,6 +8580,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def unix_date(e: Column): Column = Column.fn("unix_date", e) @@ -8268,6 +8590,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_micros(e: Column): Column = Column.fn("unix_micros", e) @@ -8287,6 +8611,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_millis(e: Column): Column = Column.fn("unix_millis", e) @@ -8296,6 +8622,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) @@ -8304,16 +8632,16 @@ object functions { * * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter' + * @param date A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates + * to a date. + * @param format: 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by + * month Other options are: 'week', 'quarter'. A column that evaluates to a string. * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an * invalid value + * Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8324,16 +8652,16 @@ object functions { * * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 * - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', - * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter' - * @param timestamp - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param format: 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by + * month, 'day', 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', + * 'second', 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. + * @param timestamp A date, timestamp or string. If a string, the data must be in a format that + * can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or * `format` was an invalid value + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8345,18 +8673,19 @@ object functions { * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 * 03:40:00.0'. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * @param ts A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a timestamp. + * @param tz A string detailing the time zone ID that the input should be adjusted to. It should + * be in the format of either region-based zone IDs or zone offsets. Region IDs must have the + * form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was * an invalid value + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8377,18 +8706,19 @@ object functions { * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield * '2017-07-14 01:40:00.0'. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * @param ts A date, timestamp or string. If a string, the data must be in a format that can be + * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that + * evaluates to a timestamp. + * @param tz A string detailing the time zone ID that the input should be adjusted to. It should + * be in the format of either region-based zone IDs or zone offsets. Region IDs must have the + * form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was * an invalid value + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8569,15 +8899,16 @@ object functions { * For a streaming query, you may use the function `current_timestamp` to generate windows on * processing time. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. - * @param gapDuration - * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. + * @param timeColumn The column or the expression to use as the timestamp for windowing by time. + * The time column must be of TimestampType or TimestampNTZType. + * @param gapDuration A string specifying the timeout of the session, e.g. `10 minutes`, `1 + * second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration + * identifiers. * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: String): Column = session_window(timeColumn, lit(gapDuration)) @@ -8622,6 +8953,8 @@ object functions { * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. * @group datetime_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) @@ -8630,6 +8963,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) @@ -8638,6 +8973,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) @@ -8707,15 +9044,14 @@ object functions { * Returns the difference between two times, measured in specified units. Throws a * SparkIllegalArgumentException, in case the specified unit is not supported. * - * @param unit - * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. - * @param start - * A starting TIME. - * @param end - * An ending TIME. + * @param unit A STRING representing the unit of the time difference. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A column + * that evaluates to a string. + * @param start A starting TIME. A column that evaluates to a time. + * @param end An ending TIME. A column that evaluates to a time. * @return * The difference between `end` and `start` times, measured in specified units. + * Returns a column that evaluates to a long. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @group datetime_funcs @@ -8728,13 +9064,13 @@ object functions { /** * Returns `time` truncated to the `unit`. * - * @param unit - * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. - * @param time - * A TIME to truncate. + * @param unit A STRING representing the unit to truncate the time to. Supported units are: + * "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param time A TIME to truncate. A column that evaluates to a time. * @return * A TIME truncated to the specified unit. + * Returns a column that evaluates to a time. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @throws IllegalArgumentException @@ -8751,6 +9087,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) @@ -8759,6 +9097,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) @@ -8767,6 +9107,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) @@ -8801,6 +9143,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ltz", timestamp, format) @@ -8821,6 +9165,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def to_timestamp_ntz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ntz", timestamp, format) @@ -8840,6 +9186,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column, format: Column): Column = Column.fn("to_unix_timestamp", timeExp, format) @@ -8858,6 +9206,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def monthname(timeExp: Column): Column = Column.fn("monthname", timeExp) @@ -8867,6 +9217,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def dayname(timeExp: Column): Column = Column.fn("dayname", timeExp) @@ -8879,6 +9231,8 @@ object functions { * Returns null if the array is null, true if the array contains `value`, and false otherwise. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def array_contains(column: Column, value: Any): Column = Column.fn("array_contains", column, lit(value)) @@ -8889,6 +9243,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_append(column: Column, element: Any): Column = Column.fn("array_append", column, lit(element)) @@ -8899,6 +9255,8 @@ object functions { * `false` otherwise. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a boolean. */ def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) @@ -8906,15 +9264,14 @@ object functions { * Returns an array containing all the elements in `x` from index `start` (or starting from the * end if `start` is negative) with the specified `length`. * - * @param x - * the array column to be sliced - * @param start - * the starting index - * @param length - * the length of the slice + * @param x the array column to be sliced. A column that evaluates to an array. + * @param start the starting index. A column that evaluates to an integer. + * @param length the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Int, length: Int): Column = slice(x, lit(start), lit(length)) @@ -8941,6 +9298,8 @@ object functions { * `nullReplacement`. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String, nullReplacement: String): Column = Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) @@ -8976,6 +9335,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a long. */ def array_position(column: Column, value: Any): Column = Column.fn("array_position", column, lit(value)) @@ -8986,6 +9347,8 @@ object functions { * * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array. */ def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) @@ -8999,6 +9362,8 @@ object functions { * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the element type of the input array. */ def try_element_at(column: Column, value: Column): Column = Column.fn("try_element_at", column, value) @@ -9009,6 +9374,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column of the element type of the input array. */ def get(column: Column, index: Column): Column = Column.fn("get", column, index) @@ -9023,6 +9390,8 @@ object functions { * * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column): Column = Column.fn("array_sort", e) @@ -9043,6 +9412,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_remove(column: Column, element: Any): Column = Column.fn("array_remove", column, lit(element)) @@ -9052,6 +9423,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_compact(column: Column): Column = Column.fn("array_compact", column) @@ -9061,6 +9434,8 @@ object functions { * * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_prepend(column: Column, element: Any): Column = Column.fn("array_prepend", column, lit(element)) @@ -9069,6 +9444,8 @@ object functions { * Removes duplicate values from the array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_distinct(e: Column): Column = Column.fn("array_distinct", e) @@ -9078,6 +9455,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_intersect(col1: Column, col2: Column): Column = Column.fn("array_intersect", col1, col2) @@ -9087,6 +9466,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_insert(arr: Column, pos: Column, value: Column): Column = Column.fn("array_insert", arr, pos, value) @@ -9096,6 +9477,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_union(col1: Column, col2: Column): Column = Column.fn("array_union", col1, col2) @@ -9106,6 +9489,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_except(col1: Column, col2: Column): Column = Column.fn("array_except", col1, col2) @@ -9138,13 +9523,13 @@ object functions { * df.select(transform(col("i"), x => x + 1)) * }}} * - * @param column - * the input array column - * @param f - * col => transformed_col, the lambda function to transform the input column + * @param column the input array column. A column that evaluates to an array. + * @param f col => transformed_col, the lambda function to transform the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: Column => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9174,13 +9559,13 @@ object functions { * df.select(exists(col("i"), _ % 2 === 0)) * }}} * - * @param column - * the input array column - * @param f - * col => predicate, the Boolean predicate to check the input column + * @param column the input array column. A column that evaluates to an array. + * @param f col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def exists(column: Column, f: Column => Column): Column = Column.fn("exists", column, createLambda(f)) @@ -9191,13 +9576,13 @@ object functions { * df.select(forall(col("i"), x => x % 2 === 0)) * }}} * - * @param column - * the input array column - * @param f - * col => predicate, the Boolean predicate to check the input column + * @param column the input array column. A column that evaluates to an array. + * @param f col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def forall(column: Column, f: Column => Column): Column = Column.fn("forall", column, createLambda(f)) @@ -9208,13 +9593,13 @@ object functions { * df.select(filter(col("s"), x => x % 2 === 0)) * }}} * - * @param column - * the input array column - * @param f - * col => predicate, the Boolean predicate to filter the input column + * @param column the input array column. A column that evaluates to an array. + * @param f col => predicate, the Boolean predicate to filter the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: Column => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9245,19 +9630,17 @@ object functions { * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) * }}} * - * @param expr - * the input array column - * @param initialValue - * the initial value - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * @param expr the input array column. A column that evaluates to an array. + * @param initialValue the initial value. A column of any type. + * @param merge (combined_value, input_value) => combined_value, the merge function to merge an + * input value to the combined_value. + * @param finish combined_value => final_value, the lambda function to convert the combined value + * of all inputs to final result. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate( expr: Column, @@ -9294,19 +9677,17 @@ object functions { * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) * }}} * - * @param expr - * the input array column - * @param initialValue - * the initial value - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * @param expr the input array column. A column that evaluates to an array. + * @param initialValue the initial value. A column of any type. + * @param merge (combined_value, input_value) => combined_value, the merge function to merge an + * input value to the combined_value. + * @param finish combined_value => final_value, the lambda function to convert the combined value + * of all inputs to final result. * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce( expr: Column, @@ -9343,15 +9724,14 @@ object functions { * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) * }}} * - * @param left - * the left input array column - * @param right - * the right input array column - * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column + * @param left the left input array column. A column that evaluates to an array. + * @param right the right input array column. A column that evaluates to an array. + * @param f (lCol, rCol) => col, the lambda function to merge two input columns into one column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = Column.fn("zip_with", left, right, createLambda(f)) @@ -9435,6 +9815,8 @@ object functions { * * @group generator_funcs * @since 1.3.0 + * @return + * Returns a column of the element type of the input array. */ def explode(e: Column): Column = Column.fn("explode", e) @@ -9445,6 +9827,8 @@ object functions { * * @group generator_funcs * @since 2.2.0 + * @return + * Returns a column of the element type of the input array. */ def explode_outer(e: Column): Column = Column.fn("explode_outer", e) @@ -9492,6 +9876,8 @@ object functions { * * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def get_json_object(e: Column, path: String): Column = Column.fn("get_json_object", e, lit(path)) @@ -9702,13 +10088,10 @@ object functions { * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns * `null`, in the case of an unparseable string. * - * @param e - * a string column containing JSON data. - * @param schema - * the schema to use when parsing the json string - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data * Source Option in the version you use. * @@ -9731,21 +10114,23 @@ object functions { * Parses a JSON string and constructs a Variant value. Returns null if the input string is not * a valid JSON value. * - * @param json - * a string column that contains JSON data. + * @param json a string column that contains JSON data. * * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) /** * Parses a JSON string and constructs a Variant value. * - * @param json - * a string column that contains JSON data. + * @param json a string column that contains JSON data. A column that evaluates to a string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def parse_json(json: Column): Column = Column.fn("parse_json", json) @@ -9754,10 +10139,11 @@ object functions { * structs are converted to variant objects which are unordered unlike SQL structs. Input maps * can only have string keys. * - * @param col - * a column with a nested schema or column name. + * @param col a column with a nested schema or column name. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) @@ -9975,15 +10361,14 @@ object functions { /** * Parses a JSON string and infers its schema in DDL format using options. * - * @param json - * a foldable string column containing JSON data. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data * Source Option in the version you use. * @return * a column with string literal containing schema in DDL format. + * Returns a column that evaluates to an integer. * * @group json_funcs * @since 3.0.0 @@ -10008,6 +10393,8 @@ object functions { * * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) @@ -10017,11 +10404,9 @@ object functions { * a JSON string with the specified schema. Throws an exception, in the case of an unsupported * type. * - * @param e - * a column containing a struct, an array or a map. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` * option which enables pretty JSON generation. @@ -10039,11 +10424,9 @@ object functions { * a JSON string with the specified schema. Throws an exception, in the case of an unsupported * type. * - * @param e - * a column containing a struct, an array or a map. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` * option which enables pretty JSON generation. @@ -10059,11 +10442,12 @@ object functions { * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string * with the specified schema. Throws an exception, in the case of an unsupported type. * - * @param e - * a column containing a struct, an array or a map. + * @param e a column containing a struct, an array, a map, or a variant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def to_json(e: Column): Column = to_json(e, Map.empty[String, String]) @@ -10073,11 +10457,12 @@ object functions { * with 'n'. This can be useful for creating copies of tables with sensitive information * removed. * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * @param input string value to mask. Supported types: STRING, VARCHAR, CHAR * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column): Column = Column.fn("mask", input) @@ -10170,6 +10555,8 @@ object functions { * * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def size(e: Column): Column = Column.fn("size", e) @@ -10182,6 +10569,8 @@ object functions { * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def cardinality(e: Column): Column = Column.fn("cardinality", e) @@ -10192,6 +10581,8 @@ object functions { * * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def sort_array(e: Column): Column = sort_array(e, asc = true) @@ -10212,6 +10603,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array. */ def array_min(e: Column): Column = Column.fn("array_min", e) @@ -10221,6 +10614,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array. */ def array_max(e: Column): Column = Column.fn("array_max", e) @@ -10229,6 +10624,8 @@ object functions { * * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def array_size(e: Column): Column = Column.fn("array_size", e) @@ -10251,6 +10648,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) @@ -10269,6 +10668,8 @@ object functions { * Returns a reversed string or an array with reverse order of elements. * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def reverse(e: Column): Column = Column.fn("reverse", e) @@ -10285,6 +10686,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def sequence(start: Column, stop: Column, step: Column): Column = Column.fn("sequence", start, stop, step) @@ -10304,6 +10707,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) @@ -10692,6 +11097,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def xpath(xml: Column, path: Column): Column = Column.fn("xpath", xml, path) @@ -10701,6 +11108,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def xpath_boolean(xml: Column, path: Column): Column = Column.fn("xpath_boolean", xml, path) @@ -10711,6 +11120,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_double(xml: Column, path: Column): Column = Column.fn("xpath_double", xml, path) @@ -10721,6 +11132,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_number(xml: Column, path: Column): Column = Column.fn("xpath_number", xml, path) @@ -10731,6 +11144,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a float. */ def xpath_float(xml: Column, path: Column): Column = Column.fn("xpath_float", xml, path) @@ -10741,6 +11156,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def xpath_int(xml: Column, path: Column): Column = Column.fn("xpath_int", xml, path) @@ -10751,6 +11168,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def xpath_long(xml: Column, path: Column): Column = Column.fn("xpath_long", xml, path) @@ -10761,6 +11180,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a short. */ def xpath_short(xml: Column, path: Column): Column = Column.fn("xpath_short", xml, path) @@ -10770,6 +11191,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def xpath_string(xml: Column, path: Column): Column = Column.fn("xpath_string", xml, path) @@ -10786,15 +11209,16 @@ object functions { * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to * `targetTz`. * - * @param sourceTz - * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. - * @param targetTz - * the time zone to which the input timestamp should be converted. - * @param sourceTs - * a timestamp without time zone. + * @param sourceTz the time zone for the input timestamp. If it is missed, the current session + * time zone is used as the source time zone. A column that evaluates to a string. + * @param targetTz the time zone to which the input timestamp should be converted. A column that + * evaluates to a string. + * @param sourceTs a timestamp without time zone. A column that evaluates to a timestamp_ntz or + * timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) @@ -10817,6 +11241,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = Column.fn("make_dt_interval", days, hours, mins, secs) @@ -10863,6 +11289,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10879,6 +11307,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11043,6 +11473,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11097,6 +11529,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11150,6 +11584,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11184,6 +11620,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11218,6 +11656,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def make_timestamp_ntz( years: Column, @@ -11243,6 +11683,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def try_make_timestamp_ntz( years: Column, @@ -11267,6 +11709,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column, months: Column): Column = Column.fn("make_ym_interval", years, months) @@ -11312,6 +11756,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) @@ -11320,6 +11766,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnotnull(col: Column): Column = Column.fn("isnotnull", col) @@ -11329,6 +11777,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) @@ -11337,6 +11787,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) @@ -11345,6 +11797,8 @@ object functions { * * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def nullifzero(col: Column): Column = Column.fn("nullifzero", col) @@ -11353,6 +11807,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) @@ -11361,6 +11817,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) @@ -11369,6 +11827,8 @@ object functions { * * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala index c226e48c6be5e..adaa340ffd402 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala @@ -30,7 +30,11 @@ import org.apache.spark.sql.internal.SQLConf arguments = """ Arguments: * input - An expression that is being compared with lower and upper bound. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * lower - Lower bound of the between check. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * upper - Upper bound of the between check. """, since = "1.0.0", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala index b167ff71449d0..2732ab1ed43f2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala @@ -58,6 +58,13 @@ case class TryEval(child: Expression) extends UnaryExpression { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sum of `expr1`and `expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `+` operator.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(1, 2); @@ -102,6 +109,13 @@ case class TryAdd(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns `dividend`/`divisor`. It always performs floating point division. Its result is always null if `expr2` is 0. " + "`dividend` must be a numeric or an interval. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a double, decimal, or interval. + * divisor - The expression to divide by. + An expression that evaluates to a double or decimal. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -141,6 +155,13 @@ case class TryDivide(left: Expression, right: Expression, replacement: Expressio @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns the remainder after `expr1`/`expr2`. " + "`dividend` must be a numeric. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a numeric. + * divisor - The expression to divide by. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -177,6 +198,13 @@ case class TryMod(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`-`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `-` operator.", + arguments = """ + Arguments: + * expr1 - The minuend to subtract from. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The subtrahend to subtract. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(2, 1); @@ -219,6 +247,13 @@ case class TrySubtract(left: Expression, right: Expression, replacement: Express @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`*`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `*` operator.", + arguments = """ + Arguments: + * expr1 - The first factor. + An expression that evaluates to a numeric or interval. + * expr2 - The second factor. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -251,6 +286,13 @@ case class TryMultiply(left: Expression, right: Expression, replacement: Express // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str[, fmt]) - This is a special version of `to_binary` that performs the same operation, but returns a NULL value instead of raising an error if the conversion cannot be performed.", + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format of the input string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala index 9fbca1629c95f..e23bd449e8626 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala @@ -34,6 +34,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns some value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return some value of for a group of rows. + An expression of any type. + * isIgnoreNull - Whether to return only non-null values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala index 7ae542f190d56..82874c9e825fa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala @@ -59,6 +59,15 @@ import org.apache.spark.unsafe.types.UTF8String `k` An optional INTEGER literal greater than 0. If k is not specified, it defaults to 5. `maxItemsTracked` An optional INTEGER literal greater than or equal to k and has upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression to compute the top k most frequent items of. + An expression of any type. + * k - The number of top items to return. + An expression that evaluates to an integer. Must be a constant. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(expr) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); @@ -488,6 +497,13 @@ object ApproxTopKAggregateBuffer { _FUNC_(expr, maxItemsTracked) - Accumulates items into a sketch. `maxItemsTracked` An optional positive INTEGER literal with upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression whose values are accumulated into the sketch. + An expression of any type. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT approx_top_k_estimate(_FUNC_(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index fe30e2ea6f3ff..85325068297a8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -145,6 +150,11 @@ case class Average( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 99fb512ea047d..651001fbf2214 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -147,6 +147,11 @@ abstract class CentralMomentAgg(child: Expression, nullOnDivideByZero: Boolean) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population standard deviation of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -178,6 +183,11 @@ case class StddevPop( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample standard deviation of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -210,6 +220,11 @@ case class StddevSamp( // Compute the population variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population variance of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -239,6 +254,11 @@ case class VariancePop( // Compute the sample variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample variance of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -281,6 +301,11 @@ case class RegrReplacement(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the skewness of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); @@ -312,6 +337,11 @@ case class Skewness( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the kurtosis of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index ea154b9bce88c..2635eadd7c5e7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -105,6 +105,13 @@ abstract class PearsonCorrelation(x: Expression, y: Expression, nullOnDivideByZe // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a double. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala index 248ade05ab1d3..f389405ec495f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala @@ -25,6 +25,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType} usage = """ _FUNC_(expr) - Returns the number of `TRUE` values for the expression. """, + arguments = """ + Arguments: + * expr - The boolean expression whose TRUE values are counted. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 8e03daee9b659..bfcf0163f5331 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -90,6 +90,13 @@ abstract class Covariance(val left: Expression, val right: Expression, nullOnDiv @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a double. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); @@ -119,6 +126,13 @@ case class CovPopulation( @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a double. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 9a39a6fe98796..eb39f96ed2ee9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -36,6 +36,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return the first value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the first non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 5840c783cb8be..a0f11ffcb7142 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -35,6 +35,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values""", + arguments = """ + Arguments: + * expr - The expression to return the last value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the last non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala index e2d85cafa52f2..21959a66b2b34 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType} // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - this function is used and can only be used to calculate a measure defined in a metric view.", + arguments = """ + Arguments: + * expr - The measure defined in a metric view to calculate. + An expression of any type. + """, examples = """ Examples: > SELECT dimension_col, _FUNC_(measure_col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala index 30e00ac68004b..b87c0ede1ab81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala @@ -233,6 +233,14 @@ case class Mode( _FUNC_() WITHIN GROUP (ORDER BY col) - Returns the most frequent value for the values within `col` (specified in ORDER BY clause). NULL values are ignored. If all the values are NULL, or there are 0 rows, returns NULL. When multiple values have the same greatest frequency only one value will be returned. The value will be chosen based on sort direction. Return the smallest value if sort direction is asc or the largest value if sort direction is desc from multiple values with the same frequency.""", + arguments = """ + Arguments: + * col - The column to compute the most frequent value of. + An expression of any type. + * deterministic - Whether to return a deterministic result when there are multiple most + frequent values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index d066a87fc7919..949a107928df7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); @@ -201,6 +206,11 @@ case class Sum( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala index 86a16ad389b5d..b1d9a46a237ec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala @@ -55,6 +55,11 @@ abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise AND of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise AND of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -76,6 +81,11 @@ case class BitAndAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise OR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise OR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -97,6 +107,11 @@ case class BitOrAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise XOR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise XOR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala index ae759abf8a4f5..3eba7352e3477 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala @@ -23,6 +23,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if all values of `expr` are true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (true), (true) AS tab(col); @@ -45,6 +50,11 @@ case class BoolAnd(child: Expression) extends AggregateFunction with RuntimeRepl @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if at least one value of `expr` is true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (false), (false) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index 8a370e9a9dc78..17e3bf3ffc3d4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -387,8 +387,10 @@ private[aggregate] object CollectTopK { arguments = """ Arguments: * expr - a string or binary expression to be concatenated. + An expression that evaluates to a string or binary. * delimiter - an optional string or binary foldable expression used to separate the input values. If NULL, the concatenation will be performed without a delimiter. Default is NULL. + An expression that evaluates to a string, binary, or null. Must be a constant. * key - an optional expression for ordering the input values. Multiple keys can be specified. If none are specified, the order of the rows in the result is non-deterministic. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala index 8fb1bf51319cc..9a7cab22f0699 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala @@ -48,6 +48,13 @@ import org.apache.spark.unsafe.types.UTF8String _FUNC_(expr, lgConfigK) - Returns the HllSketch's updatable binary representation. `lgConfigK` (optional) the log-base-2 of K, with K is the number of buckets or slots for the HllSketch. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the HLL sketch. + An expression that evaluates to an integer, long, string, or binary. + * lgConfigK - The log-base-2 of K, where K is the number of buckets for the sketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT hll_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala index 80ac45d273afa..5de6cbf8a22c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala @@ -54,6 +54,12 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, ByteType, DataT The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to a byte, integer, long, or short. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -337,6 +343,12 @@ case class KllSketchAggFloat( The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to a float or double. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala index 08cc7827b4779..924dd9d8e3e4b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala @@ -25,6 +25,13 @@ import org.apache.spark.sql.types.{AbstractDataType, DataType, DoubleType, Numer // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the number of non-null number pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -57,6 +64,13 @@ case class RegrCount(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the independent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable to average. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -92,6 +106,13 @@ case class RegrAvgX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the dependent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable to average. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -127,6 +148,13 @@ case class RegrAvgY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the coefficient of determination for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -160,6 +188,13 @@ case class RegrR2(y: Expression, x: Expression) extends PearsonCorrelation(y, x, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -195,6 +230,13 @@ case class RegrSXX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -224,6 +266,13 @@ case class RegrSXY(y: Expression, x: Expression) extends Covariance(y, x, true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -259,6 +308,13 @@ case class RegrSYY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the slope of the linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); @@ -322,6 +378,13 @@ case class RegrSlope(left: Expression, right: Expression) extends DeclarativeAgg // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the intercept of the univariate linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a double. + * x - The independent variable. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala index ac351db9e4710..d382182221b72 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala @@ -331,6 +331,11 @@ case class Percentile( @ExpressionDescription( usage = "_FUNC_(col) - Returns the median of numeric, ANSI interval or TIME column `col`.", + arguments = """ + Arguments: + * col - The column to compute the median of. + An expression that evaluates to a numeric, interval, or time. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala index 0f148d03cd70b..d8cd93129ec81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala @@ -74,6 +74,14 @@ case class FinalizedSketch(sketch: CompactSketch) extends ThetaSketchState { _FUNC_(expr, lgNomEntries) - Returns the ThetaSketch compact binary representation. `lgNomEntries` (optional) is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the ThetaSketch. """, + arguments = """ + Arguments: + * expr - The expression whose distinct values are aggregated into the sketch. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + ThetaSketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT theta_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala index 8cd864c914041..4439358ac8e11 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala @@ -54,6 +54,14 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with a double type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary)); @@ -143,6 +151,14 @@ case class TupleIntersectionAggDouble( _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with an integer type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala index 7bf9240456857..bc6cb9de57759 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala @@ -478,6 +478,16 @@ abstract class TupleSketchAggBase[U, S <: UpdatableSummary[U]] `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The double value to be aggregated as the summary. + An expression that evaluates to a double. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary); @@ -510,6 +520,16 @@ object TupleSketchAggDoubleExpressionBuilder extends ExpressionBuilder { `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The integer value to be aggregated as the summary. + An expression that evaluates to an integer. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala index 0baff38674762..ff565baaa38d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala @@ -357,6 +357,13 @@ abstract class TupleUnionAggBase[S <: Summary] `child` should be a binary TupleSketch representation created with a double type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary)); @@ -386,6 +393,14 @@ object TupleUnionAggDoubleExpressionBuilder extends ExpressionBuilder { `child` should be a binary TupleSketch representation created with an integer type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 9c72f822143f6..65685c898ab9d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -36,6 +36,11 @@ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( usage = "_FUNC_(expr) - Returns the negated value of `expr`.", + arguments = """ + Arguments: + * expr - The expression to negate. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -109,6 +114,11 @@ case class UnaryMinus( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the value of `expr`.", + arguments = """ + Arguments: + * expr - The input expression whose value is returned. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -140,6 +150,11 @@ case class UnaryPositive(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the absolute value of the numeric or interval value.", + arguments = """ + Arguments: + * expr - The numeric or interval value to take the absolute value of. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(-1); @@ -388,6 +403,13 @@ object BinaryArithmetic { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`+`expr2`.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT 1 _FUNC_ 2; @@ -562,6 +584,13 @@ object Subtract { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`*`expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand. + An expression that evaluates to a numeric or interval. + * expr2 - The second operand. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT 2 _FUNC_ 3; @@ -811,6 +840,13 @@ trait DivModLike extends BinaryArithmetic { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`/`expr2`. It always performs floating point division.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a double, decimal, or interval. + * expr2 - The divisor. + An expression that evaluates to a double or decimal. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -893,6 +929,13 @@ object Divide { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long.", + arguments = """ + Arguments: + * expr1 - The dividend to be divided. + An expression that evaluates to a long, decimal, or interval. + * expr2 - The divisor to divide by. + An expression that evaluates to a long, decimal, or interval. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -976,6 +1019,13 @@ object IntegralDivide { @ExpressionDescription( usage = "expr1 % expr2, or mod(expr1, expr2) - Returns the remainder after `expr1`/`expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT 2 % 1.8; @@ -1064,6 +1114,13 @@ object Remainder { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the positive value of `expr1` mod `expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(10, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala index 5a73fd830e037..1f943ed7edb08 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType, LongT @ExpressionDescription( usage = "_FUNC_(child) - Returns the bucket number for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bucket number for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(123); @@ -63,6 +68,11 @@ case class BitmapBucketNumber(child: Expression) @ExpressionDescription( usage = "_FUNC_(child) - Returns the bit position for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bit position for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -148,6 +158,11 @@ case class BitmapCount(child: Expression) _FUNC_(child) - Returns a bitmap with the positions of the bits set from all the values from the child expression. The child expression will most likely be bitmap_bit_position(). """, + arguments = """ + Arguments: + * child - The expression whose values set the bit positions in the bitmap. + An expression that evaluates to a long. + """, // scalastyle:off line.size.limit examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 5fac0a93bf9bf..964d6a01ae1c0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -29,6 +29,13 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise AND. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise AND. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -78,6 +85,13 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The expr1 argument. + An expression that evaluates to an integral. + * expr2 - The expr2 argument. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -127,6 +141,13 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -174,6 +195,11 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "_FUNC_ expr - Returns the result of bitwise NOT of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise NOT of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_ 0; @@ -216,6 +242,11 @@ case class BitwiseNot(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the number of bits that are set in the argument expr as an" + " unsigned 64-bit integer, or NULL if the argument is NULL.", + arguments = """ + Arguments: + * expr - The argument whose set bits are counted. + An expression that evaluates to an integral or boolean. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -266,6 +297,13 @@ object BitwiseGetUtil { The positions are numbered from right to left, starting at zero. The position argument cannot be negative. """, + arguments = """ + Arguments: + * expr - The expression to read the bit from. + An expression that evaluates to an integral. + * pos - The position of the bit to return, numbered from right to left starting at zero. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(11, 0); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala index c3db6fca6a861..bf1bc66ca8b19 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala @@ -145,6 +145,7 @@ case class ResolvedCollation(collationName: String) extends LeafExpression { arguments = """ Arguments: * expr - String expression to perform collation on. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ab56386869a02..3f8de697b49ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -101,6 +101,11 @@ trait BinaryArrayExpressionWithImplicitCast spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the default settings, the function returns null for null input. """, + arguments = """ + Arguments: + * expr - The array or map whose size is returned. + An expression that evaluates to an array or map. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -157,6 +162,11 @@ object Size { */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the size of an array. The function returns null for null input.", + arguments = """ + Arguments: + * expr - The array to return the size of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -279,6 +289,12 @@ case class MapContainsKey(left: Expression, right: Expression) _FUNC_(a1, a2, ...) - Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays. """, + arguments = """ + Arguments: + * a1 - The first array to merge. + An expression that evaluates to an array. + * a2 - The second array to merge. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4)); @@ -1044,6 +1060,13 @@ case class MapSort(base: Expression) elements for double/float type. Null elements will be placed at the beginning of the returned array in ascending order or at the end of the returned array in descending order. """, + arguments = """ + Arguments: + * array - The array to sort. + An expression that evaluates to an array. + * ascendingOrder - Whether to sort in ascending order; false sorts in descending order. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true); @@ -1244,6 +1267,11 @@ case class SortArray(base: Expression, ascendingOrder: Expression) */ @ExpressionDescription( usage = "_FUNC_(array) - Returns a random permutation of the given array.", + arguments = """ + Arguments: + * array - The array to return a random permutation of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, 3, 5)); @@ -1344,6 +1372,11 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None) extends U @ExpressionDescription( usage = """_FUNC_(expr) - Returns a reversed string, a binary value with bytes in reverse order, or an array with reverse order of elements.""", + arguments = """ + Arguments: + * expr - The string, binary value, or array to reverse. + An expression that evaluates to a string, binary, or array. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -1451,6 +1484,13 @@ case class Reverse(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, value) - Returns true if the array contains the value.", + arguments = """ + Arguments: + * array - The array to search. + An expression that evaluates to an array. + * value - The value to check for membership in the array. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -1731,6 +1771,13 @@ trait ArrayPendBase extends RuntimeReplaceable Null element is also prepended to the array. But if the array passed is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to prepend the element to. + An expression that evaluates to an array. + * element - The element to add at the beginning of the array. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1766,6 +1813,13 @@ case class ArrayPrepend(left: Expression, right: Expression) extends ArrayPendBa Null element is also appended into the array. But if the array passed, is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to append the element to. + An expression that evaluates to an array. + * element - The element to add at the end of the array. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1794,6 +1848,13 @@ case class ArrayAppend(left: Expression, right: Expression) extends ArrayPendBas // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(a1, a2) - Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.", + arguments = """ + Arguments: + * a1 - The first array to compare. + An expression that evaluates to an array. + * a2 - The second array to compare. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5)); @@ -2022,6 +2083,15 @@ case class ArraysOverlap(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(x, start, length) - Subsets array x starting from index start (array indices start at 1, or starting from the end if start is negative) with the specified length.", + arguments = """ + Arguments: + * x - The array to take a subset of. + An expression that evaluates to an array. + * start - The 1-based start index, or from the end if negative. + An expression that evaluates to an integer. + * length - The number of elements to take. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 2, 2); @@ -2121,6 +2191,15 @@ case class Slice(x: Expression, start: Expression, length: Expression) _FUNC_(array, delimiter[, nullReplacement]) - Concatenates the elements of the given array using the delimiter and an optional string to replace nulls. If no value is set for nullReplacement, any null value is filtered.""", + arguments = """ + Arguments: + * array - The array whose elements are concatenated. + An expression that evaluates to an array. + * delimiter - The delimiter placed between concatenated elements. + An expression that evaluates to a string. + * nullReplacement - The string used to replace null elements. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(array('hello', 'world'), ' '); @@ -2316,6 +2395,11 @@ case class ArrayJoin( usage = """ _FUNC_(array) - Returns the minimum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the minimum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2389,6 +2473,11 @@ case class ArrayMin(child: Expression) usage = """ _FUNC_(array) - Returns the maximum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the maximum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2469,6 +2558,13 @@ case class ArrayMax(child: Expression) _FUNC_(array, element) - Returns the (1-based) index of the first matching element of the array as long, or 0 if no match is found. """, + arguments = """ + Arguments: + * array - The array to search within. + An expression that evaluates to an array. + * element - The element to find the position of. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(312, 773, 708, 708), 708); @@ -2571,6 +2667,13 @@ case class ArrayPosition(left: Expression, right: Expression) _FUNC_(array, index) - Returns element of array at given (0-based) index. If the index points outside of the array boundaries, then this function returns NULL. """, + arguments = """ + Arguments: + * array - The array to retrieve an element from. + An expression that evaluates to an array. + * index - The 0-based index of the element to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0); @@ -2619,6 +2722,13 @@ case class Get(left: Expression, right: Expression) _FUNC_(map, key) - Returns value for given key. The function returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array to retrieve the element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the element to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -2872,6 +2982,13 @@ case class ElementAt( _FUNC_(map, key) - Returns value for given key. The function always returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array or map to retrieve an element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -3883,6 +4000,13 @@ object Sequence { */ @ExpressionDescription( usage = "_FUNC_(element, count) - Returns the array containing element count times.", + arguments = """ + Arguments: + * element - The element to repeat. + An expression of any type. + * count - The number of times to repeat the element. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -4007,6 +4131,13 @@ case class ArrayRepeat(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, element) - Remove all elements that equal to element from array.", + arguments = """ + Arguments: + * array - The array to remove elements from. + An expression that evaluates to an array. + * element - The element to remove from the array. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3), 3); @@ -4219,6 +4350,11 @@ trait ArraySetLike { */ @ExpressionDescription( usage = "_FUNC_(array) - Removes duplicate values from the array.", + arguments = """ + Arguments: + * array - The array to remove duplicate values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3)); @@ -4417,6 +4553,13 @@ trait ArrayBinaryLike _FUNC_(array1, array2) - Returns an array of the elements in the union of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to union. + An expression that evaluates to an array. + * array2 - The second array to union. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4596,6 +4739,13 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi _FUNC_(array1, array2) - Returns an array of the elements in the intersection of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to intersect. + An expression that evaluates to an array. + * array2 - The second array to intersect. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4830,6 +4980,13 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina _FUNC_(array1, array2) - Returns an array of the elements in array1 but not in array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The array to take elements from. + An expression that evaluates to an array. + * array2 - The array of elements to exclude. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -5043,6 +5200,15 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL Index above array size appends the array, or prepends the array if index is negative, with 'null' elements. """, + arguments = """ + Arguments: + * x - The array to insert the value into. + An expression that evaluates to an array. + * pos - The 1-based index at which to insert the value. + An expression that evaluates to an integer. + * val - The value to insert into the array. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 5, 5); @@ -5391,6 +5557,11 @@ case class ArrayInsert( @ExpressionDescription( usage = "_FUNC_(array) - Removes null values from the array.", + arguments = """ + Arguments: + * array - The array to remove null values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 1c601b29002e3..122272ba43339 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -299,6 +299,13 @@ object CreateMap { usage = """ _FUNC_(keys, values) - Creates a map with a pair of the given key/value arrays. All elements in keys should not be null""", + arguments = """ + Arguments: + * keys - The array providing the keys of the map. + An expression that evaluates to an array. + * values - The array providing the values of the map. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1.0, 3.0), array('2', '4')); @@ -566,6 +573,15 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression with // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for `pairDelim` and ':' for `keyValueDelim`. Both `pairDelim` and `keyValueDelim` are treated as regular expressions.", + arguments = """ + Arguments: + * text - The text to split into key/value pairs. + An expression that evaluates to a string. + * pairDelim - The delimiter regular expression separating pairs, defaults to ','. + An expression that evaluates to a string. + * keyValueDelim - The delimiter regular expression separating key and value, defaults to ':'. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index e7c6a1091861b..4c18ba18f4810 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -316,6 +316,13 @@ case class CurrentBatchTimestamp( */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to add to the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -356,6 +363,13 @@ case class DateAdd(startDate: Expression, days: Expression) */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` before `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to subtract from the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -548,6 +562,11 @@ trait GetDateField extends UnaryExpression with ImplicitCastInputTypes { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of year of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-04-09'); @@ -569,6 +588,11 @@ case class DayOfYear(child: Expression) extends GetDateField { @ExpressionDescription( usage = "_FUNC_(days) - Create date from the number of days since 1970-01-01.", + arguments = """ + Arguments: + * days - The number of days since 1970-01-01. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -597,6 +621,11 @@ case class DateFromUnixDate(child: Expression) extends UnaryExpression @ExpressionDescription( usage = "_FUNC_(date) - Returns the number of days since 1970-01-01.", + arguments = """ + Arguments: + * date - The date to convert to the number of days since 1970-01-01. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE("1970-01-02")); @@ -666,6 +695,11 @@ abstract class IntegralToTimestampBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds (can be fractional) since UTC epoch.", + arguments = """ + Arguments: + * seconds - The number of seconds since the UTC epoch. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(1230219000); @@ -802,6 +836,11 @@ case class SecondsToTimestamp(child: Expression) extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch.", + arguments = """ + Arguments: + * milliseconds - The number of milliseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123); @@ -826,6 +865,11 @@ case class MillisToTimestamp(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(microseconds) - Creates timestamp from the number of microseconds since UTC epoch.", + arguments = """ + Arguments: + * microseconds - The number of microseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123123); @@ -850,6 +894,11 @@ case class MicrosToTimestamp(child: Expression) // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(nanoseconds) - Creates timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from the number of nanoseconds since UTC epoch.", + arguments = """ + Arguments: + * nanoseconds - The number of nanoseconds since the UTC epoch. + An expression that evaluates to an integral or decimal. + """, examples = """ Examples: > SET spark.sql.timestampNanosTypes.enabled=true; @@ -965,6 +1014,11 @@ abstract class TimestampToLongBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to seconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -998,6 +1052,11 @@ case class CastTimestampNTZToLong(child: Expression) extends TimestampToLongBase // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to milliseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1018,6 +1077,11 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to microseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1089,6 +1153,11 @@ case class UnixNanos(child: Expression) // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1118,6 +1187,11 @@ case class YearOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the quarter from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31'); @@ -1140,6 +1214,11 @@ case class Quarter(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the month component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1161,6 +1240,11 @@ case class Month(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of month of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1183,6 +1267,11 @@ case class DayOfMonth(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (1 = Sunday, 2 = Monday, ..., 7 = Saturday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1205,6 +1294,11 @@ case class DayOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1227,6 +1321,11 @@ case class WeekDay(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the week of the year of the given date. A week is considered to start on a Monday and week 1 is the first week with >3 days.", + arguments = """ + Arguments: + * date - The date to extract the week of the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1249,6 +1348,11 @@ case class WeekOfYear(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated month name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated month name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1271,6 +1375,11 @@ case class MonthName(child: Expression) extends GetDateField with DefaultStringP // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated day name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated day name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE('2008-02-20')); @@ -1298,8 +1407,10 @@ case class DayName(child: Expression) extends GetDateField with DefaultStringPro arguments = """ Arguments: * timestamp - A date, time, timestamp or string to be converted to the given format. + An expression that evaluates to a timestamp or time. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1439,9 +1550,11 @@ object DateFormatClass { arguments = """ Arguments: * timeExp - A date/timestamp or string which is returned as a UNIX timestamp. + An expression that evaluates to a string, date, timestamp, or timestamp_ntz. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1493,9 +1606,11 @@ case class ToUnixTimestamp( arguments = """ Arguments: * timeExp - A date/timestamp or string. If not provided, this defaults to current time. + An expression that evaluates to a string, date, timestamp, or timestamp_ntz. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1578,8 +1693,10 @@ case class GetTimestamp( arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp without time zone. + An expression that evaluates to a string, date, timestamp, or timestamp_ntz. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1615,6 +1732,7 @@ object ParseToTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp with local time zone. + An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1653,6 +1771,7 @@ object ParseToTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1874,8 +1993,10 @@ abstract class UnixTime extends ToTimestamp { arguments = """ Arguments: * unix_time - UNIX Timestamp to be converted to the provided format. + An expression that evaluates to a long. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1953,6 +2074,11 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ */ @ExpressionDescription( usage = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", + arguments = """ + Arguments: + * date - The date whose month's last day is returned. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-01-12'); @@ -2000,6 +2126,13 @@ case class LastDay(startDate: Expression) When both of the input parameters are not NULL and day_of_week is an invalid input, the function throws SparkIllegalArgumentException if `spark.sql.ansi.enabled` is set to true, otherwise NULL. """, + arguments = """ + Arguments: + * start_date - The date after which to find the next occurrence of the given day of week. + An expression that evaluates to a date. + * day_of_week - The name of the day of week to find (e.g. "Mon", "Tuesday"). + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2015-01-14', 'TU'); @@ -2353,6 +2486,13 @@ sealed trait UTCTimestamp extends BinaryExpression with ImplicitCastInputTypes { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in UTC. + An expression that evaluates to a timestamp. + * timezone - The target time zone to render the timestamp in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2387,6 +2527,13 @@ case class FromUTCTimestamp(left: Expression, right: Expression) extends UTCTime // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in the given time zone. + An expression that evaluates to a timestamp. + * timezone - The time zone the timestamp is interpreted as being in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2426,6 +2573,13 @@ abstract class AddMonthsBase extends BinaryExpression with ImplicitCastInputType // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(start_date, num_months) - Returns the date that is `num_months` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date to add months to. + An expression that evaluates to a date. + * num_months - The number of months to add to the start date. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 1); @@ -2519,6 +2673,15 @@ case class TimestampAddYMInterval( are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false. """, + arguments = """ + Arguments: + * timestamp1 - The first timestamp to compare. + An expression that evaluates to a timestamp. + * timestamp2 - The second timestamp to compare. + An expression that evaluates to a timestamp. + * roundOff - Whether to round off the result to 8 decimal places. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); @@ -2586,8 +2749,10 @@ case class MonthsBetween( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, timestamp, or timestamp_ntz. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2662,8 +2827,10 @@ case class ParseToDate( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, timestamp, or timestamp_ntz. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2705,6 +2872,7 @@ object TryToDateExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -2864,11 +3032,13 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes { arguments = """ Arguments: * date - date value or valid date string + An expression that evaluates to a date. * fmt - the format representing the unit to be truncated to - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in - "QUARTER" - truncate to the first date of the quarter that the `date` falls in - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in - "WEEK" - truncate to the Monday of the week that the `date` falls in + An expression that evaluates to a string. """, examples = """ Examples: @@ -2933,7 +3103,9 @@ case class TruncDate(date: Expression, format: Expression) - "SECOND" - zero out the second fraction part - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - everything remains + An expression that evaluates to a string. * ts - datetime value or valid timestamp string + An expression that evaluates to a timestamp. """, examples = """ Examples: @@ -2993,6 +3165,13 @@ case class TruncTimestamp( */ @ExpressionDescription( usage = "_FUNC_(endDate, startDate) - Returns the number of days from `startDate` to `endDate`.", + arguments = """ + Arguments: + * endDate - The end date to count days up to. + An expression that evaluates to a date. + * startDate - The start date to count days from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-31', '2009-07-30'); @@ -3031,8 +3210,11 @@ case class DateDiff(endDate: Expression, startDate: Expression) arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. """, examples = """ Examples: @@ -3252,13 +3434,19 @@ case class MakeTimestampLTZNanos( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3306,13 +3494,19 @@ object MakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3363,13 +3557,19 @@ object TryMakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3430,13 +3630,19 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3702,14 +3908,20 @@ case class TryMakeTimestamp( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -3852,13 +4064,19 @@ case class MakeTimestampFromDateTime( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -4250,8 +4468,11 @@ object SubtractDates { Arguments: * sourceTz - the time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + An expression that evaluates to a string. * targetTz - the time zone to which the input timestamp should be converted + An expression that evaluates to a string. * sourceTs - a timestamp without time zone + An expression that evaluates to a timestamp_ntz or timestamp. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala index 3e6f6dbe22e07..2a9a140ef2a99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala @@ -52,6 +52,11 @@ import org.apache.spark.util.ArrayImplicits._ */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the MD5 checksum of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -94,6 +99,13 @@ case class Md5(child: Expression) _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. """, + arguments = """ + Arguments: + * expr - The expression to compute the SHA-2 checksum of. + An expression that evaluates to a binary. + * bitLength - The bit length of the SHA-2 result (224, 256, 384, or 512). + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark', 256); @@ -160,6 +172,11 @@ case class Sha2(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the SHA-1 hash of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -195,6 +212,11 @@ case class Sha1(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint.", + arguments = """ + Arguments: + * expr - The expression to compute the cyclic redundancy check value of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index f87b4f70d298a..524e27321f4fa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -393,6 +393,13 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction { */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Transforms elements in an array using the function.", + arguments = """ + Arguments: + * expr - The array whose elements are transformed. + An expression that evaluates to an array. + * func - The function applied to each element. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x + 1); @@ -513,6 +520,13 @@ case class ArrayTransform( equal to, or greater than the second element. If the comparator function returns null, the function will fail and raise an error. """, + arguments = """ + Arguments: + * expr - The input array to sort. + An expression that evaluates to an array. + * func - The comparator function used to determine the sort order. + A lambda function returning an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); @@ -700,6 +714,13 @@ case class MapFilter( */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Filters the input array using the given predicate.", + arguments = """ + Arguments: + * expr - The array to filter. + An expression that evaluates to an array. + * func - The predicate used to filter the array. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1); @@ -838,6 +859,13 @@ case class ArrayFilter( */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for one or more elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -972,6 +1000,13 @@ object ArrayExists { */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for all elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -1093,6 +1128,17 @@ case class ArrayForAll( elements in the array, and reduces this to a single state. The final state is converted into the final result by applying a finish function. """, + arguments = """ + Arguments: + * expr - The input array to reduce. + An expression that evaluates to an array. + * start - The initial state to start the reduction from. + An expression of any type. + * merge - The binary operator applied to the current state and each array element. + A lambda function. + * finish - The function that converts the final state into the result. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0, (acc, x) -> acc + x); @@ -1652,6 +1698,15 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(left, right, func) - Merges the two given arrays, element-wise, into a single array using function. If one array is shorter, nulls are appended at the end to match the length of the longer array, before applying function.", + arguments = """ + Arguments: + * left - The first array to merge. + An expression that evaluates to an array. + * right - The second array to merge. + An expression that evaluates to an array. + * func - The function applied element-wise to merge the arrays. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 3e4d6772c4fc4..29522cbcc15c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -233,6 +233,7 @@ case class DivideInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -316,6 +317,7 @@ case class TryMakeInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -474,9 +476,13 @@ case class MakeInterval( arguments = """ Arguments: * days - the number of days, positive or negative + An expression that evaluates to an integer. * hours - the number of hours, positive or negative + An expression that evaluates to an integer. * mins - the number of minutes, positive or negative + An expression that evaluates to an integer. * secs - the number of seconds with the fractional part in microsecond precision. + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -557,7 +563,9 @@ case class MakeDTInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index b4f41fda42f00..a6792d80ef341 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -42,6 +42,13 @@ import org.apache.spark.unsafe.types.UTF8String */ @ExpressionDescription( usage = "_FUNC_(json_txt, path) - Extracts a json object from `path`.", + arguments = """ + Arguments: + * json_txt - The JSON text to extract from. + An expression that evaluates to a string. + * path - The path identifying the JSON object to extract. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":"b"}', '$.a'); @@ -449,6 +456,12 @@ object JsonToStructs { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, options]) - Returns a JSON string with a given struct value", + arguments = """ + Arguments: + * expr - The struct value to convert to a JSON string. + An expression that evaluates to a struct, array, map, or variant. + * options - Options controlling how the JSON string is produced. + """, examples = """ Examples: > SELECT _FUNC_(named_struct('a', 1, 'b', 2)); @@ -611,6 +624,7 @@ case class SchemaOfJson( Arguments: * jsonArray - A JSON array. `NULL` is returned in case of any other valid JSON string, `NULL` or an invalid JSON. + An expression that evaluates to a string. """, examples = """ Examples: @@ -657,6 +671,7 @@ case class LengthOfJsonArray(child: Expression) * json_object - A JSON object. If a valid JSON object is given, all the keys of the outermost object will be returned as an array. If it is any other valid JSON string, an invalid JSON string or an empty string, the function returns null. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala index 01481050f2805..ba6c68ab95fcc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala @@ -33,6 +33,11 @@ import org.apache.spark.unsafe.types.UTF8String usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -61,6 +66,11 @@ case class KllSketchToStringBigint(child: Expression) extends KllSketchToStringB usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -89,6 +99,11 @@ case class KllSketchToStringFloat(child: Expression) extends KllSketchToStringBa usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -127,6 +142,11 @@ abstract class KllSketchToStringBase usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -155,6 +175,11 @@ case class KllSketchGetNBigint(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -183,6 +208,11 @@ case class KllSketchGetNFloat(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -221,6 +251,13 @@ abstract class KllSketchGetNBase usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -252,6 +289,13 @@ case class KllSketchMergeBigint(left: Expression, right: Expression) extends Kll usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_float(_FUNC_(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -283,6 +327,13 @@ case class KllSketchMergeFloat(left: Expression, right: Expression) extends KllS usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -327,6 +378,13 @@ abstract class KllSketchMergeBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -359,6 +417,13 @@ case class KllSketchGetQuantileBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -391,6 +456,13 @@ case class KllSketchGetQuantileFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -523,6 +595,13 @@ abstract class KllSketchGetQuantileBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a long or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -551,6 +630,13 @@ case class KllSketchGetRankBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a float or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -579,6 +665,13 @@ case class KllSketchGetRankFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala index 5bba89b3dad51..9613bc25d65f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala @@ -38,10 +38,15 @@ import org.apache.spark.unsafe.types.UTF8String arguments = """ Arguments: * input - string value to mask. Supported types: STRING, VARCHAR, CHAR + An expression that evaluates to a string. * upperChar - character to replace upper-case characters with. Specify NULL to retain original character. Default value: 'X' + An expression that evaluates to a string. Must be a constant. * lowerChar - character to replace lower-case characters with. Specify NULL to retain original character. Default value: 'x' + An expression that evaluates to a string. Must be a constant. * digitChar - character to replace digit characters with. Specify NULL to retain original character. Default value: 'n' + An expression that evaluates to a string. Must be a constant. * otherChar - character to replace all other characters with. Specify NULL to retain original character. Default value: NULL + An expression that evaluates to a string. Must be a constant. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 7bd166250db01..662e3349d3863 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -192,6 +192,11 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") _FUNC_(expr) - Returns the inverse cosine (a.k.a. arc cosine) of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -210,6 +215,11 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS" _FUNC_(expr) - Returns the inverse sine (a.k.a. arc sine) the arc sin of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -228,6 +238,11 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN" _FUNC_(expr) - Returns the inverse tangent (a.k.a. arc tangent) of `expr`, as if computed by `java.lang.Math._FUNC_` """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -241,6 +256,11 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN" @ExpressionDescription( usage = "_FUNC_(expr) - Returns the cube root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the cube root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(27.0); @@ -308,6 +328,13 @@ trait CeilFloorExpressionBuilderBase extends ExpressionBuilder { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, scale]) - Returns the smallest number after rounding up that is not smaller than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round up. + An expression that evaluates to a double, decimal, or long. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -349,6 +376,7 @@ case class RoundCeil(child: Expression, scale: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -368,6 +396,7 @@ case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -392,6 +421,7 @@ case class Sec(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -408,6 +438,11 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic cosine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -436,6 +471,15 @@ case class Acosh(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`.", + arguments = """ + Arguments: + * num - The number to convert. + An expression that evaluates to a string. + * from_base - The base the number is currently in. + An expression that evaluates to an integer. + * to_base - The base to convert the number to. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('100', 2, 10); @@ -501,6 +545,11 @@ case class Conv( @ExpressionDescription( usage = "_FUNC_(expr) - Returns e to the power of `expr`.", + arguments = """ + Arguments: + * expr - The exponent to raise e to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -517,6 +566,11 @@ case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "E @ExpressionDescription( usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.", + arguments = """ + Arguments: + * expr - The exponent used to compute exp(expr) - 1. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -564,6 +618,13 @@ case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLO // scalastyle:off line.size.limit @ExpressionDescription( usage = " _FUNC_(expr[, scale]) - Returns the largest number after rounding down that is not greater than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round down. + An expression that evaluates to a double, decimal, or long. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -630,6 +691,11 @@ object Factorial { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null.", + arguments = """ + Arguments: + * expr - The integer between 0 and 20 to compute the factorial of. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(5); @@ -676,6 +742,11 @@ case class Factorial(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the natural logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -690,6 +761,11 @@ case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LO @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.", + arguments = """ + Arguments: + * expr - The expression to compute the base-2 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2); @@ -715,6 +791,11 @@ case class Log2(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 10.", + arguments = """ + Arguments: + * expr - The expression to compute the base-10 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10); @@ -728,6 +809,11 @@ case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, @ExpressionDescription( usage = "_FUNC_(expr) - Returns log(1 + `expr`).", + arguments = """ + Arguments: + * expr - The expression used to compute log(1 + expr). + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -743,6 +829,11 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.", + arguments = """ + Arguments: + * expr - The value to round to the closest mathematical integer. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(12.3456); @@ -759,6 +850,11 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND @ExpressionDescription( usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive.", + arguments = """ + Arguments: + * expr - The expression whose sign is returned. + An expression that evaluates to a double or interval. + """, examples = """ Examples: > SELECT _FUNC_(40); @@ -782,6 +878,7 @@ case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "S arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -801,6 +898,7 @@ case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -824,6 +922,7 @@ case class Csc(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -840,6 +939,11 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic sine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -859,6 +963,11 @@ case class Asinh(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the square root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the square root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(4); @@ -877,6 +986,7 @@ case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT" arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -896,6 +1006,7 @@ case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -920,6 +1031,7 @@ case class Cot(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -936,6 +1048,11 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic tangent of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -961,6 +1078,7 @@ case class Atanh(child: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -980,6 +1098,7 @@ case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegre arguments = """ Arguments: * expr - angle in degrees + An expression that evaluates to a double. """, examples = """ Examples: @@ -997,6 +1116,11 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the string representation of the long value `expr` represented in binary.", + arguments = """ + Arguments: + * expr - The long value to represent in binary. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(13); @@ -1112,6 +1236,11 @@ object Hex { */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts `expr` to hexadecimal.", + arguments = """ + Arguments: + * expr - The expression to convert to hexadecimal. + An expression that evaluates to a long, binary, or string. + """, examples = """ Examples: > SELECT _FUNC_(17); @@ -1162,6 +1291,11 @@ case class Hex(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts hexadecimal `expr` to binary.", + arguments = """ + Arguments: + * expr - The hexadecimal expression to convert to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); @@ -1226,7 +1360,9 @@ case class Unhex(child: Expression, failOnError: Boolean = false) arguments = """ Arguments: * exprY - coordinate on y-axis + An expression that evaluates to a double. * exprX - coordinate on x-axis + An expression that evaluates to a double. """, examples = """ Examples: @@ -1253,6 +1389,13 @@ case class Atan2(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Raises `expr1` to the power of `expr2`.", + arguments = """ + Arguments: + * expr1 - The base to be raised to a power. + An expression that evaluates to a double. + * expr2 - The exponent to raise the base to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -1314,6 +1457,13 @@ sealed trait BitShiftOperation */ @ExpressionDescription( usage = "base _FUNC_ exp - Bitwise left shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted left. + An expression that evaluates to an integer or long. + * exp - The number of positions to shift left by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftleft(2, 1); @@ -1343,6 +1493,13 @@ case class ShiftLeft(left: Expression, right: Expression) extends BitShiftOperat */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise (signed) right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftright(4, 1); @@ -1371,6 +1528,13 @@ case class ShiftRight(left: Expression, right: Expression) extends BitShiftOpera */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise unsigned right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right without sign extension. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftrightunsigned(4, 1); @@ -1395,6 +1559,13 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) extends BitSh // scalastyle:off nonascii @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns sqrt(`expr1`\u00B2 + `expr2`\u00B2).", + arguments = """ + Arguments: + * expr1 - The first value whose square is summed. + An expression that evaluates to a double. + * expr2 - The second value whose square is summed. + An expression that evaluates to a double. + """, // scalastyle:on nonascii examples = """ Examples: @@ -1418,6 +1589,13 @@ case class Hypot(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(base, expr) - Returns the logarithm of `expr` with `base`.", + arguments = """ + Arguments: + * base - The base of the logarithm. + An expression that evaluates to a double. + * expr - The expression to compute the logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10, 100); @@ -1701,6 +1879,13 @@ abstract class RoundBase(child: Expression, scale: Expression, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1732,6 +1917,13 @@ case class Round( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1826,6 +2018,17 @@ object WidthBucket { `value` would be assigned in an equiwidth histogram with `num_bucket` buckets, in the range `min_value` to `max_value`." """, + arguments = """ + Arguments: + * value - The value to assign to a bucket. + An expression that evaluates to a double or interval. + * min_value - The minimum value of the histogram range. + An expression that evaluates to a double or interval. + * max_value - The maximum value of the histogram range. + An expression that evaluates to a double or interval. + * num_bucket - The number of equiwidth buckets in the histogram. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(5.3, 0.2, 10.6, 5); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 23e8b343e431e..651b1a231279d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -154,6 +154,11 @@ object RaiseError { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_( expr ) - Throws a USER_RAISED_EXCEPTION with `expr` as message.", + arguments = """ + Arguments: + * expr - The message for the raised exception. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('custom error message'); @@ -177,6 +182,13 @@ object RaiseErrorExpressionBuilder extends ExpressionBuilder { */ @ExpressionDescription( usage = "_FUNC_(expr [, message]) - Throws an exception if `expr` is not true.", + arguments = """ + Arguments: + * expr - The expression that is expected to be true. + An expression that evaluates to a boolean. + * message - The message for the exception thrown when the expression is not true. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(0 < 1); @@ -412,15 +424,21 @@ case class CurrentUser() arguments = """ Arguments: * expr - The binary value to encrypt. + An expression that evaluates to a binary. * key - The passphrase to use to encrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * iv - Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or ''. 16-byte array for CBC mode. 12-byte array for GCM mode. + An expression that evaluates to a binary. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -495,13 +513,18 @@ case class AesEncrypt( arguments = """ Arguments: * expr - The binary value to decrypt. + An expression that evaluates to a binary. * key - The passphrase to use to decrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -561,6 +584,19 @@ case class AesDecrypt( @ExpressionDescription( usage = "_FUNC_(expr, key[, mode[, padding[, aad]]]) - This is a special version of `aes_decrypt` that performs the same operation, but returns a NULL value instead of raising an error if the decryption cannot be performed.", + arguments = """ + Arguments: + * expr - The encrypted value to decrypt. + An expression that evaluates to a binary. + * key - The key used to decrypt the value. + An expression that evaluates to a binary. + * mode - The block cipher mode used for decryption. + An expression that evaluates to a string. + * padding - The padding scheme used for decryption. + An expression that evaluates to a string. + * aad - The additional authenticated data. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 1d50d8987ccee..7f22e12789f40 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -188,6 +188,13 @@ private case class TypedNullLiteral(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not equal to the other expression. + An expression of any type. + * expr2 - The value compared against the first expression. + An expression of any type. + """, examples = """ Examples: > SELECT _FUNC_(2, 2); @@ -219,6 +226,11 @@ case class NullIf(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns null if `expr` is equal to zero, or `expr` otherwise.", + arguments = """ + Arguments: + * expr - The expression that returns null when equal to zero. + An expression of any type. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -313,6 +325,11 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, replace */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if `expr` is NaN, or false otherwise.", + arguments = """ + Arguments: + * expr - The expression to test for NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double)); @@ -361,6 +378,13 @@ case class IsNaN(child: Expression) extends UnaryExpression */ @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not NaN. + An expression that evaluates to a double or float. + * expr2 - The value returned when the first expression is NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double), 123); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala index 0a6d23977d2f6..6e559c0165bf3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala @@ -127,6 +127,13 @@ abstract class ToNumberBase(left: Expression, right: Expression, errorOnFail: Bo negative number with wrapping angled brackets. ('<1>'). """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -162,6 +169,13 @@ case class ToNumber(left: Expression, right: Expression) Returns NULL if the string 'expr' does not match the expected format. The format follows the same semantics as the to_number function. """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -223,6 +237,13 @@ case class TryToNumber(left: Expression, right: Expression) 'hex': a string in the hexadecimal format. 'utf-8': the input binary is decoded to UTF-8 string. """, + arguments = """ + Arguments: + * expr - The expression to convert to a string. + An expression that evaluates to a decimal, timestamp, time, or binary. + * format - The format that specifies how to render the value as a string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(454, '999'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 7767e97d34ac5..bf98f630d7ab3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -306,6 +306,11 @@ trait PredicateHelper extends AliasHelper with Logging { @ExpressionDescription( usage = "_FUNC_ expr - Logical not.", + arguments = """ + Arguments: + * expr - The boolean expression to negate. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_ true; @@ -826,6 +831,13 @@ object InSet { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical AND.", + arguments = """ + Arguments: + * expr1 - The first boolean operand. + An expression that evaluates to a boolean. + * expr2 - The second boolean operand. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ true; @@ -922,6 +934,13 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical OR.", + arguments = """ + Arguments: + * expr1 - The left boolean operand of the logical OR. + An expression that evaluates to a boolean. + * expr2 - The right boolean operand of the logical OR. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ false; @@ -1091,6 +1110,7 @@ object Equality { * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1139,6 +1159,7 @@ case class EqualTo(left: Expression, right: Expression) * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1202,6 +1223,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1238,6 +1260,7 @@ case class EqualNull(left: Expression, right: Expression, replacement: Expressio and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1273,6 +1296,7 @@ case class LessThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1308,6 +1332,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: @@ -1344,6 +1369,7 @@ case class GreaterThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any type. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index adaaceb2e4396..b52d09fc9c709 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -92,6 +92,11 @@ private[catalyst] object ExpressionWithRandomSeed { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1).", + arguments = """ + Arguments: + * seed - The seed for the random number generator. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -146,6 +151,11 @@ object Rand { // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.""", + arguments = """ + Arguments: + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -205,6 +215,15 @@ object Randn { one or both of these are floating-point numbers, then the result will also be a floating-point number. """, + arguments = """ + Arguments: + * min - The lower bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * max - The upper bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(10, 20, 0) > 0 AS result; @@ -329,6 +348,13 @@ object Uniform { optional. The string length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). """, + arguments = """ + Arguments: + * length - The length of the random string to generate. + An expression that evaluates to an integer. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 3bd0352f6f07c..9f24c512bd335 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -127,6 +127,7 @@ private[catalyst] object StringRegexExpression { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)\ @@ -140,6 +141,7 @@ private[catalyst] object StringRegexExpression { enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -251,6 +253,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally and case-insensitively, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)

@@ -264,6 +267,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -481,7 +485,9 @@ case class NotLikeAny(child: Expression, patterns: Seq[UTF8String]) extends Like arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * regexp - a string expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL parser, see the unescaping rules at String Literal. @@ -573,14 +579,17 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress arguments = """ Arguments: * str - a string expression to split. + An expression that evaluates to a string. * regex - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. * limit - an integer expression which controls the number of times the regex is applied. * limit > 0: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last matched regex. * limit <= 0: `regex` will be applied as many times as possible, and the resulting array can be of any size. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -654,6 +663,7 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) arguments = """ Arguments: * str - a string expression to search for a regular expression pattern match. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -664,9 +674,12 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * rep - a string expression to replace matched substrings. + An expression that evaluates to a string. * position - a positive integer literal that indicates the position within `str` to begin searching. The default is 1. If position is greater than the number of characters in `str`, the result is `str`. + An expression that evaluates to an integer. Must be a constant. """, examples = """ Examples: @@ -876,6 +889,7 @@ abstract class RegExpExtractBase arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -886,11 +900,13 @@ abstract class RegExpExtractBase if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex maybe contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -949,6 +965,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -959,11 +976,13 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex may contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -1018,8 +1037,10 @@ case class RegExpExtractAll(subject: Expression, regexp: Expression, idx: Expres arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1057,7 +1078,9 @@ case class RegExpCount(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1097,6 +1120,7 @@ case class RegExpSubStr(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -1107,6 +1131,10 @@ case class RegExpSubStr(left: Expression, right: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. + * idx - the group index of the regular expression to search for. The default value is 0 + if not specified. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala index fe774a62fcc6d..556b9603e9785 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala @@ -134,6 +134,7 @@ case class ST_AsBinary(geo: Expression, endianness: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOGRAPHY value. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -179,7 +180,9 @@ case class ST_GeogFromWKB(wkb: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOMETRY value. + An expression that evaluates to a binary. * srid - The optional SRID value of the geometry. Default is 0. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 66c4a39ce8233..2144c76930f72 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -60,6 +60,11 @@ import org.apache.spark.util.ArrayImplicits._ // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values.", + arguments = """ + Arguments: + * sep - The separator placed between the concatenated values. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(' ', 'Spark', 'SQL'); @@ -452,6 +457,11 @@ trait String2StringExpression extends ImplicitCastInputTypes { */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to uppercase.", + arguments = """ + Arguments: + * str - The string to convert to uppercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -485,6 +495,11 @@ case class Upper(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to lowercase.", + arguments = """ + Arguments: + * str - The string to convert to lowercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -593,6 +608,13 @@ case class BinaryPredicate(override val prettyName: String, left: Expression, ri Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to search within. + An expression that evaluates to a string or binary. + * right - The expression to search for inside the left expression. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -635,6 +657,13 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The string to test. + An expression that evaluates to a string or binary. + * right - The prefix to check for at the start of the string. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -684,6 +713,13 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to test. + An expression that evaluates to a string or binary. + * right - The suffix expression to check the left expression ends with. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'SQL'); @@ -735,6 +771,7 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -783,6 +820,7 @@ case class IsValidUTF8(input: Expression) extends RuntimeReplaceable with Implic arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -829,6 +867,7 @@ case class MakeValidUTF8(input: Expression) extends RuntimeReplaceable with Impl arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -877,6 +916,7 @@ case class ValidateUTF8(input: Expression) extends RuntimeReplaceable with Impli arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -928,9 +968,12 @@ case class TryValidateUTF8(input: Expression) extends RuntimeReplaceable with Im arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * search - a string expression. If `search` is not found in `str`, `str` is returned unchanged. + An expression that evaluates to a string. * replace - a string expression. If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1013,6 +1056,17 @@ object Overlay { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, replace, pos[, len]) - Replace `input` with `replace` that starts at `pos` and is of length `len`.", + arguments = """ + Arguments: + * input - The string to have part of its content replaced. + An expression that evaluates to a string or binary. + * replace - The replacement content to insert. + An expression that evaluates to a string or binary. + * pos - The 1-based start position of the replacement. + An expression that evaluates to an integer. + * len - The number of characters to replace. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL' PLACING '_' FROM 6); @@ -1167,6 +1221,15 @@ object StringTranslate { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, from, to) - Translates the `input` string by replacing the characters present in the `from` string with the corresponding characters in the `to` string.", + arguments = """ + Arguments: + * input - The string to translate. + An expression that evaluates to a string. + * from - The characters to be replaced. + An expression that evaluates to a string. + * to - The corresponding replacement characters. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('AaBbCc', 'abc', '123'); @@ -1247,6 +1310,13 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac _FUNC_(str, str_array) - Returns the index (1-based) of the given string (`str`) in the comma-delimited list (`str_array`). Returns 0, if the string was not found or if the given string (`str`) contains a comma. """, + arguments = """ + Arguments: + * str - The string to search for. + An expression that evaluates to a string. + * str_array - The comma-delimited list to search within. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('ab','abc,b,ab,c,def'); @@ -1413,7 +1483,9 @@ object StringTrim { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. * BOTH, FROM - these are keywords to specify trimming string characters from both ends of the string * LEADING, FROM - these are keywords to specify trimming string characters from the left @@ -1487,7 +1559,9 @@ case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1546,7 +1620,9 @@ object StringTrimLeft { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1611,7 +1687,9 @@ object StringTrimRight { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1662,6 +1740,17 @@ case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = Non If `start` is not specified, it defaults to 1. `occurrence` must be a positive integer and defaults to 1. """, + arguments = """ + Arguments: + * str - The string to search within. + An expression that evaluates to a string. + * substr - The substring to search for. + An expression that evaluates to a string. + * start - The 1-based position from which to start the search. + An expression that evaluates to an integer. + * occurrence - Which occurrence of the substring to find. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('SparkSQL', 'SQL'); @@ -1809,6 +1898,15 @@ case class StringInstrWithOccurrence( (counting from the right) is returned. The function substring_index performs a case-sensitive match when searching for `delim`. """, + arguments = """ + Arguments: + * str - The string to take the substring from. + An expression that evaluates to a string. + * delim - The delimiter to count occurrences of. + An expression that evaluates to a string. + * count - The number of delimiter occurrences that bound the returned substring. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('www.apache.org', '.', 2); @@ -1860,6 +1958,15 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`. The given `pos` and return value are 1-based. """, + arguments = """ + Arguments: + * substr - The substring to search for. + An expression that evaluates to a string. + * str - The string to search within. + An expression that evaluates to a string. + * pos - The 1-based position from which to start the search. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('bar', 'foobarbar'); @@ -1988,6 +2095,15 @@ trait PadExpressionBuilderBase extends ExpressionBuilder { If `pad` is not specified, `str` will be padded to the left with space characters if it is a character string, and with zeros if it is a byte sequence. """, + arguments = """ + Arguments: + * str - The string to be left-padded. + An expression that evaluates to a string or binary. + * len - The target length of the resulting string. + An expression that evaluates to an integer. + * pad - The string to pad with on the left. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2073,6 +2189,15 @@ case class BinaryPad(funcName: String, str: Expression, len: Expression, pad: Ex If `pad` is not specified, `str` will be padded to the right with space characters if it is a character string, and with zeros if it is a binary string. """, + arguments = """ + Arguments: + * str - The string to right-pad. + An expression that evaluates to a string or binary. + * len - The length to pad the string to. + An expression that evaluates to an integer. + * pad - The string used to pad on the right. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2135,6 +2260,12 @@ case class StringRPad( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.", + arguments = """ + Arguments: + * strfmt - The printf-style format string. + An expression that evaluates to a string. + * obj - The value to be formatted into the string. + """, examples = """ Examples: > SELECT _FUNC_("Hello World %d %s", 100, "days"); @@ -2253,6 +2384,11 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC _FUNC_(str) - Returns `str` with the first letter of each word in uppercase. All other letters are in lowercase. Words are delimited by white space. """, + arguments = """ + Arguments: + * str - The string to capitalize the first letter of each word in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('sPark sql'); @@ -2288,6 +2424,13 @@ case class InitCap(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.", + arguments = """ + Arguments: + * str - The string to repeat. + An expression that evaluates to a string. + * n - The number of times to repeat the string. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -2328,6 +2471,11 @@ case class StringRepeat(str: Expression, times: Expression) */ @ExpressionDescription( usage = "_FUNC_(n) - Returns a string consisting of `n` spaces.", + arguments = """ + Arguments: + * n - The number of spaces to produce. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT concat(_FUNC_(2), '1'); @@ -2370,6 +2518,15 @@ case class StringSpace(child: Expression) _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`. """, + arguments = """ + Arguments: + * str - The string or byte array to take the substring from. + An expression that evaluates to a string or binary. + * pos - The 1-based starting position of the substring. + An expression that evaluates to an integer. + * len - The length of the substring to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 5); @@ -2444,6 +2601,13 @@ case class Substring(str: Expression, pos: Expression, len: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the rightmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the rightmost characters from. + An expression that evaluates to a string. + * len - The number of characters to take from the right. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2484,6 +2648,13 @@ case class Right(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the leftmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the leftmost characters from. + An expression that evaluates to a string or binary. + * len - The number of characters to take from the left. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2523,6 +2694,11 @@ case class Left(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL '); @@ -2570,6 +2746,11 @@ case class Length(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bit length of string data or number of bits of binary data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the bit length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2615,6 +2796,11 @@ case class BitLength(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the byte length of string data or number of bytes of binary " + "data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the byte length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2662,6 +2848,15 @@ case class OctetLength(child: Expression) @ExpressionDescription( usage = """ _FUNC_(str1, str2[, threshold]) - Returns the Levenshtein distance between the two given strings. If threshold is set and distance more than it, return -1.""", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + * threshold - The maximum distance to compute; if exceeded, -1 is returned. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('kitten', 'sitting'); @@ -2817,6 +3012,13 @@ case class Levenshtein( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str1, str2) - Returns the Jaro-Winkler similarity between the two given strings. The result is a double between 0 and 1, where 1 means identical.", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('MARTHA', 'MARHTA'); @@ -2855,6 +3057,11 @@ case class JaroWinkler(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns Soundex code of the string.", + arguments = """ + Arguments: + * str - The string to compute the Soundex code of. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Miller'); @@ -2886,6 +3093,11 @@ case class SoundEx(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns the numeric value of the first character of `str`.", + arguments = """ + Arguments: + * str - The string whose first character's numeric value is returned. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('222'); @@ -2919,6 +3131,11 @@ case class Ascii(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the ASCII character having the binary equivalent to `expr`. If n is larger than 256 the result is equivalent to chr(n % 256)", + arguments = """ + Arguments: + * expr - The number to convert to its ASCII character. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(65); @@ -2952,6 +3169,11 @@ case class Chr(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(bin) - Converts the argument from a binary `bin` to a base 64 string.", + arguments = """ + Arguments: + * bin - The binary value to encode as a base 64 string. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -3007,6 +3229,11 @@ object Base64 { */ @ExpressionDescription( usage = "_FUNC_(str) - Converts the argument from a base 64 string `str` to a binary.", + arguments = """ + Arguments: + * str - The base 64 string to decode to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('U3BhcmsgU1FM'); @@ -3156,7 +3383,9 @@ object Decode { arguments = """ Arguments: * bin - a binary expression to decode + An expression of any type. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to decode `bin` into a STRING. It is case insensitive. + An expression of any type. """, examples = """ Examples: @@ -3252,7 +3481,9 @@ object StringDecode { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to encode `str` into a BINARY. It is case insensitive. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3332,6 +3563,13 @@ object Encode { By default, the binary format for conversion is "hex" if `fmt` is omitted. The function returns NULL if at least one of the input parameters is NULL. """, + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format describing how to interpret the string (e.g. "hex", "utf-8", "base64"). + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); @@ -3445,6 +3683,13 @@ case class ToBinary( `expr2` also accept a user specified format. This is supposed to function like MySQL's FORMAT. """, + arguments = """ + Arguments: + * expr1 - The number to format. + An expression that evaluates to a numeric. + * expr2 - The number of decimal places to round to, or a format string. + An expression that evaluates to an integer or string. + """, examples = """ Examples: > SELECT _FUNC_(12332.123456, 4); @@ -3611,10 +3856,13 @@ case class FormatNumber(x: Expression, d: Expression) arguments = """ Arguments: * str - A STRING expression to be parsed. + An expression that evaluates to a string. * lang - An optional STRING expression with a language code from ISO 639 Alpha-2 (e.g. 'DE'), Alpha-3, or a language subtag of up to 8 characters. + An expression that evaluates to a string. * country - An optional STRING expression with a country code from ISO 3166 alpha-2 country code or a UN M.49 numeric-3 area code. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3717,6 +3965,15 @@ case class StringSplitSQL( throws an error. If `partNum` is negative, the parts are counted backward from the end of the string. If the `delimiter` is an empty string, the `str` is not split. """, + arguments = """ + Arguments: + * str - The string to split. + An expression that evaluates to a string. + * delimiter - The string used to split the input into parts. + An expression that evaluates to a string. + * partNum - The 1-based index of the split part to return. + An expression that evaluates to an integer. + """, examples = """ Examples: @@ -3781,6 +4038,11 @@ case class Empty2Null(child: Expression) extends UnaryExpression with String2Str This checksum function is widely applied on credit card numbers and government identification numbers to distinguish valid numbers from mistyped, incorrect numbers. """, + arguments = """ + Arguments: + * str - The string of digits to validate against the Luhn algorithm. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('8112189876'); @@ -3818,6 +4080,11 @@ case class Luhncheck(input: Expression) extends RuntimeReplaceable with Implicit // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` enclosed by single quotes and each instance of single quote in it is preceded by a backslash.", + arguments = """ + Arguments: + * str - The string to enclose in single quotes and escape. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Don\'t'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 1c9d6b335e8d9..827cfbda957d8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -61,8 +61,10 @@ trait TimeExpression extends Expression { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -188,8 +190,10 @@ private[expressions] object NanosTimestampCast { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -263,6 +267,11 @@ case class MinutesOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the minute of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the minute component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -330,6 +339,11 @@ case class HoursOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the hour of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the hour component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -422,6 +436,11 @@ case class SecondsOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the second of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the second component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -583,8 +602,11 @@ case class CurrentTime( arguments = """ Arguments: * hour - the hour to represent, from 0 to 23 + An expression that evaluates to an integer. * minute - the minute to represent, from 0 to 59 + An expression that evaluates to an integer. * second - the second to represent, from 0 to 59.999999 + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -706,8 +728,11 @@ case class SubtractTimes(left: Expression, right: Expression) - "SECOND" - "MILLISECOND" - "MICROSECOND" + An expression that evaluates to a string. * start - a starting TIME expression + An expression that evaluates to a time. * end - an ending TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -770,7 +795,9 @@ case class TimeDiff( - "SECOND" - zero out the fraction part of seconds - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - zero out the nanoseconds + An expression that evaluates to a string. * time - a TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -834,6 +861,7 @@ abstract class TimeFromBase extends UnaryExpression with RuntimeReplaceable with Arguments: * seconds - seconds since midnight (0 to 86399.999999). Supports decimals for fractional seconds. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -864,6 +892,7 @@ case class TimeFromSeconds(child: Expression) extends TimeFromBase { arguments = """ Arguments: * millis - milliseconds since midnight (0 to 86399999) + An expression that evaluates to an integral. """, examples = """ Examples: @@ -893,6 +922,7 @@ case class TimeFromMillis(child: Expression) extends TimeFromBase { arguments = """ Arguments: * micros - microseconds since midnight (0 to 86399999999) + An expression that evaluates to an integral. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala index b51cb74a5a8fa..b0bd67701be85 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala @@ -38,7 +38,8 @@ import org.apache.spark.unsafe.types.UTF8String """, arguments = """ Arguments: - str - a string expression to be translated + * str - a string expression to be translated + An expression that evaluates to a string. """, examples = """ Examples: @@ -80,6 +81,7 @@ case class UrlEncode(child: Expression) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -123,6 +125,7 @@ case class UrlDecode(child: Expression, failOnError: Boolean = true) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -165,6 +168,15 @@ object UrlCodec { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - This is a special version of `parse_url` that performs the same operation, but returns a NULL value instead of raising an error if the parsing cannot be performed.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); @@ -197,6 +209,15 @@ case class TryParseUrl(params: Seq[Expression], replacement: Expression) */ @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index 0f8995ce829fe..d06b7a22bd2e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -576,6 +576,11 @@ abstract class ParseJsonExpressionBuilderBase(failOnError: Boolean) extends Expr // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Throw an exception when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); @@ -590,6 +595,11 @@ object ParseJsonExpressionBuilder extends ParseJsonExpressionBuilderBase(true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Return NULL when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 800b38ea32223..0e1cafbb24c01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -78,6 +78,13 @@ abstract class XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('1','a/b'); @@ -99,6 +106,13 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -121,6 +135,13 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -143,6 +164,13 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -165,6 +193,13 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -187,6 +222,13 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -210,6 +252,13 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the node whose text is returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('bcc','a/c'); @@ -233,6 +282,13 @@ case class XPathString(xml: Expression, path: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the nodes whose values are returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); From 389914e8fa4faab2fb2f9d33af0eb08a8f4df433 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 07:15:17 +0900 Subject: [PATCH 2/9] Fix CI: scalafmt formatting and next_day docstring - Apply scalafmt to sql/api functions.scala doc comments (Scala linter). - Fix RST "unexpected unindent" in next_day docstring by de-indenting the weekday list so it is plain continuation text (Sphinx doc build). Co-authored-by: Isaac --- python/pyspark/sql/functions/builtin.py | 2 +- .../org/apache/spark/sql/functions.scala | 742 ++++++++++-------- 2 files changed, 424 insertions(+), 320 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 1b1641d1998cf..59f06c8c8399d 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -12933,7 +12933,7 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column: A column that evaluates to a date. dayOfWeek : literal string day of the week, case-insensitive, accepts: - "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column that evaluates to a string. Returns diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 48e332c5afddd..437f2109872d0 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -877,10 +877,12 @@ object functions { /** * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * - * @param hour the hour to represent, from 0 to 23. A column that evaluates to an integer. - * @param minute the minute to represent, from 0 to 59. A column that evaluates to an integer. - * @param second the second to represent, from 0 to 59.999999. A column that evaluates to a - * decimal. + * @param hour + * the hour to represent, from 0 to 23. A column that evaluates to an integer. + * @param minute + * the minute to represent, from 0 to 59. A column that evaluates to an integer. + * @param second + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. * @group datetime_funcs * @since 4.1.0 * @return @@ -3359,8 +3361,8 @@ object functions { /** * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` - * Returns a column that evaluates to a double. + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3378,8 +3380,7 @@ object functions { /** * @return - * inverse hyperbolic cosine of `e` - * Returns a column that evaluates to a double. + * inverse hyperbolic cosine of `e` Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3397,8 +3398,8 @@ object functions { /** * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` - * Returns a column that evaluates to a double. + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3416,8 +3417,7 @@ object functions { /** * @return - * inverse hyperbolic sine of `e` - * Returns a column that evaluates to a double. + * inverse hyperbolic sine of `e` Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3435,8 +3435,8 @@ object functions { /** * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan` - * Returns a column that evaluates to a double. + * inverse tangent of `e` as if computed by `java.lang.Math.atan` Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3453,13 +3453,14 @@ object functions { def atan(columnName: String): Column = atan(Column(columnName)) /** - * @param y coordinate on y-axis. A column that evaluates to a double. - * @param x coordinate on x-axis. A column that evaluates to a double. + * @param y + * coordinate on y-axis. A column that evaluates to a double. + * @param x + * coordinate on x-axis. A column that evaluates to a double. * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * Returns a column that evaluates to a double. + * `java.lang.Math.atan2` Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3574,8 +3575,7 @@ object functions { /** * @return - * inverse hyperbolic tangent of `e` - * Returns a column that evaluates to a double. + * inverse hyperbolic tangent of `e` Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3685,10 +3685,11 @@ object functions { Column.fn("conv", num, lit(fromBase), lit(toBase)) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` - * Returns a column that evaluates to a double. + * cosine of the angle, as if computed by `java.lang.Math.cos` Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -3707,10 +3708,11 @@ object functions { def cos(columnName: String): Column = cos(Column(columnName)) /** - * @param e hyperbolic angle. A column that evaluates to a double. + * @param e + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` - * Returns a column that evaluates to a double. + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3729,10 +3731,10 @@ object functions { def cosh(columnName: String): Column = cosh(Column(columnName)) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * cotangent of the angle - * Returns a column that evaluates to a double. + * cotangent of the angle Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3740,10 +3742,10 @@ object functions { def cot(e: Column): Column = Column.fn("cot", e) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * cosecant of the angle - * Returns a column that evaluates to a double. + * cosecant of the angle Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4251,10 +4253,10 @@ object functions { def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * secant of the angle - * Returns a column that evaluates to a double. + * secant of the angle Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4354,10 +4356,11 @@ object functions { def signum(columnName: String): Column = signum(Column(columnName)) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` - * Returns a column that evaluates to a double. + * sine of the angle, as if computed by `java.lang.Math.sin` Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -4376,10 +4379,11 @@ object functions { def sin(columnName: String): Column = sin(Column(columnName)) /** - * @param e hyperbolic angle. A column that evaluates to a double. + * @param e + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * Returns a column that evaluates to a double. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` Returns a + * column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4398,10 +4402,11 @@ object functions { def sinh(columnName: String): Column = sinh(Column(columnName)) /** - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` - * Returns a column that evaluates to a double. + * tangent of the given value, as if computed by `java.lang.Math.tan` Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4420,10 +4425,11 @@ object functions { def tan(columnName: String): Column = tan(Column(columnName)) /** - * @param e hyperbolic angle. A column that evaluates to a double. + * @param e + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * Returns a column that evaluates to a double. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` Returns a + * column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4459,10 +4465,11 @@ object functions { * Converts an angle measured in radians to an approximately equivalent angle measured in * degrees. * - * @param e angle in radians. A column that evaluates to a double. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` - * Returns a column that evaluates to a double. + * angle in degrees, as if computed by `java.lang.Math.toDegrees` Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4501,10 +4508,11 @@ object functions { * Converts an angle measured in degrees to an approximately equivalent angle measured in * radians. * - * @param e angle in degrees. A column that evaluates to a double. + * @param e + * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` - * Returns a column that evaluates to a double. + * angle in radians, as if computed by `java.lang.Math.toRadians` Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4530,14 +4538,18 @@ object functions { * evaluated. Note that input arguments must follow conditions listed below; otherwise, the * method will return null. * - * @param v value to compute a bucket number in the histogram. A column that evaluates to a double - * or interval. - * @param min minimum value of the histogram. A column that evaluates to a double or interval. - * @param max maximum value of the histogram. A column that evaluates to a double or interval. - * @param numBucket the number of buckets. A column that evaluates to a long. + * @param v + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. + * @param min + * minimum value of the histogram. A column that evaluates to a double or interval. + * @param max + * maximum value of the histogram. A column that evaluates to a double or interval. + * @param numBucket + * the number of buckets. A column that evaluates to a long. * @return - * the bucket number into which the value would fall after being evaluated - * Returns a column that evaluates to a long. + * the bucket number into which the value would fall after being evaluated Returns a column + * that evaluates to a long. * @group math_funcs * @since 3.5.0 */ @@ -4614,8 +4626,10 @@ object functions { * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a * hex string. * - * @param e column to compute SHA-2 on. A column that evaluates to a binary. - * @param numBits one of 224, 256, 384, or 512. A column that evaluates to an integer. + * @param e + * column to compute SHA-2 on. A column that evaluates to a binary. + * @param numBits + * one of 224, 256, 384, or 512. A column that evaluates to an integer. * * @group hash_funcs * @since 1.5.0 @@ -4733,17 +4747,23 @@ object functions { * for encryption, the identical AAD value must be provided for decryption. The default mode is * GCM. * - * @param input The binary value to encrypt. A column that evaluates to a binary. - * @param key The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode Specifies which block cipher mode should be used to encrypt messages. Valid modes: - * ECB, GCM, CBC. A column that evaluates to a string. - * @param padding Specifies how to pad messages whose length is not a multiple of the block size. - * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and - * PKCS for CBC. A column that evaluates to a string. - * @param iv Optional initialization vector. Only supported for CBC and GCM modes. Valid values: - * None or "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates - * to a binary. - * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any * free-form input and must be provided for both encryption and decryption. A column that * evaluates to a binary. * @@ -4819,14 +4839,19 @@ object functions { * only supported for GCM. If provided for encryption, the identical AAD value must be provided * for decryption. The default mode is GCM. * - * @param input The binary value to decrypt. A column that evaluates to a binary. - * @param key The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode Specifies which block cipher mode should be used to decrypt messages. Valid modes: - * ECB, GCM, CBC. A column that evaluates to a string. - * @param padding Specifies how to pad messages whose length is not a multiple of the block size. - * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and - * PKCS for CBC. A column that evaluates to a string. - * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any * free-form input and must be provided for both encryption and decryption. A column that * evaluates to a binary. * @@ -4883,14 +4908,19 @@ object functions { * This is a special version of `aes_decrypt` that performs the same operation, but returns a * NULL value instead of raising an error if the decryption cannot be performed. * - * @param input The binary value to decrypt. A column that evaluates to a binary. - * @param key The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode Specifies which block cipher mode should be used to decrypt messages. Valid modes: - * ECB, GCM, CBC. A column that evaluates to a string. - * @param padding Specifies how to pad messages whose length is not a multiple of the block size. - * Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and - * PKCS for CBC. A column that evaluates to a string. - * @param aad Optional additional authenticated data. Only supported for GCM mode. This can be any + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any * free-form input and must be provided for both encryption and decryption. A column that * evaluates to a binary. * @@ -5427,8 +5457,7 @@ object functions { * Computes the Levenshtein distance of the two given string columns if it's less than or equal * to a given threshold. * @return - * result distance, or -1 - * Returns a column that evaluates to an integer. + * result distance, or -1 Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -5807,9 +5836,11 @@ object functions { /** * Splits str around matches of the given pattern. * - * @param str a string expression to split - * @param pattern a string representing a regular expression. The regex string should be a Java - * regular expression. + * @param str + * a string expression to split + * @param pattern + * a string representing a regular expression. The regex string should be a Java regular + * expression. * * @group string_funcs * @since 1.5.0 @@ -6120,11 +6151,14 @@ object functions { /** * Replaces all occurrences of `search` with `replace`. * - * @param src A column of string to be replaced. A column that evaluates to a string. - * @param search A column of string, If `search` is not found in `str`, `str` is returned - * unchanged. A column that evaluates to a string. - * @param replace A column of string, If `replace` is not specified or is an empty string, nothing - * replaces the string that is removed from `str`. A column that evaluates to a string. + * @param src + * A column of string to be replaced. A column that evaluates to a string. + * @param search + * A column of string, If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. + * @param replace + * A column of string, If `replace` is not specified or is an empty string, nothing replaces + * the string that is removed from `str`. A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 @@ -7786,14 +7820,16 @@ object functions { /** * Returns the date that is `numMonths` after `startDate`. * - * @param startDate A date, timestamp or string. If a string, the data must be in a format that - * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a date. - * @param numMonths The number of months to add to `startDate`, can be negative to subtract - * months. A column that evaluates to an integer. + * @param startDate + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param numMonths + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date - * Returns a column that evaluates to a date. + * A date, or null if `startDate` was a string that could not be cast to a date Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7876,14 +7912,16 @@ object functions { * See Datetime * Patterns for valid date and time format patterns * - * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that can - * be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a timestamp or time. - * @param format A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that - * evaluates to a string. + * @param dateExpr + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. + * @param format + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp - * Returns a column that evaluates to a string. + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp Returns + * a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a * specialized implementation. @@ -7898,14 +7936,16 @@ object functions { /** * Returns the date that is `days` days after `start` * - * @param start A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param days The number of days to add to `start`, can be negative to subtract days. A column - * that evaluates to an integer, short, or byte. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date - * Returns a column that evaluates to a date. + * A date, or null if `start` was a string that could not be cast to a date Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7929,14 +7969,16 @@ object functions { /** * Returns the date that is `days` days after `start` * - * @param start A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param days A column of the number of days to add to `start`, can be negative to subtract days. - * A column that evaluates to an integer, short, or byte. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date - * Returns a column that evaluates to a date. + * A date, or null if `start` was a string that could not be cast to a date Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.5.0 */ @@ -7945,14 +7987,16 @@ object functions { /** * Returns the date that is `days` days before `start` * - * @param start A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param days The number of days to subtract from `start`, can be negative to add days. A column - * that evaluates to an integer, short, or byte. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date - * Returns a column that evaluates to a date. + * A date, or null if `start` was a string that could not be cast to a date Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7983,16 +8027,17 @@ object functions { * // returns 1 * }}} * - * @param end A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param start A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` - * Returns a column that evaluates to an integer. + * date. Negative if `end` is before `start` Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8007,16 +8052,17 @@ object functions { * // returns 1 * }}} * - * @param end A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param start A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` - * Returns a column that evaluates to an integer. + * date. Negative if `end` is before `start` Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -8035,8 +8081,8 @@ object functions { /** * Extracts the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8045,8 +8091,8 @@ object functions { /** * Extracts the quarter as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8055,8 +8101,8 @@ object functions { /** * Extracts the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8066,8 +8112,8 @@ object functions { * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 * for a Sunday through to 7 for a Saturday * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 */ @@ -8076,8 +8122,8 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8086,8 +8132,8 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -8096,8 +8142,8 @@ object functions { /** * Extracts the day of the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8108,8 +8154,8 @@ object functions { * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8118,11 +8164,13 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field selects which part of the source should be extracted. - * @param source a date/timestamp or interval column from where `field` should be extracted. + * @param field + * selects which part of the source should be extracted. + * @param source + * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -8133,12 +8181,14 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field selects which part of the source should be extracted, and supported string values - * are as same as the fields of the equivalent function `extract`. - * @param source a date/timestamp or interval column from where `field` should be extracted. + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `extract`. + * @param source + * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -8149,12 +8199,14 @@ object functions { /** * Extracts a part of the date/timestamp or interval source. * - * @param field selects which part of the source should be extracted, and supported string values - * are as same as the fields of the equivalent function `EXTRACT`. - * @param source a date/timestamp or interval column from where `field` should be extracted. + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `EXTRACT`. + * @param source + * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -8166,12 +8218,13 @@ object functions { * Returns the last day of the month which the given date belongs to. For example, input * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. * - * @param e A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. + * @param e + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * A date, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to a date. + * A date, or null if the input was a string that could not be cast to a date Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8182,8 +8235,8 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8201,8 +8254,7 @@ object functions { /** * @return - * A date created from year, month and day fields. - * Returns a column that evaluates to a date. + * A date created from year, month and day fields. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.3.0 */ @@ -8223,14 +8275,15 @@ object functions { * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 * }}} * - * @param end A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` - * @param start A date, timestamp or string. If a string, the data must be in a format that can - * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` * @return * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start` - * Returns a column that evaluates to a double. + * timestamp. Negative if `end` is before `start` Returns a column that evaluates to a double. * @group datetime_funcs * @since 1.5.0 */ @@ -8253,15 +8306,16 @@ object functions { * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first * Sunday after 2015-07-27. * - * @param date A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param dayOfWeek Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", - * "Sun". A column that evaluates to a string. + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param dayOfWeek + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value - * Returns a column that evaluates to a date. + * was an invalid value Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8294,8 +8348,8 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a timestamp - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a timestamp Returns + * a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8308,8 +8362,8 @@ object functions { * as defined by ISO 8601 * * @return - * An integer, or null if the input was a string that could not be cast to a date - * Returns a column that evaluates to an integer. + * An integer, or null if the input was a string that could not be cast to a date Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8320,11 +8374,12 @@ object functions { * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd * HH:mm:ss format. * - * @param ut A number of a type that is castable to a long, such as string or integer. Can be - * negative for timestamps before the unix epoch + * @param ut + * A number of a type that is castable to a long, such as string or integer. Can be negative + * for timestamps before the unix epoch * @return - * A string, or null if the input was a string that could not be cast to a long - * Returns a column that evaluates to a string. + * A string, or null if the input was a string that could not be cast to a long Returns a + * column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8403,10 +8458,11 @@ object functions { /** * Parses a string value to a time value. * - * @param str A string to be parsed to time. + * @param str + * A string to be parsed to time. * @return - * A time, or raises an error if the input is malformed. - * Returns a column that evaluates to a string. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * string. * * @group datetime_funcs * @since 4.1.0 @@ -8438,9 +8494,10 @@ object functions { /** * Converts to a timestamp by casting rules to `TimestampType`. * - * @param s A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, timestamp_ntz, or numeric. * @return * A timestamp, or null if the input was a string that could not be cast to a timestamp * Returns a column that evaluates to a timestamp. @@ -8471,10 +8528,10 @@ object functions { /** * Parses a string value to a time value. * - * @param str A string to be parsed to time. + * @param str + * A string to be parsed to time. * @return - * A time, or null if the input is malformed. - * Returns a column that evaluates to a string. + * A time, or null if the input is malformed. Returns a column that evaluates to a string. * * @group datetime_funcs * @since 4.1.0 @@ -8632,16 +8689,17 @@ object functions { * * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 * - * @param date A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates - * to a date. - * @param format: 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by - * month Other options are: 'week', 'quarter'. A column that evaluates to a string. + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value - * Returns a column that evaluates to a date. + * invalid value Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8652,16 +8710,17 @@ object functions { * * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 * - * @param format: 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by - * month, 'day', 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', - * 'second', 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. - * @param timestamp A date, timestamp or string. If a string, the data must be in a format that - * can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a timestamp. + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', + * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. + * @param timestamp + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value - * Returns a column that evaluates to a timestamp. + * `format` was an invalid value Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8673,19 +8732,20 @@ object functions { * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 * 03:40:00.0'. * - * @param ts A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a timestamp. - * @param tz A string detailing the time zone ID that the input should be adjusted to. It should - * be in the format of either region-based zone IDs or zone offsets. Region IDs must have the - * form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value - * Returns a column that evaluates to a timestamp. + * an invalid value Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8706,19 +8766,20 @@ object functions { * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield * '2017-07-14 01:40:00.0'. * - * @param ts A date, timestamp or string. If a string, the data must be in a format that can be - * cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that - * evaluates to a timestamp. - * @param tz A string detailing the time zone ID that the input should be adjusted to. It should - * be in the format of either region-based zone IDs or zone offsets. Region IDs must have the - * form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value - * Returns a column that evaluates to a timestamp. + * an invalid value Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8899,11 +8960,12 @@ object functions { * For a streaming query, you may use the function `current_timestamp` to generate windows on * processing time. * - * @param timeColumn The column or the expression to use as the timestamp for windowing by time. - * The time column must be of TimestampType or TimestampNTZType. - * @param gapDuration A string specifying the timeout of the session, e.g. `10 minutes`, `1 - * second`. Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration - * identifiers. + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. + * @param gapDuration + * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. * * @group datetime_funcs * @since 3.2.0 @@ -9044,14 +9106,17 @@ object functions { * Returns the difference between two times, measured in specified units. Throws a * SparkIllegalArgumentException, in case the specified unit is not supported. * - * @param unit A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A column - * that evaluates to a string. - * @param start A starting TIME. A column that evaluates to a time. - * @param end An ending TIME. A column that evaluates to a time. + * @param unit + * A STRING representing the unit of the time difference. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param start + * A starting TIME. A column that evaluates to a time. + * @param end + * An ending TIME. A column that evaluates to a time. * @return - * The difference between `end` and `start` times, measured in specified units. - * Returns a column that evaluates to a long. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @group datetime_funcs @@ -9064,13 +9129,14 @@ object functions { /** * Returns `time` truncated to the `unit`. * - * @param unit A STRING representing the unit to truncate the time to. Supported units are: - * "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * @param unit + * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A * column that evaluates to a string. - * @param time A TIME to truncate. A column that evaluates to a time. + * @param time + * A TIME to truncate. A column that evaluates to a time. * @return - * A TIME truncated to the specified unit. - * Returns a column that evaluates to a time. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @throws IllegalArgumentException @@ -9264,9 +9330,12 @@ object functions { * Returns an array containing all the elements in `x` from index `start` (or starting from the * end if `start` is negative) with the specified `length`. * - * @param x the array column to be sliced. A column that evaluates to an array. - * @param start the starting index. A column that evaluates to an integer. - * @param length the length of the slice. A column that evaluates to an integer. + * @param x + * the array column to be sliced. A column that evaluates to an array. + * @param start + * the starting index. A column that evaluates to an integer. + * @param length + * the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 2.4.0 @@ -9523,8 +9592,10 @@ object functions { * df.select(transform(col("i"), x => x + 1)) * }}} * - * @param column the input array column. A column that evaluates to an array. - * @param f col => transformed_col, the lambda function to transform the input column. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => transformed_col, the lambda function to transform the input column. * * @group collection_funcs * @since 3.0.0 @@ -9559,8 +9630,10 @@ object functions { * df.select(exists(col("i"), _ % 2 === 0)) * }}} * - * @param column the input array column. A column that evaluates to an array. - * @param f col => predicate, the Boolean predicate to check the input column. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 @@ -9576,8 +9649,10 @@ object functions { * df.select(forall(col("i"), x => x % 2 === 0)) * }}} * - * @param column the input array column. A column that evaluates to an array. - * @param f col => predicate, the Boolean predicate to check the input column. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 @@ -9593,8 +9668,10 @@ object functions { * df.select(filter(col("s"), x => x % 2 === 0)) * }}} * - * @param column the input array column. A column that evaluates to an array. - * @param f col => predicate, the Boolean predicate to filter the input column. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to filter the input column. * * @group collection_funcs * @since 3.0.0 @@ -9630,12 +9707,16 @@ object functions { * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) * }}} * - * @param expr the input array column. A column that evaluates to an array. - * @param initialValue the initial value. A column of any type. - * @param merge (combined_value, input_value) => combined_value, the merge function to merge an - * input value to the combined_value. - * @param finish combined_value => final_value, the lambda function to convert the combined value - * of all inputs to final result. + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. * * @group collection_funcs * @since 3.0.0 @@ -9677,12 +9758,16 @@ object functions { * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) * }}} * - * @param expr the input array column. A column that evaluates to an array. - * @param initialValue the initial value. A column of any type. - * @param merge (combined_value, input_value) => combined_value, the merge function to merge an - * input value to the combined_value. - * @param finish combined_value => final_value, the lambda function to convert the combined value - * of all inputs to final result. + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. * * @group collection_funcs * @since 3.5.0 @@ -9724,9 +9809,12 @@ object functions { * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) * }}} * - * @param left the left input array column. A column that evaluates to an array. - * @param right the right input array column. A column that evaluates to an array. - * @param f (lCol, rCol) => col, the lambda function to merge two input columns into one column. + * @param left + * the left input array column. A column that evaluates to an array. + * @param right + * the right input array column. A column that evaluates to an array. + * @param f + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. * * @group collection_funcs * @since 3.0.0 @@ -10088,10 +10176,13 @@ object functions { * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns * `null`, in the case of an unparseable string. * - * @param e a string column containing JSON data. - * @param schema the schema to use when parsing the json string - * @param options options to control how the json is parsed. accepts the same options and the json - * data source. See Data * Source Option in the version you use. * @@ -10114,7 +10205,8 @@ object functions { * Parses a JSON string and constructs a Variant value. Returns null if the input string is not * a valid JSON value. * - * @param json a string column that contains JSON data. + * @param json + * a string column that contains JSON data. * * @group variant_funcs * @since 4.0.0 @@ -10126,7 +10218,8 @@ object functions { /** * Parses a JSON string and constructs a Variant value. * - * @param json a string column that contains JSON data. A column that evaluates to a string. + * @param json + * a string column that contains JSON data. A column that evaluates to a string. * @group variant_funcs * @since 4.0.0 * @return @@ -10139,7 +10232,8 @@ object functions { * structs are converted to variant objects which are unordered unlike SQL structs. Input maps * can only have string keys. * - * @param col a column with a nested schema or column name. + * @param col + * a column with a nested schema or column name. * @group variant_funcs * @since 4.0.0 * @return @@ -10361,14 +10455,16 @@ object functions { /** * Parses a JSON string and infers its schema in DDL format using options. * - * @param json a foldable string column containing JSON data. - * @param options options to control how the json is parsed. accepts the same options and the json - * data source. See Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. - * Returns a column that evaluates to an integer. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to an integer. * * @group json_funcs * @since 3.0.0 @@ -10404,9 +10500,11 @@ object functions { * a JSON string with the specified schema. Throws an exception, in the case of an unsupported * type. * - * @param e a column containing a struct, an array, a map, or a variant. - * @param options options to control how the struct column is converted into a json string. - * accepts the same options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` * option which enables pretty JSON generation. @@ -10424,9 +10522,11 @@ object functions { * a JSON string with the specified schema. Throws an exception, in the case of an unsupported * type. * - * @param e a column containing a struct, an array, a map, or a variant. - * @param options options to control how the struct column is converted into a json string. - * accepts the same options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` * option which enables pretty JSON generation. @@ -10442,7 +10542,8 @@ object functions { * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string * with the specified schema. Throws an exception, in the case of an unsupported type. * - * @param e a column containing a struct, an array, a map, or a variant. + * @param e + * a column containing a struct, an array, a map, or a variant. * * @group json_funcs * @since 2.1.0 @@ -10457,7 +10558,8 @@ object functions { * with 'n'. This can be useful for creating copies of tables with sensitive information * removed. * - * @param input string value to mask. Supported types: STRING, VARCHAR, CHAR + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR * * @group string_funcs * @since 3.5.0 @@ -11209,12 +11311,14 @@ object functions { * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to * `targetTz`. * - * @param sourceTz the time zone for the input timestamp. If it is missed, the current session - * time zone is used as the source time zone. A column that evaluates to a string. - * @param targetTz the time zone to which the input timestamp should be converted. A column that - * evaluates to a string. - * @param sourceTs a timestamp without time zone. A column that evaluates to a timestamp_ntz or - * timestamp. + * @param sourceTz + * the time zone for the input timestamp. If it is missed, the current session time zone is + * used as the source time zone. A column that evaluates to a string. + * @param targetTz + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. + * @param sourceTs + * a timestamp without time zone. A column that evaluates to a timestamp_ntz or timestamp. * @group datetime_funcs * @since 3.5.0 * @return From e5091caa4bb958d4ae5acf621d3ce567b737e290 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 09:06:37 +0900 Subject: [PATCH 3/9] Address review: correct type-gated "any type" docs and Scala @return run-ons - approx_top_k/approx_top_k_accumulate: expr is gated by isDataTypeSupported (atomic allowlist), not any type - decode: bin/charset are binary/string, not any type - comparison ops (=, <=>, <, <=, >, >=), equal_null, nullif, nullifzero, max_by/min_by ord: gated by checkForOrderingExpr -> "any orderable type" - functions.scala: add missing period before appended "Returns a column..." sentence in 33 @return blocks (run-on Scaladoc) Co-authored-by: Isaac --- python/pyspark/sql/functions/builtin.py | 14 ++-- .../org/apache/spark/sql/functions.scala | 66 +++++++++---------- .../aggregate/ApproxTopKAggregates.scala | 4 +- .../expressions/nullExpressions.scala | 6 +- .../sql/catalyst/expressions/predicates.scala | 14 ++-- .../expressions/stringExpressions.scala | 4 +- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 59f06c8c8399d..134a9b4f01829 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -1380,7 +1380,7 @@ def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> ord : :class:`~pyspark.sql.Column` or column name The column that needs to be maximized. This could be the column instance or the column name as string. - A column of any type. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the top `k` maximum ordering values, sorted in descending order by the ordering column. @@ -1494,7 +1494,7 @@ def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> ord : :class:`~pyspark.sql.Column` or column name The column that needs to be minimized. This could be the column instance or the column name as string. - A column of any type. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the bottom `k` minimum ordering values, sorted in ascending order by the ordering column. @@ -30626,9 +30626,9 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name - A column of any type. + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name - A column of any type. + A column of any orderable type. Examples -------- @@ -30663,9 +30663,9 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name - A column of any type. + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name - A column of any type. + A column of any orderable type. Examples -------- @@ -30700,7 +30700,7 @@ def nullifzero(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - A column of any type. + A column of any orderable type. Examples -------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 437f2109872d0..4abcdc1ffa68c 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -3361,7 +3361,7 @@ object functions { /** * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` Returns a column + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column * that evaluates to a double. * * @group math_funcs @@ -3380,7 +3380,7 @@ object functions { /** * @return - * inverse hyperbolic cosine of `e` Returns a column that evaluates to a double. + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3398,7 +3398,7 @@ object functions { /** * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` Returns a column + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column * that evaluates to a double. * * @group math_funcs @@ -3417,7 +3417,7 @@ object functions { /** * @return - * inverse hyperbolic sine of `e` Returns a column that evaluates to a double. + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3435,7 +3435,7 @@ object functions { /** * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan` Returns a column that + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that * evaluates to a double. * * @group math_funcs @@ -3460,7 +3460,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` Returns a column that evaluates to a double. + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3575,7 +3575,7 @@ object functions { /** * @return - * inverse hyperbolic tangent of `e` Returns a column that evaluates to a double. + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3688,7 +3688,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` Returns a column that evaluates + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates * to a double. * * @group math_funcs @@ -3711,7 +3711,7 @@ object functions { * @param e * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` Returns a column + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column * that evaluates to a double. * * @group math_funcs @@ -3734,7 +3734,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * cotangent of the angle Returns a column that evaluates to a double. + * cotangent of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3745,7 +3745,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * cosecant of the angle Returns a column that evaluates to a double. + * cosecant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4256,7 +4256,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * secant of the angle Returns a column that evaluates to a double. + * secant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4359,7 +4359,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` Returns a column that evaluates + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates * to a double. * * @group math_funcs @@ -4405,7 +4405,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` Returns a column that + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that * evaluates to a double. * * @group math_funcs @@ -4468,7 +4468,7 @@ object functions { * @param e * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` Returns a column that + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that * evaluates to a double. * * @group math_funcs @@ -4511,7 +4511,7 @@ object functions { * @param e * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` Returns a column that + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that * evaluates to a double. * * @group math_funcs @@ -4548,7 +4548,7 @@ object functions { * @param numBucket * the number of buckets. A column that evaluates to a long. * @return - * the bucket number into which the value would fall after being evaluated Returns a column + * the bucket number into which the value would fall after being evaluated. Returns a column * that evaluates to a long. * @group math_funcs * @since 3.5.0 @@ -5457,7 +5457,7 @@ object functions { * Computes the Levenshtein distance of the two given string columns if it's less than or equal * to a given threshold. * @return - * result distance, or -1 Returns a column that evaluates to an integer. + * result distance, or -1. Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -7944,7 +7944,7 @@ object functions { * The number of days to add to `start`, can be negative to subtract days. A column that * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date Returns a column + * A date, or null if `start` was a string that could not be cast to a date. Returns a column * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 @@ -7977,7 +7977,7 @@ object functions { * A column of the number of days to add to `start`, can be negative to subtract days. A * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date Returns a column + * A date, or null if `start` was a string that could not be cast to a date. Returns a column * that evaluates to a date. * @group datetime_funcs * @since 3.5.0 @@ -7995,7 +7995,7 @@ object functions { * The number of days to subtract from `start`, can be negative to add days. A column that * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date Returns a column + * A date, or null if `start` was a string that could not be cast to a date. Returns a column * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 @@ -8037,7 +8037,7 @@ object functions { * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` Returns a column that evaluates to an integer. + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8062,7 +8062,7 @@ object functions { * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` Returns a column that evaluates to an integer. + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -8169,7 +8169,7 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source Returns a column that evaluates to a + * a part of the date/timestamp or interval source. Returns a column that evaluates to a * double. * @group datetime_funcs * @since 3.5.0 @@ -8187,7 +8187,7 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source Returns a column that evaluates to a + * a part of the date/timestamp or interval source. Returns a column that evaluates to a * double. * @group datetime_funcs * @since 3.5.0 @@ -8205,7 +8205,7 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source Returns a column that evaluates to a + * a part of the date/timestamp or interval source. Returns a column that evaluates to a * double. * @group datetime_funcs * @since 3.5.0 @@ -8223,7 +8223,7 @@ object functions { * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a * date. * @return - * A date, or null if the input was a string that could not be cast to a date Returns a column + * A date, or null if the input was a string that could not be cast to a date. Returns a column * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 @@ -8283,7 +8283,7 @@ object functions { * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` * @return * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start` Returns a column that evaluates to a double. + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a double. * @group datetime_funcs * @since 1.5.0 */ @@ -8315,7 +8315,7 @@ object functions { * that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value Returns a column that evaluates to a date. + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8699,7 +8699,7 @@ object functions { * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value Returns a column that evaluates to a date. + * invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8720,7 +8720,7 @@ object functions { * a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value Returns a column that evaluates to a timestamp. + * `format` was an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8745,7 +8745,7 @@ object functions { * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value Returns a column that evaluates to a timestamp. + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8779,7 +8779,7 @@ object functions { * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value Returns a column that evaluates to a timestamp. + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala index 82874c9e825fa..0fcd8aec16186 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala @@ -62,7 +62,7 @@ import org.apache.spark.unsafe.types.UTF8String arguments = """ Arguments: * expr - The expression to compute the top k most frequent items of. - An expression of any type. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. * k - The number of top items to return. An expression that evaluates to an integer. Must be a constant. * maxItemsTracked - The maximum number of items to track in the sketch. @@ -500,7 +500,7 @@ object ApproxTopKAggregateBuffer { arguments = """ Arguments: * expr - The expression whose values are accumulated into the sketch. - An expression of any type. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. * maxItemsTracked - The maximum number of items to track in the sketch. An expression that evaluates to an integer. Must be a constant. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 7f22e12789f40..d4bced7fd7444 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -191,9 +191,9 @@ private case class TypedNullLiteral(child: Expression) arguments = """ Arguments: * expr1 - The value returned when it is not equal to the other expression. - An expression of any type. + An expression of any orderable type. * expr2 - The value compared against the first expression. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -229,7 +229,7 @@ case class NullIf(left: Expression, right: Expression, replacement: Expression) arguments = """ Arguments: * expr - The expression that returns null when equal to zero. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index bf98f630d7ab3..d9f9c7fec6dab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -1110,7 +1110,7 @@ object Equality { * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1159,7 +1159,7 @@ case class EqualTo(left: Expression, right: Expression) * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1223,7 +1223,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1260,7 +1260,7 @@ case class EqualNull(left: Expression, right: Expression, replacement: Expressio and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1296,7 +1296,7 @@ case class LessThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1332,7 +1332,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: @@ -1369,7 +1369,7 @@ case class GreaterThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. - An expression of any type. + An expression of any orderable type. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 2144c76930f72..2aed0d9c3abe4 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -3383,9 +3383,9 @@ object Decode { arguments = """ Arguments: * bin - a binary expression to decode - An expression of any type. + An expression that evaluates to a binary. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to decode `bin` into a STRING. It is case insensitive. - An expression of any type. + An expression that evaluates to a string. """, examples = """ Examples: From 34c2a5ef7f15032aaa507d6e830727e25b7bbb76 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 09:27:08 +0900 Subject: [PATCH 4/9] Address review: fix remaining 16 run-on @return blocks in functions.scala The earlier sweep matched "Returns a column" on a single line, missing 16 blocks where the appended sentence wraps as "...`sinh` Returns a" / "column..." across two lines. Add the missing period before "Returns" in sinh, tanh, add_months, date_format, the date-part extractors (year/quarter/month/ dayofweek/dayofmonth/dayofyear and siblings), last_day, and 3 more. Co-authored-by: Isaac --- .../org/apache/spark/sql/functions.scala | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 4abcdc1ffa68c..106f303fc0844 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -4382,7 +4382,7 @@ object functions { * @param e * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` Returns a + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a * column that evaluates to a double. * * @group math_funcs @@ -4428,7 +4428,7 @@ object functions { * @param e * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` Returns a + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a * column that evaluates to a double. * * @group math_funcs @@ -7828,7 +7828,7 @@ object functions { * The number of months to add to `startDate`, can be negative to subtract months. A column * that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date Returns a + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 @@ -7920,7 +7920,7 @@ object functions { * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to * a string. * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp Returns + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns * a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a @@ -8081,7 +8081,7 @@ object functions { /** * Extracts the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8091,7 +8091,7 @@ object functions { /** * Extracts the quarter as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8101,7 +8101,7 @@ object functions { /** * Extracts the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8112,7 +8112,7 @@ object functions { * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 * for a Sunday through to 7 for a Saturday * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 @@ -8122,7 +8122,7 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8132,7 +8132,7 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 @@ -8142,7 +8142,7 @@ object functions { /** * Extracts the day of the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8154,7 +8154,7 @@ object functions { * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8235,7 +8235,7 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8348,7 +8348,7 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a timestamp Returns + * An integer, or null if the input was a string that could not be cast to a timestamp. Returns * a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8362,7 +8362,7 @@ object functions { * as defined by ISO 8601 * * @return - * An integer, or null if the input was a string that could not be cast to a date Returns a + * An integer, or null if the input was a string that could not be cast to a date. Returns a * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 @@ -8378,7 +8378,7 @@ object functions { * A number of a type that is castable to a long, such as string or integer. Can be negative * for timestamps before the unix epoch * @return - * A string, or null if the input was a string that could not be cast to a long Returns a + * A string, or null if the input was a string that could not be cast to a long. Returns a * column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 From 2eb0662e4b5eb046ccc5fa98c88bbbc45a50538d Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 09:48:51 +0900 Subject: [PATCH 5/9] Address review: fix last @return run-on (to_timestamp) and any_value wording - to_timestamp @return: the 17th run-on block missed by the prior sweep (prose line ended without a period, fusing with the next "Returns a column..." line). Add the terminal period. - any_value: "The expression to return some value of for a group of rows." read broken ("of for"); match the first/last peers: "...some value of." Co-authored-by: Isaac --- sql/api/src/main/scala/org/apache/spark/sql/functions.scala | 2 +- .../spark/sql/catalyst/expressions/aggregate/AnyValue.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 106f303fc0844..7776ab7f766ae 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -8499,7 +8499,7 @@ object functions { * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to * a string, date, timestamp, timestamp_ntz, or numeric. * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp + * A timestamp, or null if the input was a string that could not be cast to a timestamp. * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala index e23bd449e8626..86192a955b889 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.types._ If `isIgnoreNull` is true, returns only non-null values.""", arguments = """ Arguments: - * expr - The expression to return some value of for a group of rows. + * expr - The expression to return some value of. An expression of any type. * isIgnoreNull - Whether to return only non-null values. An expression that evaluates to a boolean. Must be a constant. From 6c64ef947fad389dcaf4c6b78c6d6d43975362cb Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 10:19:42 +0900 Subject: [PATCH 6/9] Document @return type for all remaining functions.scala functions; fix review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consistency: many functions.scala functions documented @param but had no @return (or only one overload did), while the PR documents return types elsewhere. Add "Returns a column..." @return to every function/overload that was missing one (355 functions), leaving functions where a value return type is not meaningful (sort builders asc/desc, column constructors col/column, partition transforms bucket/days/months/years, UDF/dispatch udf/udaf/callUDF/ reflect, generators posexplode) without one. Return types derived from the ExpressionsSchemaSuite golden file (sql-expression-schema.md) and Catalyst dataType; polymorphic functions (max/min/first/last/coalesce/greatest/...) use "of the same type as the input" rather than the misleading example type. Review findings (pullrequestreview-4667871389): - to_time/try_to_time @return said "string" but return TimeType — corrected on both Scala @return and PySpark Returns docstrings. - bit_or: replaced content-free "The expr1 argument." arg docs with wording matching the BitwiseAnd/BitwiseXor peers ("The first/second operand ..."). Co-authored-by: Isaac --- python/pyspark/sql/functions/builtin.py | 4 +- .../org/apache/spark/sql/functions.scala | 1336 ++++++++++++++++- .../expressions/bitwiseExpressions.scala | 4 +- 3 files changed, 1270 insertions(+), 74 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 134a9b4f01829..a1d9d0f786b3c 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -12213,7 +12213,7 @@ def to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Col ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. - Returns a column that evaluates to a string. + Returns a column that evaluates to a time. See Also -------- @@ -12360,7 +12360,7 @@ def try_to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. - Returns a column that evaluates to a string. + Returns a column that evaluates to a time. See Also -------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 7776ab7f766ae..6d30d6c1751ca 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -113,6 +113,8 @@ object functions { * * @group normal_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def lit(literal: Any): Column = { literal match { @@ -135,6 +137,8 @@ object functions { * * @group normal_funcs * @since 2.2.0 + * @return + * Returns a column of the same type as the input. */ def typedLit[T: TypeTag](literal: T): Column = { typedlit(literal) @@ -154,6 +158,8 @@ object functions { * * @group normal_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def typedlit[T: TypeTag](literal: T): Column = { literal match { @@ -246,6 +252,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column): Column = approx_count_distinct(e) @@ -253,6 +261,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) @@ -260,6 +270,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) @@ -267,6 +279,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String, rsd: Double): Column = { @@ -278,6 +292,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) @@ -286,6 +302,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String): Column = approx_count_distinct( column(columnName)) @@ -298,6 +316,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column, rsd: Double): Column = { Column.fn("approx_count_distinct", e, lit(rsd)) @@ -311,6 +331,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String, rsd: Double): Column = { approx_count_distinct(Column(columnName), rsd) @@ -321,6 +343,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(e: Column): Column = Column.fn("avg", e) @@ -329,6 +353,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(columnName: String): Column = avg(Column(columnName)) @@ -341,6 +367,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(e: Column): Column = Column.fn("collect_list", e) @@ -353,6 +381,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(columnName: String): Column = collect_list(Column(columnName)) @@ -365,6 +395,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(e: Column): Column = Column.fn("collect_set", e) @@ -377,6 +409,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(columnName: String): Column = collect_set(Column(columnName)) @@ -388,6 +422,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = Column.fn("count_min_sketch", e, eps, confidence, seed) @@ -400,6 +436,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) @@ -409,6 +447,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) @@ -417,6 +457,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(columnName1: String, columnName2: String): Column = { corr(Column(columnName1), Column(columnName2)) @@ -427,6 +469,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(e: Column): Column = Column.fn("count", e) @@ -436,6 +480,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(columnName: String): TypedColumn[Any, Long] = count(Column(columnName)).as(PrimitiveLongEncoder) @@ -447,6 +493,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) @@ -458,6 +506,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(columnName: String, columnNames: String*): Column = @@ -468,6 +518,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def count_distinct(expr: Column, exprs: Column*): Column = @@ -478,6 +530,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(column1: Column, column2: Column): Column = Column.fn("covar_pop", column1, column2) @@ -487,6 +541,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(columnName1: String, columnName2: String): Column = { covar_pop(Column(columnName1), Column(columnName2)) @@ -497,6 +553,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(column1: Column, column2: Column): Column = Column.fn("covar_samp", column1, column2) @@ -506,6 +564,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(columnName1: String, columnName2: String): Column = { covar_samp(Column(columnName1), Column(columnName2)) @@ -523,6 +583,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column, ignoreNulls: Boolean): Column = Column.fn("first", false, e, lit(ignoreNulls)) @@ -539,6 +601,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String, ignoreNulls: Boolean): Column = { first(Column(columnName), ignoreNulls) @@ -556,6 +620,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column): Column = first(e, ignoreNulls = false) @@ -571,6 +637,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String): Column = first(Column(columnName)) @@ -583,6 +651,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column): Column = Column.fn("first_value", e) @@ -598,6 +668,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column, ignoreNulls: Column): Column = Column.fn("first_value", e, ignoreNulls) @@ -608,6 +680,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(e: Column): Column = Column.fn("grouping", e) @@ -617,6 +691,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(columnName: String): Column = grouping(Column(columnName)) @@ -633,6 +709,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) @@ -649,6 +727,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(colName: String, colNames: String*): Column = { @@ -661,6 +741,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Column): Column = Column.fn("hll_sketch_agg", e, lgConfigK) @@ -671,6 +753,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Int): Column = Column.fn("hll_sketch_agg", e, lit(lgConfigK)) @@ -681,6 +765,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { hll_sketch_agg(Column(columnName), lgConfigK) @@ -692,6 +778,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column): Column = Column.fn("hll_sketch_agg", e) @@ -702,6 +790,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String): Column = { hll_sketch_agg(Column(columnName)) @@ -715,6 +805,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = Column.fn("hll_union_agg", e, allowDifferentLgConfigK) @@ -727,6 +819,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) @@ -739,6 +833,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { hll_union_agg(Column(columnName), allowDifferentLgConfigK) @@ -751,6 +847,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column): Column = Column.fn("hll_union_agg", e) @@ -762,6 +860,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String): Column = { hll_union_agg(Column(columnName)) @@ -772,6 +872,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(e: Column): Column = Column.fn("kurtosis", e) @@ -780,6 +882,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) @@ -795,6 +899,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column, ignoreNulls: Boolean): Column = Column.fn("last", false, e, lit(ignoreNulls)) @@ -811,6 +917,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String, ignoreNulls: Boolean): Column = { last(Column(columnName), ignoreNulls) @@ -828,6 +936,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column): Column = last(e, ignoreNulls = false) @@ -843,6 +953,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) @@ -855,6 +967,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column): Column = Column.fn("last_value", e) @@ -870,6 +984,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column, ignoreNulls: Column): Column = Column.fn("last_value", e, ignoreNulls) @@ -897,6 +1013,8 @@ object functions { * * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column): Column = Column.fn("mode", e) @@ -909,6 +1027,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) @@ -917,6 +1037,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(e: Column): Column = Column.fn("max", e) @@ -925,6 +1047,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(columnName: String): Column = max(Column(columnName)) @@ -937,6 +1061,8 @@ object functions { * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) @@ -955,6 +1081,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) @@ -973,6 +1101,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) @@ -981,6 +1111,8 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(e: Column): Column = avg(e) @@ -989,6 +1121,8 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(columnName: String): Column = avg(columnName) @@ -997,6 +1131,8 @@ object functions { * * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a double. */ def median(e: Column): Column = Column.fn("median", e) @@ -1005,6 +1141,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(e: Column): Column = Column.fn("min", e) @@ -1013,6 +1151,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(columnName: String): Column = min(Column(columnName)) @@ -1025,6 +1165,8 @@ object functions { * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) @@ -1044,6 +1186,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) @@ -1063,6 +1207,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) @@ -1072,6 +1218,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) @@ -1081,6 +1229,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column, frequency: Column): Column = Column.fn("percentile", e, percentage, frequency) @@ -1099,6 +1249,8 @@ object functions { * * @group agg_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = Column.fn("percentile_approx", e, percentage, accuracy) @@ -1117,6 +1269,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { Column.fn("approx_percentile", e, percentage, accuracy) @@ -1127,6 +1281,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a double. */ def product(e: Column): Column = Column.internalFn("product", e) @@ -1135,6 +1291,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(e: Column): Column = Column.fn("skewness", e) @@ -1143,6 +1301,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(columnName: String): Column = skewness(Column(columnName)) @@ -1151,6 +1311,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def std(e: Column): Column = Column.fn("std", e) @@ -1159,6 +1321,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(e: Column): Column = Column.fn("stddev", e) @@ -1167,6 +1331,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(columnName: String): Column = stddev(Column(columnName)) @@ -1175,6 +1341,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) @@ -1183,6 +1351,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) @@ -1191,6 +1361,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) @@ -1199,6 +1371,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) @@ -1207,6 +1381,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(e: Column): Column = Column.fn("sum", e) @@ -1215,6 +1391,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(columnName: String): Column = sum(Column(columnName)) @@ -1223,6 +1401,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(e: Column): Column = sum_distinct(e) @@ -1232,6 +1412,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) @@ -1241,6 +1423,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) @@ -1251,6 +1435,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(e: Column): Column = Column.fn("theta_intersection_agg", e) @@ -1262,6 +1448,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(columnName: String): Column = theta_intersection_agg(Column(columnName)) @@ -1273,6 +1461,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_sketch_agg", e, lgNomEntries) @@ -1284,6 +1474,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) @@ -1295,6 +1487,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = theta_sketch_agg(Column(columnName), lgNomEntries) @@ -1306,6 +1500,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column): Column = Column.fn("theta_sketch_agg", e) @@ -1317,6 +1513,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String): Column = theta_sketch_agg(Column(columnName)) @@ -1329,6 +1527,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_union_agg", e, lgNomEntries) @@ -1341,6 +1541,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_union_agg", e, lit(lgNomEntries)) @@ -1353,6 +1555,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String, lgNomEntries: Int): Column = theta_union_agg(Column(columnName), lgNomEntries) @@ -1365,6 +1569,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column): Column = Column.fn("theta_union_agg", e) @@ -1377,6 +1583,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String): Column = theta_union_agg(Column(columnName)) @@ -1389,6 +1597,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_double", e, mode) @@ -1401,6 +1611,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_double", e, lit(mode)) @@ -1413,6 +1625,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String, mode: String): Column = tuple_intersection_agg_double(Column(columnName), mode) @@ -1425,6 +1639,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column): Column = Column.fn("tuple_intersection_agg_double", e) @@ -1437,6 +1653,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String): Column = tuple_intersection_agg_double(Column(columnName)) @@ -1450,6 +1668,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_integer", e, mode) @@ -1463,6 +1683,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_integer", e, lit(mode)) @@ -1476,6 +1698,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String, mode: String): Column = tuple_intersection_agg_integer(Column(columnName), mode) @@ -1488,6 +1712,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column): Column = Column.fn("tuple_intersection_agg_integer", e) @@ -1500,6 +1726,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String): Column = tuple_intersection_agg_integer(Column(columnName)) @@ -1512,6 +1740,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1528,6 +1758,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1544,6 +1776,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1559,6 +1793,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) @@ -1570,6 +1806,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1584,6 +1822,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_double", key, summary) @@ -1595,6 +1835,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) @@ -1607,6 +1849,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1623,6 +1867,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1639,6 +1885,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1654,6 +1902,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) @@ -1665,6 +1915,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1679,6 +1931,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_integer", key, summary) @@ -1690,6 +1944,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) @@ -1703,6 +1959,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) @@ -1716,6 +1974,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) @@ -1729,6 +1989,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_double(Column(columnName), lgNomEntries, mode) @@ -1741,6 +2003,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) @@ -1753,6 +2017,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_double(Column(columnName), lgNomEntries) @@ -1765,6 +2031,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column): Column = Column.fn("tuple_union_agg_double", e) @@ -1777,6 +2045,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String): Column = tuple_union_agg_double(Column(columnName)) @@ -1790,6 +2060,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) @@ -1803,6 +2075,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) @@ -1816,6 +2090,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) @@ -1828,6 +2104,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) @@ -1840,6 +2118,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries) @@ -1852,6 +2132,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column): Column = Column.fn("tuple_union_agg_integer", e) @@ -1864,6 +2146,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String): Column = tuple_union_agg_integer(Column(columnName)) @@ -1875,6 +2159,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_bigint", e, k) @@ -1886,6 +2172,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_bigint", e, lit(k)) @@ -1897,6 +2185,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String, k: Int): Column = kll_sketch_agg_bigint(Column(columnName), k) @@ -1907,6 +2197,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column): Column = Column.fn("kll_sketch_agg_bigint", e) @@ -1917,6 +2209,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String): Column = kll_sketch_agg_bigint(Column(columnName)) @@ -1928,6 +2222,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_float", e, k) @@ -1939,6 +2235,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_float", e, lit(k)) @@ -1950,6 +2248,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String, k: Int): Column = kll_sketch_agg_float(Column(columnName), k) @@ -1960,6 +2260,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column): Column = Column.fn("kll_sketch_agg_float", e) @@ -1970,6 +2272,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String): Column = kll_sketch_agg_float(Column(columnName)) @@ -1981,6 +2285,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_double", e, k) @@ -1992,6 +2298,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_double", e, lit(k)) @@ -2003,6 +2311,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String, k: Int): Column = kll_sketch_agg_double(Column(columnName), k) @@ -2013,6 +2323,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column): Column = Column.fn("kll_sketch_agg_double", e) @@ -2023,6 +2335,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String): Column = kll_sketch_agg_double(Column(columnName)) @@ -2035,6 +2349,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_merge_agg_bigint", e, k) @@ -2047,6 +2363,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_merge_agg_bigint", e, lit(k)) @@ -2059,6 +2377,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String, k: Int): Column = kll_merge_agg_bigint(Column(columnName), k) @@ -2070,6 +2390,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column): Column = Column.fn("kll_merge_agg_bigint", e) @@ -2081,6 +2403,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String): Column = kll_merge_agg_bigint(Column(columnName)) @@ -2092,6 +2416,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Column): Column = Column.fn("kll_merge_agg_float", e, k) @@ -2103,6 +2429,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Int): Column = Column.fn("kll_merge_agg_float", e, lit(k)) @@ -2114,6 +2442,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String, k: Int): Column = kll_merge_agg_float(Column(columnName), k) @@ -2124,6 +2454,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column): Column = Column.fn("kll_merge_agg_float", e) @@ -2134,6 +2466,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String): Column = kll_merge_agg_float(Column(columnName)) @@ -2145,6 +2479,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Column): Column = Column.fn("kll_merge_agg_double", e, k) @@ -2156,6 +2492,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Int): Column = Column.fn("kll_merge_agg_double", e, lit(k)) @@ -2167,6 +2505,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String, k: Int): Column = kll_merge_agg_double(Column(columnName), k) @@ -2177,6 +2517,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column): Column = Column.fn("kll_merge_agg_double", e) @@ -2187,6 +2529,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String): Column = kll_merge_agg_double(Column(columnName)) @@ -2196,6 +2540,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg(e: Column): Column = Column.fn("listagg", e) @@ -2205,6 +2551,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) @@ -2213,6 +2561,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) @@ -2222,6 +2572,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column, delimiter: Column): Column = Column.fn("listagg", isDistinct = true, e, delimiter) @@ -2231,6 +2583,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg(e: Column): Column = Column.fn("string_agg", e) @@ -2240,6 +2594,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) @@ -2249,6 +2605,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) @@ -2258,6 +2616,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column, delimiter: Column): Column = Column.fn("string_agg", isDistinct = true, e, delimiter) @@ -2267,6 +2627,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(e: Column): Column = Column.fn("variance", e) @@ -2275,6 +2637,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(columnName: String): Column = variance(Column(columnName)) @@ -2283,6 +2647,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(e: Column): Column = Column.fn("var_samp", e) @@ -2291,6 +2657,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(columnName: String): Column = var_samp(Column(columnName)) @@ -2299,6 +2667,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(e: Column): Column = Column.fn("var_pop", e) @@ -2307,6 +2677,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(columnName: String): Column = var_pop(Column(columnName)) @@ -2316,6 +2688,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) @@ -2325,6 +2699,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) @@ -2334,6 +2710,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) @@ -2344,6 +2722,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) @@ -2353,6 +2733,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) @@ -2362,6 +2744,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) @@ -2371,6 +2755,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) @@ -2380,6 +2766,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) @@ -2389,6 +2777,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) @@ -2397,6 +2787,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column): Column = Column.fn("any_value", e) @@ -2406,6 +2798,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column, ignoreNulls: Column): Column = Column.fn("any_value", e, ignoreNulls) @@ -2415,6 +2809,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def count_if(e: Column): Column = Column.fn("count_if", e) @@ -2423,8 +2819,7 @@ object functions { * 6 fractional digits of seconds. * * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2439,8 +2834,7 @@ object functions { * An integer literal in the range [0..6], indicating how many fractional digits of seconds to * include in the result. * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2461,6 +2855,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def histogram_numeric(e: Column, nBins: Column): Column = Column.fn("histogram_numeric", e, nBins) @@ -2470,6 +2866,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def every(e: Column): Column = Column.fn("every", e) @@ -2478,6 +2876,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_and(e: Column): Column = Column.fn("bool_and", e) @@ -2486,6 +2886,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def some(e: Column): Column = Column.fn("some", e) @@ -2494,6 +2896,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def any(e: Column): Column = Column.fn("any", e) @@ -2502,6 +2906,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_or(e: Column): Column = Column.fn("bool_or", e) @@ -2510,6 +2916,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_and(e: Column): Column = Column.fn("bit_and", e) @@ -2518,6 +2926,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_or(e: Column): Column = Column.fn("bit_or", e) @@ -2526,6 +2936,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_xor(e: Column): Column = Column.fn("bit_xor", e) @@ -2552,8 +2964,7 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2582,8 +2993,7 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2601,6 +3011,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def cume_dist(): Column = Column.fn("cume_dist") @@ -2617,6 +3029,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def dense_rank(): Column = Column.fn("dense_rank") @@ -2629,6 +3043,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int): Column = lag(e, offset, null) @@ -2641,6 +3057,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) @@ -2653,6 +3071,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int, defaultValue: Any): Column = { lag(Column(columnName), offset, defaultValue) @@ -2667,6 +3087,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any): Column = { lag(e, offset, defaultValue, false) @@ -2683,6 +3105,8 @@ object functions { * * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2696,6 +3120,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } @@ -2708,6 +3134,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } @@ -2720,6 +3148,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int, defaultValue: Any): Column = { lead(Column(columnName), offset, defaultValue) @@ -2734,6 +3164,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any): Column = { lead(e, offset, defaultValue, false) @@ -2750,6 +3182,8 @@ object functions { * * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2765,6 +3199,8 @@ object functions { * * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) @@ -2777,6 +3213,8 @@ object functions { * * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) @@ -2789,6 +3227,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def ntile(n: Int): Column = Column.fn("ntile", lit(n)) @@ -2805,6 +3245,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def percent_rank(): Column = Column.fn("percent_rank") @@ -2821,6 +3263,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def rank(): Column = Column.fn("rank") @@ -2829,6 +3273,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def row_number(): Column = Column.fn("row_number") @@ -2841,6 +3287,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(cols: Column*): Column = Column.fn("array", cols: _*) @@ -2850,6 +3298,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(colName: String, colNames: String*): Column = { @@ -2863,6 +3313,8 @@ object functions { * * @group map_funcs * @since 2.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map(cols: Column*): Column = Column.fn("map", cols: _*) @@ -2872,6 +3324,8 @@ object functions { * * @group struct_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) @@ -2906,6 +3360,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column): Column = Column.fn("str_to_map", text, pairDelim) @@ -2915,6 +3371,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column): Column = Column.fn("str_to_map", text) @@ -2942,6 +3400,8 @@ object functions { * * @group conditional_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) @@ -2951,6 +3411,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def input_file_name(): Column = Column.fn("input_file_name") @@ -2991,6 +3453,8 @@ object functions { * * @group misc_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use monotonically_increasing_id()", "2.0.0") def monotonicallyIncreasingId(): Column = monotonically_increasing_id() @@ -3012,6 +3476,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a long. */ def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") @@ -3040,6 +3506,8 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def negate(e: Column): Column = -e @@ -3083,6 +3551,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(): Column = rand(SparkClassUtils.random.nextLong) @@ -3109,6 +3579,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(): Column = randn(SparkClassUtils.random.nextLong) @@ -3132,6 +3604,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) @@ -3143,6 +3617,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def spark_partition_id(): Column = Column.fn("spark_partition_id") @@ -3161,6 +3637,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(colName: String): Column = sqrt(Column(colName)) @@ -3180,6 +3658,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def try_avg(e: Column): Column = Column.fn("try_avg", e) @@ -3231,6 +3711,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a numeric. */ def try_sum(e: Column): Column = Column.fn("try_sum", e) @@ -3242,6 +3724,8 @@ object functions { * * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(cols: Column*): Column = Column.fn("struct", cols: _*) @@ -3251,6 +3735,8 @@ object functions { * * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(colName: String, colNames: String*): Column = { @@ -3288,6 +3774,8 @@ object functions { * * @group bitwise_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use bitwise_not", "3.2.0") def bitwiseNOT(e: Column): Column = bitwise_not(e) @@ -3297,6 +3785,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def bitwise_not(e: Column): Column = Column.fn("~", e) @@ -3371,8 +3861,7 @@ object functions { /** * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos` - * + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3389,8 +3878,7 @@ object functions { /** * @return - * inverse hyperbolic cosine of `columnName` - * + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3408,8 +3896,7 @@ object functions { /** * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin` - * + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3426,8 +3913,7 @@ object functions { /** * @return - * inverse hyperbolic sine of `columnName` - * + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3445,8 +3931,7 @@ object functions { /** * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan` - * + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3475,8 +3960,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3490,8 +3974,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3505,8 +3988,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3521,8 +4003,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3536,8 +4017,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3551,8 +4031,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3566,8 +4045,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3584,8 +4062,7 @@ object functions { /** * @return - * inverse hyperbolic tangent of `columnName` - * + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3608,6 +4085,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(columnName: String): Column = bin(Column(columnName)) @@ -3626,6 +4105,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(columnName: String): Column = cbrt(Column(columnName)) @@ -3644,6 +4125,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column): Column = Column.fn("ceil", e) @@ -3652,6 +4135,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(columnName: String): Column = ceil(Column(columnName)) @@ -3670,6 +4155,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column): Column = Column.fn("ceiling", e) @@ -3700,8 +4187,7 @@ object functions { * @param columnName * angle in radians * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` - * + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3723,8 +4209,7 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` - * + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3757,6 +4242,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def e(): Column = Column.fn("e") @@ -3775,6 +4262,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(columnName: String): Column = exp(Column(columnName)) @@ -3793,6 +4282,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(columnName: String): Column = expm1(Column(columnName)) @@ -3821,6 +4312,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column): Column = Column.fn("floor", e) @@ -3829,6 +4322,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(columnName: String): Column = floor(Column(columnName)) @@ -3838,6 +4333,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) @@ -3848,6 +4345,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(columnName: String, columnNames: String*): Column = { @@ -3890,6 +4389,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) @@ -3898,6 +4399,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) @@ -3906,6 +4409,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, rightName: String): Column = hypot(Column(leftName), Column(rightName)) @@ -3915,6 +4420,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) @@ -3923,6 +4430,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) @@ -3931,6 +4440,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, r: Column): Column = hypot(lit(l), r) @@ -3939,6 +4450,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) @@ -3948,6 +4461,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(exprs: Column*): Column = Column.fn("least", exprs: _*) @@ -3958,6 +4473,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(columnName: String, columnNames: String*): Column = { @@ -3989,6 +4506,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(columnName: String): Column = log(Column(columnName)) @@ -3997,6 +4516,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) @@ -4005,6 +4526,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, columnName: String): Column = log(base, Column(columnName)) @@ -4023,6 +4546,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(columnName: String): Column = log10(Column(columnName)) @@ -4041,6 +4566,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(columnName: String): Column = log1p(Column(columnName)) @@ -4059,6 +4586,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(columnName: String): Column = log2(Column(columnName)) @@ -4077,6 +4606,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def pi(): Column = Column.fn("pi") @@ -4105,6 +4636,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) @@ -4113,6 +4646,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) @@ -4121,6 +4656,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) @@ -4129,6 +4666,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Double): Column = pow(l, lit(r)) @@ -4137,6 +4676,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) @@ -4145,6 +4686,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, r: Column): Column = pow(lit(l), r) @@ -4153,6 +4696,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) @@ -4193,6 +4738,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(columnName: String): Column = rint(Column(columnName)) @@ -4212,6 +4759,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) @@ -4221,6 +4770,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) @@ -4240,6 +4791,8 @@ object functions { * * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) @@ -4249,6 +4802,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) @@ -4269,6 +4824,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftleft", "3.2.0") def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) @@ -4290,6 +4847,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftright", "3.2.0") def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) @@ -4311,6 +4870,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftrightunsigned", "3.2.0") def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) @@ -4352,6 +4913,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(columnName: String): Column = signum(Column(columnName)) @@ -4371,8 +4934,7 @@ object functions { * @param columnName * angle in radians * @return - * sine of the angle, as if computed by `java.lang.Math.sin` - * + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4384,7 +4946,6 @@ object functions { * @return * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a * column that evaluates to a double. - * * @group math_funcs * @since 1.4.0 */ @@ -4394,8 +4955,7 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4417,8 +4977,7 @@ object functions { * @param columnName * angle in radians * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` - * + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4430,7 +4989,6 @@ object functions { * @return * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a * column that evaluates to a double. - * * @group math_funcs * @since 1.4.0 */ @@ -4440,8 +4998,7 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4450,6 +5007,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(e: Column): Column = degrees(e) @@ -4457,6 +5016,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(columnName: String): Column = degrees(Column(columnName)) @@ -4483,8 +5044,7 @@ object functions { * @param columnName * angle in radians * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` - * + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4493,6 +5053,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(e: Column): Column = radians(e) @@ -4500,6 +5062,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(columnName: String): Column = radians(Column(columnName)) @@ -4526,8 +5090,7 @@ object functions { * @param columnName * angle in degrees * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` - * + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4565,6 +5128,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_catalog(): Column = Column.fn("current_catalog") @@ -4573,6 +5138,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_database(): Column = Column.fn("current_database") @@ -4581,6 +5148,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_schema(): Column = Column.fn("current_schema") @@ -4589,6 +5158,8 @@ object functions { * * @group misc_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a string. */ def current_path(): Column = Column.fn("current_path") @@ -4597,6 +5168,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_user(): Column = Column.fn("current_user") @@ -4659,6 +5232,8 @@ object functions { * * @group hash_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to an integer. */ @scala.annotation.varargs def hash(cols: Column*): Column = Column.fn("hash", cols: _*) @@ -4669,6 +5244,8 @@ object functions { * * @group hash_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) @@ -4688,6 +5265,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) @@ -4706,6 +5285,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def user(): Column = Column.fn("user") @@ -4714,6 +5295,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def session_user(): Column = Column.fn("session_user") @@ -4734,6 +5317,8 @@ object functions { * * @group misc_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(seed: Column): Column = Column.fn("uuid", seed) @@ -4789,6 +5374,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv) @@ -4802,6 +5389,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding) @@ -4815,6 +5404,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_encrypt", input, key, mode) @@ -4828,6 +5419,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column): Column = Column.fn("aes_encrypt", input, key) @@ -4876,6 +5469,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_decrypt", input, key, mode, padding) @@ -4888,6 +5483,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_decrypt", input, key, mode) @@ -4900,6 +5497,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column): Column = Column.fn("aes_decrypt", input, key) @@ -4945,6 +5544,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("try_aes_decrypt", input, key, mode, padding) @@ -4957,6 +5558,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("try_aes_decrypt", input, key, mode) @@ -4969,6 +5572,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column): Column = Column.fn("try_aes_decrypt", input, key) @@ -4988,6 +5593,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_length(): Column = Column.fn("input_file_block_length") @@ -4996,6 +5603,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_start(): Column = Column.fn("input_file_block_start") @@ -5004,6 +5613,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) @@ -5013,6 +5624,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) @@ -5023,6 +5636,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) @@ -5033,6 +5648,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def version(): Column = Column.fn("version") @@ -5052,6 +5669,8 @@ object functions { * * @group generator_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def stack(cols: Column*): Column = Column.fn("stack", cols: _*) @@ -5080,6 +5699,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column, seed: Column): Column = Column.fn("uniform", min, max, seed) @@ -5101,6 +5722,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(): Column = random(lit(SparkClassUtils.random.nextLong)) @@ -5132,6 +5755,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_construct_agg(col: Column): Column = Column.fn("bitmap_construct_agg", col) @@ -5141,6 +5766,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) @@ -5150,6 +5777,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) @@ -5159,6 +5788,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) @@ -5207,6 +5838,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def concat_ws(sep: String, exprs: Column*): Column = @@ -5219,6 +5852,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def decode(value: Column, charset: String): Column = Column.fn("decode", value, lit(charset)) @@ -5301,6 +5936,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def format_string(format: String, arguments: Column*): Column = @@ -5344,6 +5981,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) @@ -5360,6 +5999,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int): Column = Column.fn("instr", str, substring, lit(start)) @@ -5377,6 +6018,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column): Column = Column.fn("instr", str, substring, start) @@ -5396,6 +6039,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = Column.fn("instr", str, substring, lit(start), lit(occurrence)) @@ -5415,6 +6060,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = Column.fn("instr", str, substring, start, occurrence) @@ -5468,6 +6115,8 @@ object functions { * Computes the Levenshtein distance of the two given string columns. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) @@ -5505,6 +6154,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column, pos: Int): Column = Column.fn("locate", lit(substr), str, lit(pos)) @@ -5526,6 +6177,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) @@ -5535,6 +6188,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) @@ -5552,6 +6207,8 @@ object functions { * Trim the specified character string from left end for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) @@ -5559,6 +6216,8 @@ object functions { * Trim the specified character string from left end for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) @@ -5577,6 +6236,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) @@ -5663,6 +6324,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_extract_all", str, regexp, idx) @@ -5684,6 +6347,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) @@ -5693,6 +6358,8 @@ object functions { * * @group string_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = Column.fn("regexp_replace", e, pattern, replacement) @@ -5703,6 +6370,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = Column.fn("regexp_replace", e, pattern, replacement, pos) @@ -5737,6 +6406,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_instr", str, regexp, idx) @@ -5769,6 +6440,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) @@ -5778,6 +6451,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) @@ -5796,6 +6471,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) @@ -5813,6 +6490,8 @@ object functions { * Trim the specified character string from right end for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) @@ -5820,6 +6499,8 @@ object functions { * Trim the specified character string from right end for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) @@ -5860,6 +6541,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) @@ -5880,6 +6563,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String, limit: Int): Column = Column.fn("split", str, lit(pattern), lit(limit)) @@ -5901,6 +6586,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column, limit: Column): Column = Column.fn("split", str, pattern, limit) @@ -5929,6 +6616,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Column, len: Column): Column = Column.fn("substring", str, pos, len) @@ -5964,6 +6653,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def overlay(src: Column, replace: Column, pos: Column): Column = Column.fn("overlay", src, replace, pos) @@ -5983,6 +6674,8 @@ object functions { * default `country`('') is used. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column): Column = Column.fn("sentences", string, language) @@ -5992,6 +6685,8 @@ object functions { * default locale is used. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column): Column = Column.fn("sentences", string) @@ -6022,6 +6717,8 @@ object functions { * Trim the specified character from both ends for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) @@ -6029,6 +6726,8 @@ object functions { * Trim the specified character from both ends for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) @@ -6061,6 +6760,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column): Column = Column.fn("to_binary", e) @@ -6090,6 +6791,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) @@ -6120,6 +6823,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) @@ -6178,6 +6883,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) @@ -6213,6 +6920,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) @@ -6232,6 +6941,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column): Column = Column.fn("try_parse_url", url, partToExtract) @@ -6252,6 +6963,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column): Column = Column.fn("parse_url", url, partToExtract) @@ -6261,6 +6974,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def printf(format: Column, arguments: Column*): Column = @@ -6317,6 +7032,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column): Column = Column.fn("position", substr, str) @@ -6373,6 +7090,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) @@ -6393,6 +7112,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) @@ -6461,6 +7182,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) @@ -6495,6 +7218,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) @@ -6516,6 +7241,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) @@ -6582,6 +7309,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) @@ -6591,6 +7320,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(columnName: String): Column = { hll_sketch_estimate(Column(columnName)) @@ -6602,6 +7333,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column): Column = Column.fn("hll_union", c1, c2) @@ -6612,6 +7345,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(columnName1: String, columnName2: String): Column = { hll_union(Column(columnName1), Column(columnName2)) @@ -6624,6 +7359,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) @@ -6635,6 +7372,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union( columnName1: String, @@ -6649,6 +7388,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(c1: Column, c2: Column): Column = Column.fn("theta_difference", c1, c2) @@ -6659,6 +7400,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(columnName1: String, columnName2: String): Column = { theta_difference(Column(columnName1), Column(columnName2)) @@ -6670,6 +7413,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(c1: Column, c2: Column): Column = Column.fn("theta_intersection", c1, c2) @@ -6680,6 +7425,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(columnName1: String, columnName2: String): Column = { theta_intersection(Column(columnName1), Column(columnName2)) @@ -6691,6 +7438,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) @@ -6700,6 +7449,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(columnName: String): Column = { theta_sketch_estimate(Column(columnName)) @@ -6712,6 +7463,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column): Column = Column.fn("theta_union", c1, c2) @@ -6723,6 +7476,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String): Column = { theta_union(Column(columnName1), Column(columnName2)) @@ -6735,6 +7490,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("theta_union", c1, c2, lit(lgNomEntries)) @@ -6746,6 +7503,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { theta_union(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6758,6 +7517,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = Column.fn("theta_union", c1, c2, lgNomEntries) @@ -6769,6 +7530,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_double", c1, c2) @@ -6780,6 +7543,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(columnName1: String, columnName2: String): Column = tuple_difference_double(Column(columnName1), Column(columnName2)) @@ -6791,6 +7556,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_integer", c1, c2) @@ -6802,6 +7569,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(columnName1: String, columnName2: String): Column = tuple_difference_integer(Column(columnName1), Column(columnName2)) @@ -6814,6 +7583,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_double", c1, c2) @@ -6826,6 +7597,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2)) @@ -6838,6 +7611,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_double", c1, c2, lit(mode)) @@ -6850,6 +7625,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2), mode) @@ -6862,6 +7639,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_double", c1, c2, mode) @@ -6874,6 +7653,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_integer", c1, c2) @@ -6886,6 +7667,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2)) @@ -6898,6 +7681,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) @@ -6910,6 +7695,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) @@ -6922,6 +7709,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_integer", c1, c2, mode) @@ -6932,6 +7721,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(c: Column): Column = Column.fn("tuple_sketch_estimate_double", c) @@ -6942,6 +7733,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(columnName: String): Column = tuple_sketch_estimate_double(Column(columnName)) @@ -6952,6 +7745,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(c: Column): Column = Column.fn("tuple_sketch_estimate_integer", c) @@ -6962,6 +7757,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(columnName: String): Column = tuple_sketch_estimate_integer(Column(columnName)) @@ -6973,6 +7770,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column): Column = Column.fn("tuple_sketch_summary_double", c) @@ -6984,6 +7783,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String): Column = tuple_sketch_summary_double(Column(columnName)) @@ -6994,6 +7795,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_double", c, lit(mode)) @@ -7004,6 +7807,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String, mode: String): Column = tuple_sketch_summary_double(Column(columnName), mode) @@ -7014,6 +7819,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_double", c, mode) @@ -7025,6 +7832,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column): Column = Column.fn("tuple_sketch_summary_integer", c) @@ -7036,6 +7845,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String): Column = tuple_sketch_summary_integer(Column(columnName)) @@ -7046,6 +7857,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_integer", c, lit(mode)) @@ -7056,6 +7869,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String, mode: String): Column = tuple_sketch_summary_integer(Column(columnName), mode) @@ -7066,6 +7881,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_integer", c, mode) @@ -7077,6 +7894,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(c: Column): Column = Column.fn("tuple_sketch_theta_double", c) @@ -7088,6 +7907,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(columnName: String): Column = tuple_sketch_theta_double(Column(columnName)) @@ -7099,6 +7920,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(c: Column): Column = Column.fn("tuple_sketch_theta_integer", c) @@ -7110,6 +7933,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(columnName: String): Column = tuple_sketch_theta_integer(Column(columnName)) @@ -7121,6 +7946,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_double", c1, c2) @@ -7132,6 +7959,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String): Column = tuple_union_double(Column(columnName1), Column(columnName2)) @@ -7144,6 +7973,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) @@ -7156,6 +7987,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) @@ -7168,6 +8001,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7180,6 +8015,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double( columnName1: String, @@ -7196,6 +8033,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) @@ -7207,6 +8046,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_integer", c1, c2) @@ -7218,6 +8059,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String): Column = tuple_union_integer(Column(columnName1), Column(columnName2)) @@ -7230,6 +8073,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) @@ -7242,6 +8087,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) @@ -7254,6 +8101,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7266,6 +8115,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer( columnName1: String, @@ -7282,6 +8133,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) @@ -7293,6 +8146,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_double", c1, c2) @@ -7304,6 +8159,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = tuple_difference_theta_double(Column(columnName1), Column(columnName2)) @@ -7315,6 +8172,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_integer", c1, c2) @@ -7326,6 +8185,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) @@ -7338,6 +8199,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2) @@ -7350,6 +8213,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) @@ -7362,6 +8227,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) @@ -7374,6 +8241,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double( columnName1: String, @@ -7389,6 +8258,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2, mode) @@ -7401,6 +8272,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2) @@ -7413,6 +8286,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) @@ -7425,6 +8300,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) @@ -7437,6 +8314,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer( columnName1: String, @@ -7452,6 +8331,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, mode) @@ -7463,6 +8344,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_double", c1, c2) @@ -7474,6 +8357,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(columnName1: String, columnName2: String): Column = tuple_union_theta_double(Column(columnName1), Column(columnName2)) @@ -7486,6 +8371,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) @@ -7498,6 +8385,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7513,6 +8402,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7525,6 +8416,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7541,6 +8434,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( c1: Column, @@ -7556,6 +8451,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_integer", c1, c2) @@ -7567,6 +8464,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = tuple_union_theta_integer(Column(columnName1), Column(columnName2)) @@ -7579,6 +8478,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) @@ -7591,6 +8492,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7606,6 +8509,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7618,6 +8523,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7634,6 +8541,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( c1: Column, @@ -7845,7 +8754,7 @@ object functions { * @param numMonths * A column of the number of months to add to `startDate`, can be negative to subtract months * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7858,6 +8767,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def curdate(): Column = Column.fn("curdate") @@ -7867,6 +8778,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def current_date(): Column = Column.fn("current_date") @@ -7875,6 +8788,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_timezone(): Column = Column.fn("current_timezone") @@ -7884,6 +8799,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def current_timestamp(): Column = Column.fn("current_timestamp") @@ -7892,6 +8809,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def now(): Column = Column.fn("now") @@ -7902,6 +8821,8 @@ object functions { * * @group datetime_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def localtimestamp(): Column = Column.fn("localtimestamp") @@ -7960,7 +8881,7 @@ object functions { * @param days * A column of the number of days to add to `start`, can be negative to subtract days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -8011,7 +8932,7 @@ object functions { * @param days * A column of the number of days to subtract from `start`, can be negative to add days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -8295,6 +9216,8 @@ object functions { * result is rounded off to 8 digits; it is not rounded otherwise. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a double. */ def months_between(end: Column, start: Column, roundOff: Boolean): Column = Column.fn("months_between", end, start, lit(roundOff)) @@ -8336,7 +9259,7 @@ object functions { * "Fri", "Sat", "Sun" * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.2.0 */ @@ -8400,7 +9323,7 @@ object functions { * A date time pattern that the input will be formatted to * @return * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern + * invalid date time pattern. Returns a column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8429,7 +9352,7 @@ object functions { * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` * format * @return - * A long, or null if the input was a string not of the correct format + * A long, or null if the input was a string not of the correct format. Returns a column that evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8448,7 +9371,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format + * format. Returns a column that evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8462,7 +9385,7 @@ object functions { * A string to be parsed to time. * @return * A time, or raises an error if the input is malformed. Returns a column that evaluates to a - * string. + * time. * * @group datetime_funcs * @since 4.1.0 @@ -8482,8 +9405,7 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or raises an error if the input is malformed. - * + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -8519,7 +9441,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format + * an invalid format. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8531,7 +9453,7 @@ object functions { * @param str * A string to be parsed to time. * @return - * A time, or null if the input is malformed. Returns a column that evaluates to a string. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * * @group datetime_funcs * @since 4.1.0 @@ -8551,8 +9473,7 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or null if the input is malformed. - * + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -8580,6 +9501,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) @@ -8606,7 +9529,7 @@ object functions { * A date time pattern detailing the format of `e` when `e`is a string * @return * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format + * invalid format. Returns a column that evaluates to a date. * @group datetime_funcs * @since 2.2.0 */ @@ -8629,6 +9552,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) @@ -8659,6 +9584,8 @@ object functions { * * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a long. */ def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) @@ -8757,6 +9684,8 @@ object functions { * 03:40:00.0'. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def from_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("from_utc_timestamp", ts, tz) @@ -8791,6 +9720,8 @@ object functions { * '2017-07-14 01:40:00.0'. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) @@ -8838,6 +9769,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window( timeColumn: Column, @@ -8887,6 +9820,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { window(timeColumn, windowDuration, slideDuration, "0 second") @@ -8925,6 +9860,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String): Column = { window(timeColumn, windowDuration, windowDuration, "0 second") @@ -8943,6 +9880,8 @@ object functions { * * @group datetime_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) @@ -9007,6 +9946,8 @@ object functions { * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: Column): Column = Column.fn("session_window", timeColumn, gapDuration) @@ -9046,6 +9987,8 @@ object functions { * * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) @@ -9055,6 +9998,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a long. */ def timestamp_diff(unit: String, start: Column, end: Column): Column = Column.internalFn("timestampdiff", lit(unit), start, end) @@ -9064,6 +10009,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def timestamp_add(unit: String, quantity: Column, ts: Column): Column = Column.internalFn("timestampadd", lit(unit), quantity, ts) @@ -9080,6 +10027,8 @@ object functions { * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column): Column = Column.fn("time_bucket", bucketSize, ts) @@ -9098,6 +10047,8 @@ object functions { * Alignment anchor. Must be the same type as `ts` and must be foldable. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = Column.fn("time_bucket", bucketSize, ts, origin) @@ -9184,6 +10135,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a decimal. */ def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) @@ -9192,6 +10145,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) @@ -9200,6 +10155,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) @@ -9221,6 +10178,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column): Column = Column.fn("to_timestamp_ltz", timestamp) @@ -9243,6 +10202,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def to_timestamp_ntz(timestamp: Column): Column = Column.fn("to_timestamp_ntz", timestamp) @@ -9263,6 +10224,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column): Column = Column.fn("to_unix_timestamp", timeExp) @@ -9358,6 +10321,8 @@ object functions { * * @group array_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Column, length: Column): Column = Column.fn("slice", x, start, length) @@ -9377,6 +10342,8 @@ object functions { * Concatenates the elements of `column` using the `delimiter`. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String): Column = Column.fn("array_join", column, lit(delimiter)) @@ -9390,6 +10357,8 @@ object functions { * * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) @@ -9472,6 +10441,8 @@ object functions { * * @group collection_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column, comparator: (Column, Column) => Column): Column = Column.fn("array_sort", e, createLambda(comparator)) @@ -9620,6 +10591,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: (Column, Column) => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9695,6 +10668,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: (Column, Column) => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9746,6 +10721,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = aggregate(expr, initialValue, merge, c => c) @@ -9797,6 +10774,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = reduce(expr, initialValue, merge, c => c) @@ -9838,6 +10817,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_keys(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_keys", expr, createLambda(f)) @@ -9856,6 +10837,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_values(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_values", expr, createLambda(f)) @@ -9873,6 +10856,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_filter(expr: Column, f: (Column, Column) => Column): Column = Column.fn("map_filter", expr, createLambda(f)) @@ -9892,6 +10877,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = Column.fn("map_zip_with", left, right, createLambda(f)) @@ -9946,6 +10933,8 @@ object functions { * * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline(e: Column): Column = Column.fn("inline", e) @@ -9955,6 +10944,8 @@ object functions { * * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline_outer(e: Column): Column = Column.fn("inline_outer", e) @@ -9975,6 +10966,8 @@ object functions { * * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def json_tuple(json: Column, fields: String*): Column = { @@ -9999,6 +10992,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -10022,6 +11017,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { @@ -10045,6 +11042,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -10068,6 +11067,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { @@ -10085,6 +11086,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: StructType): Column = from_json(e, schema, Map.empty[String, String]) @@ -10101,6 +11104,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: DataType): Column = from_json(e, schema, Map.empty[String, String]) @@ -10123,6 +11128,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -10147,6 +11154,8 @@ object functions { * * @group json_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: Map[String, String]): Column = { @@ -10165,6 +11174,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: Column): Column = { from_json(e, schema, Map.empty[String, String].asJava) @@ -10188,6 +11199,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { @@ -10249,6 +11262,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) @@ -10260,6 +11275,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) @@ -10277,6 +11294,8 @@ object functions { * additional JSONPath arguments, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: Column, paths: Column*): Column = @@ -10296,6 +11315,8 @@ object functions { * additional JSONPath strings, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: String, paths: String*): Column = @@ -10316,6 +11337,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: Column, value: Column): Column = Column.fn("variant_insert", v, path, value) @@ -10335,6 +11358,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: String, value: Column): Column = Column.fn("variant_insert", v, lit(path), value) @@ -10352,6 +11377,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: String, targetType: String): Column = Column.fn("variant_get", v, lit(path), lit(targetType)) @@ -10370,6 +11397,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("variant_get", v, path, lit(targetType)) @@ -10387,6 +11416,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: String, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -10405,6 +11436,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -10416,6 +11449,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) @@ -10426,6 +11461,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) @@ -10437,6 +11474,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: String): Column = schema_of_json(lit(json)) @@ -10448,6 +11487,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) @@ -10479,6 +11520,8 @@ object functions { * * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def json_array_length(e: Column): Column = Column.fn("json_array_length", e) @@ -10511,6 +11554,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_json(e: Column, options: Map[String, String]): Column = @@ -10533,6 +11578,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_json(e: Column, options: java.util.Map[String, String]): Column = @@ -10580,6 +11627,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column): Column = Column.fn("mask", input, upperChar) @@ -10598,6 +11647,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column, lowerChar: Column): Column = Column.fn("mask", input, upperChar, lowerChar) @@ -10618,6 +11669,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = Column.fn("mask", input, upperChar, lowerChar, digitChar) @@ -10639,6 +11692,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask( input: Column, @@ -10696,6 +11751,8 @@ object functions { * * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) @@ -10739,6 +11796,8 @@ object functions { * order of the rows which may be non-deterministic after a shuffle. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_agg(e: Column): Column = Column.fn("array_agg", e) @@ -10763,6 +11822,8 @@ object functions { * * @group array_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) @@ -10780,6 +11841,8 @@ object functions { * than two levels, only one level of nesting is removed. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def flatten(e: Column): Column = Column.fn("flatten", e) @@ -10800,6 +11863,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) @@ -10820,6 +11885,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) @@ -10827,6 +11894,8 @@ object functions { * Returns true if the map contains the key. * @group map_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def map_contains_key(column: Column, key: Any): Column = Column.fn("map_contains_key", column, lit(key)) @@ -10835,6 +11904,8 @@ object functions { * Returns an unordered array containing the keys of the map. * @group map_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to an array. */ def map_keys(e: Column): Column = Column.fn("map_keys", e) @@ -10842,6 +11913,8 @@ object functions { * Returns an unordered array containing the values of the map. * @group map_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to an array. */ def map_values(e: Column): Column = Column.fn("map_values", e) @@ -10849,6 +11922,8 @@ object functions { * Returns an unordered array of all entries in the given map. * @group map_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def map_entries(e: Column): Column = Column.fn("map_entries", e) @@ -10856,6 +11931,8 @@ object functions { * Returns a map created from the given array of entries. * @group map_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a map. */ def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) @@ -10864,6 +11941,8 @@ object functions { * arrays. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) @@ -10872,6 +11951,8 @@ object functions { * Returns the union of all the given maps. * @group map_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) @@ -10893,6 +11974,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -10915,6 +11998,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -10931,6 +12016,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) @@ -10942,6 +12029,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) @@ -10957,8 +12046,7 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-csv.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. - * + * a column with string literal containing schema in DDL format. Returns a column that evaluates to a string. * @group csv_funcs * @since 3.0.0 */ @@ -10981,6 +12069,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_csv(e: Column, options: java.util.Map[String, String]): Column = @@ -10995,6 +12085,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) @@ -11014,6 +12106,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -11035,6 +12129,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -11052,6 +12148,8 @@ object functions { * the schema to use when parsing the XML string * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column): Column = { @@ -11074,6 +12172,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -11090,6 +12190,8 @@ object functions { * * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ def from_xml(e: Column, schema: StructType): Column = from_xml(e, schema, Map.empty[String, String].asJava) @@ -11105,6 +12207,8 @@ object functions { * a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) @@ -11115,6 +12219,8 @@ object functions { * a foldable string column containing a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) @@ -11131,7 +12237,7 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-xml.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that evaluates to a string. * @group xml_funcs * @since 4.0.0 */ @@ -11154,6 +12260,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_xml(e: Column, options: java.util.Map[String, String]): Column = @@ -11167,6 +12275,8 @@ object functions { * a column containing a struct. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) @@ -11336,6 +12446,8 @@ object functions { * a timestamp without time zone. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def convert_timezone(targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", targetTz, sourceTs) @@ -11356,6 +12468,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column): Column = Column.fn("make_dt_interval", days, hours, mins) @@ -11365,6 +12479,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column): Column = Column.fn("make_dt_interval", days, hours) @@ -11374,6 +12490,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column): Column = Column.fn("make_dt_interval", days) @@ -11383,6 +12501,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(): Column = Column.fn("make_dt_interval") @@ -11430,6 +12550,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -11445,6 +12567,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11461,6 +12585,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -11475,6 +12601,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11490,6 +12618,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("try_make_interval", years, months, weeks, days) @@ -11499,6 +12629,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("make_interval", years, months, weeks, days) @@ -11509,6 +12641,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("try_make_interval", years, months, weeks) @@ -11518,6 +12652,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("make_interval", years, months, weeks) @@ -11528,6 +12664,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column): Column = Column.fn("try_make_interval", years, months) @@ -11537,6 +12675,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column): Column = Column.fn("make_interval", years, months) @@ -11547,6 +12687,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column): Column = Column.fn("try_make_interval", years) @@ -11556,6 +12698,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column): Column = Column.fn("make_interval", years) @@ -11565,6 +12709,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(): Column = Column.fn("make_interval") @@ -11598,6 +12744,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11613,6 +12761,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("make_timestamp", date, time, timezone) @@ -11622,6 +12772,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column): Column = Column.fn("make_timestamp", date, time) @@ -11653,6 +12805,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11668,6 +12822,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("try_make_timestamp", date, time, timezone) @@ -11677,6 +12833,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column): Column = Column.fn("try_make_timestamp", date, time) @@ -11708,6 +12866,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11743,6 +12903,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11777,6 +12939,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("make_timestamp_ntz", date, time) @@ -11804,6 +12968,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp_ntz. */ def try_make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("try_make_timestamp_ntz", date, time) @@ -11824,6 +12990,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) @@ -11832,6 +13000,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(): Column = Column.fn("make_ym_interval") @@ -11989,6 +13159,8 @@ object functions { * * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column): Column = Column.fn("st_asbinary", geo) @@ -11998,6 +13170,8 @@ object functions { * * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: Column): Column = Column.fn("st_asbinary", geo, endianness) @@ -12007,6 +13181,8 @@ object functions { * * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: String): Column = Column.fn("st_asbinary", geo, lit(endianness)) @@ -12070,6 +13246,8 @@ object functions { * * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to an integer. */ def st_srid(geo: Column): Column = Column.fn("st_srid", geo) @@ -12657,6 +13835,8 @@ object functions { * Returns the cosine similarity between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_cosine_similarity(left: Column, right: Column): Column = Column.fn("vector_cosine_similarity", left, right) @@ -12665,6 +13845,8 @@ object functions { * Returns the inner product (dot product) between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_inner_product(left: Column, right: Column): Column = Column.fn("vector_inner_product", left, right) @@ -12673,6 +13855,8 @@ object functions { * Returns the Euclidean (L2) distance between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_l2_distance(left: Column, right: Column): Column = Column.fn("vector_l2_distance", left, right) @@ -12681,6 +13865,8 @@ object functions { * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column, degree: Column): Column = Column.fn("vector_norm", vector, degree) @@ -12689,6 +13875,8 @@ object functions { * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column): Column = Column.fn("vector_norm", vector) @@ -12697,6 +13885,8 @@ object functions { * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column, degree: Column): Column = Column.fn("vector_normalize", vector, degree) @@ -12705,6 +13895,8 @@ object functions { * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column): Column = Column.fn("vector_normalize", vector) @@ -12713,6 +13905,8 @@ object functions { * Aggregate function: returns the element-wise mean of float vectors in a group. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_avg(col: Column): Column = Column.fn("vector_avg", col) @@ -12720,6 +13914,8 @@ object functions { * Aggregate function: returns the element-wise sum of float vectors in a group. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_sum(col: Column): Column = Column.fn("vector_sum", col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 964d6a01ae1c0..7fcc150cf05c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -87,9 +87,9 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of `expr1` and `expr2`.", arguments = """ Arguments: - * expr1 - The expr1 argument. + * expr1 - The first operand of the bitwise OR. An expression that evaluates to an integral. - * expr2 - The expr2 argument. + * expr2 - The second operand of the bitwise OR. An expression that evaluates to an integral. """, examples = """ From 845265d11b5179766717a18e1bc4ab948e539bc0 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 14:41:35 +0900 Subject: [PATCH 7/9] Make accepted-input and return type docs consistent and analyzer-accurate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audited every accepted-input and return type phrase across the three doc surfaces (functions.scala Scaladoc, builtin.py docstrings, SQL ExpressionDescription blocks) against each expression's inputTypes / dataType. - Fold TIMESTAMP_NTZ into "timestamp" in all type phrases (e.g. "string, date, timestamp, or timestamp_ntz" -> "string, date, or timestamp"), keeping the distinction only in function names / SQL literals / code, not user-facing type descriptions. - Numeric umbrella: where numeric subtypes cast in, say "a numeric" (or "a numeric or interval") rather than enumerating double/decimal/long — e.g. divide operands, ceil/floor, and the statistical aggregates (corr, covar_*, stddev_*, var_*, skewness, kurtosis, regr_*) which accept any numeric via implicit cast to double. Kept genuinely-narrow sets ("an integral" for kll_sketch_agg_bigint, "an integral, decimal, or interval" for div). - Fix incorrect accepted types: to_char/to_varchar accept numeric, date, timestamp, time, or binary (date was missing); divide dividend is "numeric or interval", divisor "numeric"; array element-args (array_contains/append/prepend/position/remove/insert) take the array's element type, not an integer; element_at index is an integer for an array or the key type for a map; nullifzero takes a numeric. - Fix incorrect return types: unix_nanos returns a decimal (DECIMAL(21,0)), not a long; hll_union_agg usage said "estimated number of unique values" but returns the merged sketch's binary representation. - element_at/try_element_at/explode/explode_outer @return now document the map case (value type / key and value columns), not only the array case. Verified against the ExpressionsSchemaSuite golden file and Catalyst source via an audit pass with adversarial verification; no run-on/orphan @return regressions. Co-authored-by: Isaac --- python/pyspark/sql/functions/builtin.py | 118 +++++++++--------- .../org/apache/spark/sql/functions.scala | 34 ++--- .../sql/catalyst/expressions/TryEval.scala | 4 +- .../aggregate/CentralMomentAgg.scala | 12 +- .../catalyst/expressions/aggregate/Corr.scala | 4 +- .../expressions/aggregate/Covariance.scala | 8 +- .../aggregate/datasketchesAggregates.scala | 2 +- .../expressions/aggregate/kllAggregates.scala | 2 +- .../aggregate/linearRegression.scala | 24 ++-- .../sql/catalyst/expressions/arithmetic.scala | 8 +- .../expressions/collectionOperations.scala | 16 +-- .../expressions/datetimeExpressions.scala | 18 +-- .../expressions/mathExpressions.scala | 4 +- .../expressions/nullExpressions.scala | 2 +- .../expressions/numberFormatExpressions.scala | 2 +- 15 files changed, 130 insertions(+), 128 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index a1d9d0f786b3c..ee41aec64e130 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -697,10 +697,10 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: ---------- left : :class:`~pyspark.sql.Column` or column name dividend. - A column that evaluates to a double, decimal, or interval. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name divisor. - A column that evaluates to a double or decimal. + A column that evaluates to a numeric. Examples -------- @@ -2696,7 +2696,7 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. - A column that evaluates to a double, decimal, or long. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. A column that evaluates to an integer. Must be a constant. @@ -2757,7 +2757,7 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. - A column that evaluates to a double, decimal, or long. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int An optional parameter to control the rounding behavior. A column that evaluates to an integer. Must be a constant. @@ -3162,7 +3162,7 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the floor on. - A column that evaluates to a double, decimal, or long. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. A column that evaluates to an integer. Must be a constant. @@ -4370,7 +4370,7 @@ def stddev(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. See Also -------- @@ -4410,7 +4410,7 @@ def std(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4454,7 +4454,7 @@ def stddev_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4496,7 +4496,7 @@ def stddev_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4537,7 +4537,7 @@ def variance(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4582,7 +4582,7 @@ def var_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4623,7 +4623,7 @@ def var_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -4939,10 +4939,10 @@ def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -5031,10 +5031,10 @@ def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -5123,10 +5123,10 @@ def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -5215,10 +5215,10 @@ def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -5307,10 +5307,10 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. See Also -------- @@ -5399,10 +5399,10 @@ def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -5932,7 +5932,7 @@ def skewness(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. See Also -------- @@ -5974,7 +5974,7 @@ def kurtosis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -7013,10 +7013,10 @@ def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate correlation. - A column that evaluates to a double. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate correlation. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -7053,10 +7053,10 @@ def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. - A column that evaluates to a double. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -7097,10 +7097,10 @@ def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. - A column that evaluates to a double. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. - A column that evaluates to a double. + A column that evaluates to a numeric. Returns ------- @@ -11832,7 +11832,7 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. - A column that evaluates to a string, date, timestamp, or timestamp_ntz. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. A column that evaluates to a string. @@ -11892,7 +11892,7 @@ def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. - A column that evaluates to a string, date, timestamp, or timestamp_ntz. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. A column that evaluates to a string. @@ -12276,7 +12276,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. - A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -12421,7 +12421,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. - A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -13090,7 +13090,7 @@ def unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name, optional timestamps of string values. - A column that evaluates to a string, date, timestamp, or timestamp_ntz. + A column that evaluates to a string, date, or timestamp. format : literal string, optional alternative format to use for converting (default: yyyy-MM-dd HH:mm:ss). A column that evaluates to a string. @@ -14035,7 +14035,7 @@ def to_unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. - A column that evaluates to a string, date, timestamp, or timestamp_ntz. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. A column that evaluates to a string. @@ -14110,7 +14110,7 @@ def to_timestamp_ltz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. - A column that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + A column that evaluates to a string, date, timestamp, or numeric. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. @@ -14182,7 +14182,7 @@ def to_timestamp_ntz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. - A column that evaluates to a string, date, timestamp, or timestamp_ntz. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. A column that evaluates to a string. @@ -17814,7 +17814,7 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The value to convert to a string. - A column that evaluates to a decimal, timestamp, time, or binary. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. A column that evaluates to a string. Must be a constant. @@ -17862,7 +17862,7 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The value to convert to a string. - A column that evaluates to a decimal, timestamp, time, or binary. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. A column that evaluates to a string. Must be a constant. @@ -19701,7 +19701,7 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column: A column that evaluates to an array. value : The value or column to check for in the array. - A column that evaluates to an integer. + A column of the same type as the array elements. Returns ------- @@ -20144,7 +20144,7 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: A column that evaluates to an array. value : Any value or a :class:`~pyspark.sql.Column` expression to look for. - A column that evaluates to an integer. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `value` now also accepts a Column type. @@ -20259,7 +20259,7 @@ def element_at(col: "ColumnOrName", extraction: Any) -> Column: ------- :class:`~pyspark.sql.Column` value at given position. - Returns a column of the element type of the input array. + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -20357,7 +20357,7 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - Returns a column of the element type of the input array. + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -20554,7 +20554,7 @@ def array_prepend(col: "ColumnOrName", value: Any) -> Column: A column that evaluates to an array. value : a literal value, or a :class:`~pyspark.sql.Column` expression. - A column that evaluates to an integer. + A column of the same type as the array elements. Returns ------- @@ -20647,7 +20647,7 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: A column that evaluates to an array. element : element or a :class:`~pyspark.sql.Column` expression to be removed from the array. - A column that evaluates to an integer. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `element` now also accepts a Column type. @@ -20839,10 +20839,10 @@ def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: An pos : :class:`~pyspark.sql.Column` or str or int name of Numeric type column indicating position of insertion (starting at index 1, negative position is a start from the back of the array). - A column that evaluates to an integer. + A column of the same type as the array elements. value : a literal value, or a :class:`~pyspark.sql.Column` expression. - A column that evaluates to an integer. + A column of the same type as the array elements. Returns ------- @@ -21313,7 +21313,7 @@ def array_append(col: "ColumnOrName", value: Any) -> Column: A column that evaluates to an array. value : A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. - A column that evaluates to an integer. + A column of the same type as the array elements. Returns ------- @@ -21414,7 +21414,7 @@ def explode(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` One row per array item or map key value. - Returns a column of the element type of the input array. + Returns a column of the element type of the input array, or the key and value columns of the input map. See Also -------- @@ -21729,7 +21729,7 @@ def explode_outer(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` one row per array item or map key value. - Returns a column of the element type of the input array. + Returns a column of the element type of the input array, or the key and value columns of the input map. See Also -------- @@ -25833,13 +25833,13 @@ def convert_timezone( A column that evaluates to a string. sourceTs : :class:`~pyspark.sql.Column` or column name A timestamp without time zone. - A column that evaluates to a timestamp_ntz or timestamp. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp for converted time zone. - Returns a column that evaluates to a timestamp_ntz. + Returns a column that evaluates to a timestamp. See Also -------- @@ -27316,7 +27316,7 @@ def make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time. - Returns a column that evaluates to a timestamp_ntz. + Returns a column that evaluates to a timestamp. See Also -------- @@ -27478,7 +27478,7 @@ def try_make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time, or NULL in case of an error. - Returns a column that evaluates to a timestamp_ntz. + Returns a column that evaluates to a timestamp. See Also -------- @@ -28726,7 +28726,7 @@ def kll_sketch_agg_bigint( ---------- col : :class:`~pyspark.sql.Column` or column name The column containing bigint values to aggregate. - A column that evaluates to a byte, integer, long, or short. + A column that evaluates to an integral. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -30700,7 +30700,7 @@ def nullifzero(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. + A column that evaluates to a numeric. Examples -------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 6d30d6c1751ca..9ccb0a98b2c27 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -8822,7 +8822,7 @@ object functions { * @group datetime_funcs * @since 3.3.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def localtimestamp(): Column = Column.fn("localtimestamp") @@ -9419,7 +9419,7 @@ object functions { * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a string, date, timestamp, timestamp_ntz, or numeric. + * a string, date, timestamp, or numeric. * @return * A timestamp, or null if the input was a string that could not be cast to a timestamp. * Returns a column that evaluates to a timestamp. @@ -9585,7 +9585,7 @@ object functions { * @group datetime_funcs * @since 4.3.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a decimal. */ def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) @@ -10191,7 +10191,7 @@ object functions { * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ntz", timestamp, format) @@ -10203,7 +10203,7 @@ object functions { * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column): Column = Column.fn("to_timestamp_ntz", timestamp) @@ -10386,7 +10386,7 @@ object functions { * @group collection_funcs * @since 2.4.0 * @return - * Returns a column of the element type of the input array. + * Returns a column of the element type of the input array, or the value type of the input map. */ def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) @@ -10401,7 +10401,7 @@ object functions { * @group collection_funcs * @since 3.5.0 * @return - * Returns a column of the element type of the input array. + * Returns a column of the element type of the input array, or the value type of the input map. */ def try_element_at(column: Column, value: Column): Column = Column.fn("try_element_at", column, value) @@ -10891,7 +10891,8 @@ object functions { * @group generator_funcs * @since 1.3.0 * @return - * Returns a column of the element type of the input array. + * Returns a column of the element type of the input array, or the key and value columns of the + * input map. */ def explode(e: Column): Column = Column.fn("explode", e) @@ -10903,7 +10904,8 @@ object functions { * @group generator_funcs * @since 2.2.0 * @return - * Returns a column of the element type of the input array. + * Returns a column of the element type of the input array, or the key and value columns of the + * input map. */ def explode_outer(e: Column): Column = Column.fn("explode_outer", e) @@ -12428,11 +12430,11 @@ object functions { * the time zone to which the input timestamp should be converted. A column that evaluates to * a string. * @param sourceTs - * a timestamp without time zone. A column that evaluates to a timestamp_ntz or timestamp. + * a timestamp without time zone. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) @@ -12447,7 +12449,7 @@ object functions { * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def convert_timezone(targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", targetTz, sourceTs) @@ -12923,7 +12925,7 @@ object functions { * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz( years: Column, @@ -12940,7 +12942,7 @@ object functions { * @group datetime_funcs * @since 4.1.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("make_timestamp_ntz", date, time) @@ -12952,7 +12954,7 @@ object functions { * @group datetime_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz( years: Column, @@ -12969,7 +12971,7 @@ object functions { * @group datetime_funcs * @since 4.1.0 * @return - * Returns a column that evaluates to a timestamp_ntz. + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("try_make_timestamp_ntz", date, time) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala index 2732ab1ed43f2..289b102c1f6f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala @@ -112,9 +112,9 @@ case class TryAdd(left: Expression, right: Expression, replacement: Expression) arguments = """ Arguments: * dividend - The expression to be divided. - An expression that evaluates to a double, decimal, or interval. + An expression that evaluates to a numeric or interval. * divisor - The expression to divide by. - An expression that evaluates to a double or decimal. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 651001fbf2214..eb4f2c1191523 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -150,7 +150,7 @@ abstract class CentralMomentAgg(child: Expression, nullOnDivideByZero: Boolean) arguments = """ Arguments: * expr - The expression to compute the population standard deviation of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -186,7 +186,7 @@ case class StddevPop( arguments = """ Arguments: * expr - The expression to compute the sample standard deviation of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -223,7 +223,7 @@ case class StddevSamp( arguments = """ Arguments: * expr - The expression to compute the population variance of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -257,7 +257,7 @@ case class VariancePop( arguments = """ Arguments: * expr - The expression to compute the sample variance of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -304,7 +304,7 @@ case class RegrReplacement(child: Expression) arguments = """ Arguments: * expr - The expression to compute the skewness of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -340,7 +340,7 @@ case class Skewness( arguments = """ Arguments: * expr - The expression to compute the kurtosis of. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index 2635eadd7c5e7..72f682c00b4ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -108,9 +108,9 @@ abstract class PearsonCorrelation(x: Expression, y: Expression, nullOnDivideByZe arguments = """ Arguments: * expr1 - The first variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * expr2 - The second variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index bfcf0163f5331..7d047aca06068 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -93,9 +93,9 @@ abstract class Covariance(val left: Expression, val right: Expression, nullOnDiv arguments = """ Arguments: * expr1 - The first variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * expr2 - The second variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -129,9 +129,9 @@ case class CovPopulation( arguments = """ Arguments: * expr1 - The first variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * expr2 - The second variable of the number pairs. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala index 9a7cab22f0699..952b331b7227d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala @@ -240,7 +240,7 @@ object HllSketchAgg { // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(expr, allowDifferentLgConfigK) - Returns the estimated number of unique values. + _FUNC_(expr, allowDifferentLgConfigK) - Returns the merged HllSketch's updatable binary representation. `allowDifferentLgConfigK` (optional) Allow sketches with different lgConfigK values to be unioned (defaults to false).""", examples = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala index 5de6cbf8a22c6..d9a517be86d06 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala @@ -57,7 +57,7 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, ByteType, DataT arguments = """ Arguments: * expr - The expression to aggregate into the KLL sketch. - An expression that evaluates to a byte, integer, long, or short. + An expression that evaluates to an integral. * k - The parameter controlling the size and accuracy of the sketch. """, examples = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala index 924dd9d8e3e4b..b397b7fdb0d2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala @@ -151,9 +151,9 @@ case class RegrAvgY( arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -191,9 +191,9 @@ case class RegrR2(y: Expression, x: Expression) extends PearsonCorrelation(y, x, arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -233,9 +233,9 @@ case class RegrSXX( arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -269,9 +269,9 @@ case class RegrSXY(y: Expression, x: Expression) extends Covariance(y, x, true) arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -311,9 +311,9 @@ case class RegrSYY( arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -381,9 +381,9 @@ case class RegrSlope(left: Expression, right: Expression) extends DeclarativeAgg arguments = """ Arguments: * y - The dependent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. * x - The independent variable. - An expression that evaluates to a double. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 65685c898ab9d..03bb84c2623ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -843,9 +843,9 @@ trait DivModLike extends BinaryArithmetic { arguments = """ Arguments: * expr1 - The dividend. - An expression that evaluates to a double, decimal, or interval. + An expression that evaluates to a numeric or interval. * expr2 - The divisor. - An expression that evaluates to a double or decimal. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -932,9 +932,9 @@ object Divide { arguments = """ Arguments: * expr1 - The dividend to be divided. - An expression that evaluates to a long, decimal, or interval. + An expression that evaluates to an integral, decimal, or interval. * expr2 - The divisor to divide by. - An expression that evaluates to a long, decimal, or interval. + An expression that evaluates to an integral, decimal, or interval. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 3f8de697b49ac..195113695f1ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -1489,7 +1489,7 @@ case class Reverse(child: Expression) * array - The array to search. An expression that evaluates to an array. * value - The value to check for membership in the array. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: @@ -1776,7 +1776,7 @@ trait ArrayPendBase extends RuntimeReplaceable * array - The array to prepend the element to. An expression that evaluates to an array. * element - The element to add at the beginning of the array. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: @@ -1818,7 +1818,7 @@ case class ArrayPrepend(left: Expression, right: Expression) extends ArrayPendBa * array - The array to append the element to. An expression that evaluates to an array. * element - The element to add at the end of the array. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: @@ -2563,7 +2563,7 @@ case class ArrayMax(child: Expression) * array - The array to search within. An expression that evaluates to an array. * element - The element to find the position of. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: @@ -2726,8 +2726,8 @@ case class Get(left: Expression, right: Expression) Arguments: * array - The array to retrieve the element from. An expression that evaluates to an array or map. - * index - The 1-based index of the element to return. - An expression that evaluates to an integer. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. """, examples = """ Examples: @@ -4136,7 +4136,7 @@ case class ArrayRepeat(left: Expression, right: Expression) * array - The array to remove elements from. An expression that evaluates to an array. * element - The element to remove from the array. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: @@ -5207,7 +5207,7 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL * pos - The 1-based index at which to insert the value. An expression that evaluates to an integer. * val - The value to insert into the array. - An expression that evaluates to an integer. + An expression of the same type as the array elements. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 4c18ba18f4810..f966b04bec263 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1550,7 +1550,7 @@ object DateFormatClass { arguments = """ Arguments: * timeExp - A date/timestamp or string which is returned as a UNIX timestamp. - An expression that evaluates to a string, date, timestamp, or timestamp_ntz. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. @@ -1606,7 +1606,7 @@ case class ToUnixTimestamp( arguments = """ Arguments: * timeExp - A date/timestamp or string. If not provided, this defaults to current time. - An expression that evaluates to a string, date, timestamp, or timestamp_ntz. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. @@ -1693,7 +1693,7 @@ case class GetTimestamp( arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp without time zone. - An expression that evaluates to a string, date, timestamp, or timestamp_ntz. + An expression that evaluates to a string, date, or timestamp. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. An expression that evaluates to a string. @@ -1732,7 +1732,7 @@ object ParseToTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp with local time zone. - An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1771,7 +1771,7 @@ object ParseToTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. - An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -2749,7 +2749,7 @@ case class MonthsBetween( arguments = """ Arguments: * date_str - A string to be parsed to date. - An expression that evaluates to a string, date, timestamp, or timestamp_ntz. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. An expression that evaluates to a string. @@ -2827,7 +2827,7 @@ case class ParseToDate( arguments = """ Arguments: * date_str - A string to be parsed to date. - An expression that evaluates to a string, date, timestamp, or timestamp_ntz. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. An expression that evaluates to a string. @@ -2872,7 +2872,7 @@ object TryToDateExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. - An expression that evaluates to a string, date, timestamp, timestamp_ntz, or numeric. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -4472,7 +4472,7 @@ object SubtractDates { * targetTz - the time zone to which the input timestamp should be converted An expression that evaluates to a string. * sourceTs - a timestamp without time zone - An expression that evaluates to a timestamp_ntz or timestamp. + An expression that evaluates to a timestamp. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 662e3349d3863..5fecba3fc54f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -331,7 +331,7 @@ trait CeilFloorExpressionBuilderBase extends ExpressionBuilder { arguments = """ Arguments: * expr - The expression to round up. - An expression that evaluates to a double, decimal, or long. + An expression that evaluates to a numeric. * scale - The number of decimal places to round to. An expression that evaluates to an integer. Must be a constant. """, @@ -621,7 +621,7 @@ case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLO arguments = """ Arguments: * expr - The expression to round down. - An expression that evaluates to a double, decimal, or long. + An expression that evaluates to a numeric. * scale - The number of decimal places to round to. An expression that evaluates to an integer. Must be a constant. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index d4bced7fd7444..eb8e67eca3f14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -229,7 +229,7 @@ case class NullIf(left: Expression, right: Expression, replacement: Expression) arguments = """ Arguments: * expr - The expression that returns null when equal to zero. - An expression of any orderable type. + An expression that evaluates to a numeric. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala index 6e559c0165bf3..6389a9e0b5660 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala @@ -240,7 +240,7 @@ case class TryToNumber(left: Expression, right: Expression) arguments = """ Arguments: * expr - The expression to convert to a string. - An expression that evaluates to a decimal, timestamp, time, or binary. + An expression that evaluates to a numeric, date, timestamp, time, or binary. * format - The format that specifies how to render the value as a string. An expression that evaluates to a string. Must be a constant. """, From 3b6307ca9f3bbdc19ff589816e86970fd9327b38 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 16:04:27 +0900 Subject: [PATCH 8/9] Fix mis-assigned and inaccurate type phrases found by cross-surface audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A multi-agent verification pass over every multi-argument function's type phrases (all three surfaces, checked against Catalyst inputTypes/dataType) surfaced these: - array_insert: the `pos` arg (1-based integer index) was mislabeled "the same type as the array elements" on the PySpark surface — only `value` is the element type. `pos` is coerced to IntegerType. (review finding) - max_by/min_by (3-arg, with `k`): route through MaxMinByK whose dataType is ArrayType(valueType), so @return is "an array", not "the same type as the input". - listagg/string_agg: ListAgg.dataType = child.dataType (string or binary), so the return is polymorphic — "of the same type as the input", not "a string". - overlay: Overlay.dataType = input.dataType (string or binary) — polymorphic. - shiftleft/shiftright/shiftrightunsigned: dataType = left.dataType (integer or long) — "of the same type as the input", not "an integer". - try_divide: Divide.dataType mirrors the input (double or decimal) — matches the try_add/try_subtract/try_multiply peers' "of the same type as the input". - reduce: `merge`/`finish` are lambda args; removed the stray column type-phrase lines so they match `aggregate` and the other higher-order functions (whose lambda params carry no "A column ..." type line on the PySpark surface). Co-authored-by: Isaac --- python/pyspark/sql/functions/builtin.py | 12 ++++----- .../org/apache/spark/sql/functions.scala | 26 +++++++++---------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index ee41aec64e130..541b8c84433dd 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -8206,7 +8206,7 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column: ------- :class:`~pyspark.sql.Column` shifted value. - Returns a column that evaluates to an integer. + Returns a column of the same type as the input. Examples -------- @@ -8264,7 +8264,7 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column: ------- :class:`~pyspark.sql.Column` shifted values. - Returns a column that evaluates to an integer. + Returns a column of the same type as the input. Examples -------- @@ -8322,7 +8322,7 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: ------- :class:`~pyspark.sql.Column` shifted value. - Returns a column that evaluates to an integer. + Returns a column of the same type as the input. Examples -------- @@ -15833,7 +15833,7 @@ def overlay( ------- :class:`~pyspark.sql.Column` string with replaced values. - Returns a column that evaluates to a string. + Returns a column of the same type as the input. Examples -------- @@ -20839,7 +20839,7 @@ def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: An pos : :class:`~pyspark.sql.Column` or str or int name of Numeric type column indicating position of insertion (starting at index 1, negative position is a start from the back of the array). - A column of the same type as the array elements. + A column that evaluates to an integer. value : a literal value, or a :class:`~pyspark.sql.Column` expression. A column of the same type as the array elements. @@ -25204,11 +25204,9 @@ def reduce( merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression of the same type as ``zero``. - A column that evaluates to an array. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. - A column of any type. Returns ------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 9ccb0a98b2c27..ccd203ef11d30 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -1082,7 +1082,7 @@ object functions { * @group agg_funcs * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) @@ -1102,7 +1102,7 @@ object functions { * @group agg_funcs * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) @@ -1187,7 +1187,7 @@ object functions { * @group agg_funcs * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) @@ -1208,7 +1208,7 @@ object functions { * @group agg_funcs * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) @@ -2541,7 +2541,7 @@ object functions { * @group agg_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ def listagg(e: Column): Column = Column.fn("listagg", e) @@ -2552,7 +2552,7 @@ object functions { * @group agg_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) @@ -2584,7 +2584,7 @@ object functions { * @group agg_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ def string_agg(e: Column): Column = Column.fn("string_agg", e) @@ -2595,7 +2595,7 @@ object functions { * @group agg_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) @@ -3670,7 +3670,7 @@ object functions { * @group math_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) @@ -4837,7 +4837,7 @@ object functions { * @group bitwise_funcs * @since 3.2.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) @@ -4860,7 +4860,7 @@ object functions { * @group bitwise_funcs * @since 3.2.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) @@ -4883,7 +4883,7 @@ object functions { * @group bitwise_funcs * @since 3.2.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ def shiftrightunsigned(e: Column, numBits: Int): Column = Column.fn("shiftrightunsigned", e, lit(numBits)) @@ -6642,7 +6642,7 @@ object functions { * @group string_funcs * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = Column.fn("overlay", src, replace, pos, len) From 81c5c2e315cd78ee26c8864cd02b3ff84821049f Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 16:13:26 +0900 Subject: [PATCH 9/9] Fix scalastyle line-length failures and schema_of_json return type - Wrap 20 Scaladoc @return/@param lines that exceeded 100 characters after the type sentences were appended (the scalastyle failures dongjoon-hyun flagged), preserving the standard 3-space payload indentation. - schema_of_json(json, options): @return wrongly said "evaluates to an integer"; SchemaOfJson returns a string (DDL schema). Corrected to "a string". Co-authored-by: Isaac --- .../org/apache/spark/sql/functions.scala | 62 ++++++++++++------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index ccd203ef11d30..cb87005dcc285 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -2964,7 +2964,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the same type as the input. + * according to the order defined by the window's ORDER BY clause. Returns a column of the same + * type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2993,7 +2994,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the same type as the input. + * according to the order defined by the window's ORDER BY clause. Returns a column of the same + * type as the input. * @group window_funcs * @since 4.3.0 */ @@ -3861,7 +3863,8 @@ object functions { /** * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column that evaluates to a double. + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3896,7 +3899,8 @@ object functions { /** * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column that evaluates to a double. + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3931,7 +3935,8 @@ object functions { /** * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column that evaluates to a double. + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4187,7 +4192,8 @@ object functions { * @param columnName * angle in radians * @return - * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates to a double. + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates + * to a double. * @group math_funcs * @since 1.4.0 */ @@ -4209,7 +4215,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column that evaluates to a double. + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4934,7 +4941,8 @@ object functions { * @param columnName * angle in radians * @return - * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates to a double. + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates to + * a double. * @group math_funcs * @since 1.4.0 */ @@ -4955,7 +4963,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a column that evaluates to a double. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4977,7 +4986,8 @@ object functions { * @param columnName * angle in radians * @return - * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that evaluates to a double. + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4998,7 +5008,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a column that evaluates to a double. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -5044,7 +5055,8 @@ object functions { * @param columnName * angle in radians * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that evaluates to a double. + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -5090,7 +5102,8 @@ object functions { * @param columnName * angle in degrees * @return - * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that evaluates to a double. + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -8754,7 +8767,8 @@ object functions { * @param numMonths * A column of the number of months to add to `startDate`, can be negative to subtract months * @return - * A date, or null if `startDate` was a string that could not be cast to a date. Returns a column that evaluates to a date. + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -8881,7 +8895,8 @@ object functions { * @param days * A column of the number of days to add to `start`, can be negative to subtract days * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column that evaluates to a date. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -8932,7 +8947,8 @@ object functions { * @param days * A column of the number of days to subtract from `start`, can be negative to add days * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column that evaluates to a date. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -9352,7 +9368,8 @@ object functions { * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` * format * @return - * A long, or null if the input was a string not of the correct format. Returns a column that evaluates to a long. + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -9405,7 +9422,8 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or raises an error if the input is malformed. Returns a column that evaluates to a time. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * @group datetime_funcs * @since 4.1.0 */ @@ -11507,7 +11525,7 @@ object functions { * Source Option in the version you use. * @return * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to an integer. + * evaluates to a string. * * @group json_funcs * @since 3.0.0 @@ -12048,7 +12066,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-csv.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. Returns a column that evaluates to a string. + * a column with string literal containing schema in DDL format. Returns a column that evaluates + * to a string. * @group csv_funcs * @since 3.0.0 */ @@ -12239,7 +12258,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-xml.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. Returns a column that evaluates to a string. + * a column with string literal containing schema in DDL format. Returns a column that evaluates + * to a string. * @group xml_funcs * @since 4.0.0 */