diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 1e348cfd42874..541b8c84433dd 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -517,11 +517,13 @@ def sqrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -553,7 +555,9 @@ def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -634,6 +638,7 @@ def try_avg(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -691,9 +696,11 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a numeric. Examples -------- @@ -754,9 +761,11 @@ def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a numeric. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a numeric. Examples -------- @@ -803,9 +812,11 @@ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - multiplicand + multiplicand. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name - multiplier + multiplier. + A column that evaluates to a numeric or interval. Examples -------- @@ -867,7 +878,9 @@ def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -948,6 +961,7 @@ def try_sum(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -1017,11 +1031,13 @@ def abs(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the absolute value on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the absolute value of the input. + Returns a column of the same type as the input. Examples -------- @@ -1083,8 +1099,10 @@ def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column of any type. deterministic : bool, optional if there are multiple equally-frequent results then return the lowest (defaults to false). + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -1358,13 +1376,16 @@ def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values to be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be maximized. This could be the column instance or the column name as string. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the top `k` maximum ordering values, sorted in descending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1469,13 +1490,16 @@ def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values that will be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be minimized. This could be the column instance or the column name as string. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the bottom `k` minimum ordering values, sorted in ascending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1638,6 +1662,7 @@ def sum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1702,6 +1727,7 @@ def avg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1756,6 +1782,7 @@ def mean(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1800,6 +1827,7 @@ def median(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric, interval, or time. Returns ------- @@ -1939,8 +1967,10 @@ def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2094,8 +2124,10 @@ def string_agg( ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2286,11 +2318,13 @@ def acos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2341,11 +2375,13 @@ def acosh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse hyperbolic cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse hyperbolic cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2392,11 +2428,13 @@ def asin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse sine of `col`, as if computed by `java.lang.Math.asin()` + Returns a column that evaluates to a double. Examples -------- @@ -2444,11 +2482,13 @@ def asinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2495,11 +2535,13 @@ def atan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse tangent of `col`, as if computed by `java.lang.Math.atan()` + Returns a column that evaluates to a double. Examples -------- @@ -2546,11 +2588,13 @@ def atanh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2599,11 +2643,13 @@ def cbrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2650,8 +2696,10 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2659,6 +2707,7 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2708,8 +2757,10 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2717,6 +2768,7 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2765,12 +2817,14 @@ def cos(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosine of the angle, as if computed by `java.lang.Math.cos()`. + Returns a column that evaluates to a double. Examples -------- @@ -2817,12 +2871,14 @@ def cosh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()` + Returns a column that evaluates to a double. Examples -------- @@ -2869,11 +2925,13 @@ def cot(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cotangent of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2921,11 +2979,13 @@ def csc(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosecant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2992,11 +3052,13 @@ def exp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential of the given value. + Returns a column that evaluates to a double. Examples -------- @@ -3045,11 +3107,13 @@ def expm1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential less one. + Returns a column that evaluates to a double. Examples -------- @@ -3098,8 +3162,10 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the floor on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -3108,6 +3174,7 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` nearest integer that is less than or equal to given value. + Returns a column that evaluates to a long or decimal. Examples -------- @@ -3153,11 +3220,13 @@ def log10(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logarithm of the given value in Base 10. + Returns a column that evaluates to a double. Examples -------- @@ -3206,11 +3275,13 @@ def log1p(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate natural logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of the "given value plus one". + Returns a column that evaluates to a double. Examples -------- @@ -3245,11 +3316,13 @@ def negative(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate negative value for. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` negative value. + Returns a column of the same type as the input. Examples -------- @@ -3300,11 +3373,13 @@ def positive(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input value column. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` value. + Returns a column of the same type as the input. Examples -------- @@ -3337,11 +3412,13 @@ def rint(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -3376,12 +3453,14 @@ def sec(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Angle in radians + Angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` Secant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -3428,11 +3507,13 @@ def signum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3470,11 +3551,13 @@ def sign(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3512,11 +3595,13 @@ def sin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` sine of the angle, as if computed by `java.lang.Math.sin()` + Returns a column that evaluates to a double. Examples -------- @@ -3564,12 +3649,14 @@ def sinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh()` + Returns a column that evaluates to a double. Examples -------- @@ -3615,12 +3702,14 @@ def tan(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` tangent of the given value, as if computed by `java.lang.Math.tan()` + Returns a column that evaluates to a double. Examples -------- @@ -3667,13 +3756,15 @@ def tanh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic tangent of the given value as if computed by `java.lang.Math.tanh()` + Returns a column that evaluates to a double. Examples -------- @@ -3810,12 +3901,14 @@ def bit_count(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral or boolean. Returns ------- :class:`~pyspark.sql.Column` the number of bits that are set in the argument expr as an unsigned 64-bit integer, or NULL if the argument is NULL. + Returns a column that evaluates to an integer. See Also -------- @@ -3853,13 +3946,16 @@ def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -3912,13 +4008,16 @@ def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -4271,6 +4370,7 @@ def stddev(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. See Also -------- @@ -4310,6 +4410,7 @@ def std(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4353,6 +4454,7 @@ def stddev_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4394,6 +4496,7 @@ def stddev_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4434,6 +4537,7 @@ def variance(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4478,6 +4582,7 @@ def var_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4518,6 +4623,7 @@ def var_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4556,8 +4662,10 @@ def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4646,8 +4754,10 @@ def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4736,8 +4846,10 @@ def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. See Also -------- @@ -4827,8 +4939,10 @@ def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4917,8 +5031,10 @@ def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5007,8 +5123,10 @@ def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5097,8 +5215,10 @@ def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5187,8 +5307,10 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. See Also -------- @@ -5277,8 +5399,10 @@ def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5366,6 +5490,7 @@ def every(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. See Also -------- @@ -5422,6 +5547,7 @@ def bool_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. Returns ------- @@ -5475,6 +5601,7 @@ def some(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5531,6 +5658,7 @@ def bool_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5579,6 +5707,7 @@ def bit_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5652,6 +5781,7 @@ def bit_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5725,6 +5855,7 @@ def bit_xor(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5801,6 +5932,7 @@ def skewness(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. See Also -------- @@ -5842,6 +5974,7 @@ def kurtosis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -6096,12 +6229,14 @@ def degrees(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in degrees, as if computed by `java.lang.Math.toDegrees()` + Returns a column that evaluates to a double. Examples -------- @@ -6135,12 +6270,14 @@ def radians(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in degrees + angle in degrees. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in radians, as if computed by `java.lang.Math.toRadians()` + Returns a column that evaluates to a double. Examples -------- @@ -6174,9 +6311,11 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] Parameters ---------- col1 : :class:`~pyspark.sql.Column`, column name or float - coordinate on y-axis + coordinate on y-axis. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float - coordinate on x-axis + coordinate on x-axis. + A column that evaluates to a double. Returns ------- @@ -6186,6 +6325,7 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] in polar coordinates that corresponds to the point (`x`, `y`) in Cartesian coordinates, as if computed by `java.lang.Math.atan2()` + Returns a column that evaluates to a double. Examples -------- @@ -6214,13 +6354,16 @@ def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] ---------- col1 : :class:`~pyspark.sql.Column`, column name or float a leg. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float b leg. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` length of the hypotenuse. + Returns a column that evaluates to a double. Examples -------- @@ -6249,13 +6392,16 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) ---------- col1 : :class:`~pyspark.sql.Column`, column name or float the base number. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float the exponent number. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the base rased to the power the argument. + Returns a column that evaluates to a double. Examples -------- @@ -6287,14 +6433,17 @@ def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", Parameters ---------- dividend : :class:`~pyspark.sql.Column`, column name or float - the column that contains dividend, or the specified dividend value + the column that contains dividend, or the specified dividend value. + A column that evaluates to a numeric. divisor : :class:`~pyspark.sql.Column`, column name or float - the column that contains divisor, or the specified divisor value + the column that contains divisor, or the specified divisor value. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` positive value of dividend mod divisor. + Returns a column of the same type as the input. Notes ----- @@ -6343,18 +6492,23 @@ def width_bucket( Parameters ---------- v : :class:`~pyspark.sql.Column` or column name - value to compute a bucket number in the histogram + value to compute a bucket number in the histogram. + A column that evaluates to a double or interval. min : :class:`~pyspark.sql.Column` or column name - minimum value of the histogram + minimum value of the histogram. + A column that evaluates to a double or interval. max : :class:`~pyspark.sql.Column` or column name - maximum value of the histogram + maximum value of the histogram. + A column that evaluates to a double or interval. numBucket : :class:`~pyspark.sql.Column`, column name or int - the number of buckets + the number of buckets. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` the bucket number into which the value would fall after being evaluated + Returns a column that evaluates to a long. Examples -------- @@ -6809,6 +6963,7 @@ def coalesce(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` value of the first column that is not null. + Returns a column of the same type as the input. Examples -------- @@ -6858,8 +7013,10 @@ def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate correlation. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate correlation. + A column that evaluates to a numeric. Returns ------- @@ -6896,8 +7053,10 @@ def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- @@ -6938,8 +7097,10 @@ def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- @@ -7088,8 +7249,10 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch first value for. + A column of any type. ignorenulls : bool if first value is null then look for first non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7354,11 +7517,13 @@ def isnan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` True if value is NaN and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7393,6 +7558,7 @@ def isnull(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` True if value is null and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7430,8 +7596,10 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch last value for. + A column of any type. ignorenulls : bool if last value is null then look for non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7533,13 +7701,16 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to check. + A column that evaluates to a double or float. col2 : :class:`~pyspark.sql.Column` or column name second column to return if first is NaN. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` value from first column or second if first is NaN . + Returns a column that evaluates to a double. Examples -------- @@ -7772,11 +7943,13 @@ def rand(seed: Optional[int] = None) -> Column: ---------- seed : int, optional Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7834,11 +8007,13 @@ def randn(seed: Optional[int] = None) -> Column: ---------- seed : int (default: None) Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7890,8 +8065,10 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7900,6 +8077,7 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -7946,8 +8124,10 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7956,6 +8136,7 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -8016,13 +8197,16 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column of the same type as the input. Examples -------- @@ -8071,13 +8255,16 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted values. + Returns a column of the same type as the input. Examples -------- @@ -8126,13 +8313,16 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column of the same type as the input. Examples -------- @@ -8254,6 +8444,7 @@ def struct( ------- :class:`~pyspark.sql.Column` a struct type column of given columns. + Returns a column that evaluates to a struct. See Also -------- @@ -8330,6 +8521,7 @@ def greatest(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` greatest value. + Returns a column of the same type as the input. See Also -------- @@ -8374,6 +8566,7 @@ def least(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` least value. + Returns a column of the same type as the input. See Also -------- @@ -8420,6 +8613,7 @@ def when(condition: Column, value: Any) -> Column: ------- :class:`~pyspark.sql.Column` column representing when expression. + Returns a column of the same type as the input. See Also -------- @@ -8486,14 +8680,17 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non Parameters ---------- arg1 : :class:`~pyspark.sql.Column`, str or float - base number or actual number (in this case base is `e`) + base number or actual number (in this case base is `e`). + A column that evaluates to a double. arg2 : :class:`~pyspark.sql.Column`, str or float, optional number to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8560,11 +8757,13 @@ def ln(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8605,11 +8804,13 @@ def log2(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. Examples -------- @@ -8647,15 +8848,19 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to convert base for. + A column that evaluates to a string. fromBase: int from base number. + A column that evaluates to an integer. toBase: int to base number. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a string. Examples -------- @@ -8691,11 +8896,13 @@ def factorial(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate factorial for. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` factorial of given value. + Returns a column that evaluates to a long. Examples -------- @@ -8990,8 +9197,10 @@ def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9036,8 +9245,10 @@ def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9089,8 +9300,10 @@ def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9142,6 +9355,7 @@ def count_if(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a boolean. Returns ------- @@ -9604,8 +9818,10 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to format. + A column that evaluates to a timestamp or time. format: literal string format to use to represent datetime values. + A column that evaluates to a string. See Also -------- @@ -9620,6 +9836,7 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ------- :class:`~pyspark.sql.Column` string value representing formatted datetime. + Returns a column that evaluates to a string. Examples -------- @@ -9696,11 +9913,13 @@ def year(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` year part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9787,11 +10006,13 @@ def quarter(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` quarter of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9878,11 +10099,13 @@ def month(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` month part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9971,11 +10194,13 @@ def dayofweek(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the week for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10057,6 +10282,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. See Also -------- @@ -10069,6 +10295,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10140,11 +10367,13 @@ def day(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10236,11 +10465,13 @@ def dayofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the year for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10326,11 +10557,13 @@ def hour(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` hour part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10408,6 +10641,7 @@ def minute(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. See Also -------- @@ -10425,6 +10659,7 @@ def minute(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` minutes part of the timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10490,11 +10725,13 @@ def second(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` `seconds` part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10571,11 +10808,13 @@ def weekofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` `week` of the year for given date as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10654,11 +10893,13 @@ def weekday(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + Returns a column that evaluates to an integer. See Also -------- @@ -10738,11 +10979,13 @@ def monthname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of month name for date/timestamp (Jan, Feb, Mar...) + Returns a column that evaluates to a string. See Also -------- @@ -10819,11 +11062,13 @@ def dayname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of day name for date/timestamp (Mon, Tue, Wed...) + Returns a column that evaluates to a string. See Also -------- @@ -10907,6 +11152,7 @@ def extract(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -10962,6 +11208,7 @@ def date_part(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -11017,6 +11264,7 @@ def datepart(field: Column, source: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column that evaluates to a double. See Also -------- @@ -11066,16 +11314,20 @@ def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") Parameters ---------- year : :class:`~pyspark.sql.Column` or column name - The year to build the date + The year to build the date. + A column that evaluates to an integer. month : :class:`~pyspark.sql.Column` or column name - The month to build the date + The month to build the date. + A column that evaluates to an integer. day : :class:`~pyspark.sql.Column` or column name - The day to build the date + The day to build the date. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date built from given parts. + Returns a column that evaluates to a date. See Also -------- @@ -11112,14 +11364,17 @@ def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11172,14 +11427,17 @@ def dateadd(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11235,14 +11493,17 @@ def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days before the given date to calculate. Accepts negative value as well to calculate forward in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date before/after given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11295,13 +11556,16 @@ def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11343,13 +11607,16 @@ def date_diff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11392,11 +11659,13 @@ def date_from_unix_date(days: "ColumnOrName") -> Column: ---------- days : :class:`~pyspark.sql.Column` or column name the target column to work on. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the date from the number of days since 1970-01-01. + Returns a column that evaluates to a date. See Also -------- @@ -11434,14 +11703,17 @@ def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Col ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. months : :class:`~pyspark.sql.Column` or column name or int how many months after the given date to calculate. Accepts negative value as well to calculate backwards. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of months. + Returns a column that evaluates to a date. See Also -------- @@ -11496,15 +11768,19 @@ def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool ---------- date1 : :class:`~pyspark.sql.Column` or column name first date column. + A column that evaluates to a timestamp. date2 : :class:`~pyspark.sql.Column` or column name second date column. + A column that evaluates to a timestamp. roundOff : bool, optional whether to round (to 8 digits) the final value or not (default: True). + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` number of months between two dates. + Returns a column that evaluates to a double. Examples -------- @@ -11556,13 +11832,16 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11613,13 +11892,16 @@ def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11675,11 +11957,13 @@ def unix_date(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the number of days since 1970-01-01. + Returns a column that evaluates to an integer. See Also -------- @@ -11718,11 +12002,13 @@ def unix_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of microseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11762,11 +12048,13 @@ def unix_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of milliseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11857,11 +12145,13 @@ def unix_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of seconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11914,13 +12204,16 @@ def to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Col ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a time. See Also -------- @@ -11983,6 +12276,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -11990,6 +12284,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ------- :class:`~pyspark.sql.Column` timestamp value as :class:`pyspark.sql.types.TimestampType` type. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12056,13 +12351,16 @@ def try_to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a time. See Also -------- @@ -12123,6 +12421,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -12518,15 +12817,18 @@ def trunc(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a date. format : literal string 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - Other options are: 'week', 'quarter' + Other options are: 'week', 'quarter'. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` truncated date. + Returns a column that evaluates to a date. See Also -------- @@ -12573,14 +12875,17 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column: 'month', 'mon', 'mm' to truncate by month, 'day', 'dd' to truncate by day, Other options are: - 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter' + 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'. + A column that evaluates to a string. timestamp : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` truncated timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12625,14 +12930,17 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. dayOfWeek : literal string day of the week, case-insensitive, accepts: - "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column of computed results. + Returns a column that evaluates to a date. Examples -------- @@ -12671,11 +12979,13 @@ def last_day(date: "ColumnOrName") -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` last day of the month. + Returns a column that evaluates to a date. Examples -------- @@ -12716,13 +13026,16 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss" ---------- timestamp : :class:`~pyspark.sql.Column` or column name column of unix time values. + A column that evaluates to a long. format : literal string, optional - format to use to convert to (default: yyyy-MM-dd HH:mm:ss) + format to use to convert to (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` formatted timestamp as string. + Returns a column that evaluates to a string. See Also -------- @@ -12777,13 +13090,16 @@ def unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name, optional timestamps of string values. + A column that evaluates to a string, date, or timestamp. format : literal string, optional alternative format to use for converting (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` unix time as long integer. + Returns a column that evaluates to a long. Examples -------- @@ -12854,7 +13170,8 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12862,6 +13179,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12870,6 +13188,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col ------- :class:`~pyspark.sql.Column` timestamp value represented in given timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12923,7 +13242,8 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12931,6 +13251,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4.0 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12939,6 +13260,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum ------- :class:`~pyspark.sql.Column` timestamp value represented in UTC timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12983,11 +13305,13 @@ def timestamp_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13027,15 +13351,19 @@ def time_diff(unit: "ColumnOrName", start: "ColumnOrName", end: "ColumnOrName") unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or column name A starting time. + A column that evaluates to a time. end : :class:`~pyspark.sql.Column` or column name An ending time. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` The difference between two times, in the specified units. + Returns a column that evaluates to a long. See Also -------- @@ -13070,13 +13398,16 @@ def time_trunc(unit: "ColumnOrName", time: "ColumnOrName") -> Column: unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. time : :class:`~pyspark.sql.Column` or column name A time to truncate. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` A time truncated to the specified unit. + Returns a column that evaluates to a time. See Also -------- @@ -13110,11 +13441,13 @@ def timestamp_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13152,11 +13485,13 @@ def timestamp_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13195,6 +13530,7 @@ def timestamp_nanos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name a column of ``BIGINT`` or ``DECIMAL`` nanosecond values since the UTC epoch. + A column that evaluates to an integral or decimal. Returns ------- @@ -13639,6 +13975,7 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str]) ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a struct. See Also -------- @@ -13698,8 +14035,10 @@ def to_unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. + A column that evaluates to a string. See Also -------- @@ -13771,6 +14110,7 @@ def to_timestamp_ltz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, timestamp, or numeric. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. @@ -13842,8 +14182,10 @@ def to_timestamp_ntz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. + A column that evaluates to a string. See Also -------- @@ -14133,6 +14475,7 @@ def crc32(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- @@ -14140,6 +14483,7 @@ def crc32(col: "ColumnOrName") -> Column: the column for computed results. .. versionadded:: 1.5.0 + Returns a column that evaluates to a long. Examples -------- @@ -14168,11 +14512,13 @@ def md5(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. Examples -------- @@ -14201,11 +14547,13 @@ def sha1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14241,14 +14589,17 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. numBits : int the desired bit length of the result, which must have a value of 224, 256, 384, 512, or 0 (which is equivalent to 256). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14298,6 +14649,7 @@ def hash(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as int column. + Returns a column that evaluates to an integer. See Also -------- @@ -14350,6 +14702,7 @@ def xxhash64(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as long column. + Returns a column that evaluates to a long. See Also -------- @@ -14397,14 +14750,17 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column name or column that represents the input column to test + column name or column that represents the input column to test. + A column that evaluates to a boolean. errMsg : :class:`~pyspark.sql.Column` or literal string, optional - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` `null` if the input column is `true` otherwise throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14468,12 +14824,14 @@ def raise_error(errMsg: Union[Column, str]) -> Column: Parameters ---------- errMsg : :class:`~pyspark.sql.Column` or literal string - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14517,11 +14875,13 @@ def upper(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` upper case values. + Returns a column that evaluates to a string. See Also -------- @@ -14558,11 +14918,13 @@ def lower(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` lower case values. + Returns a column that evaluates to a string. See Also -------- @@ -14599,11 +14961,13 @@ def ascii(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` numeric value. + Returns a column that evaluates to an integer. Examples -------- @@ -14635,11 +14999,13 @@ def base64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` BASE64 encoding of string value. + Returns a column that evaluates to a string. See Also -------- @@ -14675,11 +15041,13 @@ def unbase64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` encoded string value. + Returns a column that evaluates to a binary. See Also -------- @@ -14715,8 +15083,10 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14724,6 +15094,7 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` left trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14790,8 +15161,10 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14799,6 +15172,7 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` right trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14865,8 +15239,10 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14874,6 +15250,7 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` trimmed values from both sides. + Returns a column that evaluates to a string. See Also -------- @@ -14941,6 +15318,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ---------- sep : literal string words separator. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name list of columns to work on. @@ -14948,6 +15326,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` string of concatenated words. + Returns a column that evaluates to a string. See Also -------- @@ -15028,13 +15407,16 @@ def encode(col: "ColumnOrName", charset: str) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. charset : literal string charset to use to encode. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a binary. See Also -------- @@ -15067,11 +15449,13 @@ def is_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` whether the input string is a valid UTF-8 string. + Returns a column that evaluates to a boolean. See Also -------- @@ -15104,11 +15488,13 @@ def make_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the valid UTF-8 version of the given input string. + Returns a column that evaluates to a string. See Also -------- @@ -15140,11 +15526,13 @@ def validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, error otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15176,11 +15564,13 @@ def try_validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, null otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15215,14 +15605,17 @@ def format_number(col: "ColumnOrName", d: int) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - the column name of the numeric value to be formatted + the column name of the numeric value to be formatted. + A column that evaluates to a numeric. d : int - the N decimal places + the N decimal places. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. Examples -------- @@ -15253,7 +15646,8 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: Parameters ---------- format : literal string - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -15261,6 +15655,7 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. See Also -------- @@ -15313,21 +15708,26 @@ def instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. substr : :class:`~pyspark.sql.Column` or literal string substring to look for. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `substr` now accepts column. start : int or :class:`~pyspark.sql.Column`, optional Starting position (1-based, can be negative for backward search). If not specified, defaults to 1. + A column that evaluates to an integer. occurrence : int or :class:`~pyspark.sql.Column`, optional Which occurrence to locate (must be > 0). Defaults to 1. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` location of the substring as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -15416,19 +15816,24 @@ def overlay( Parameters ---------- src : :class:`~pyspark.sql.Column` or column name - the string that will be replaced + the string that will be replaced. + A column that evaluates to a string or binary. replace : :class:`~pyspark.sql.Column` or column name - the substitution string + the substitution string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int - the starting position in src + the starting position in src. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name or int, optional the number of bytes to replace in src - string by 'replace' defaults to -1, which represents the length of the 'replace' string + string by 'replace' defaults to -1, which represents the length of the 'replace' string. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` string with replaced values. + Returns a column of the same type as the input. Examples -------- @@ -15516,16 +15921,20 @@ def sentences( Parameters ---------- string : :class:`~pyspark.sql.Column` or column name - a string to be split + a string to be split. + A column that evaluates to a string. language : :class:`~pyspark.sql.Column` or column name, optional - a language of the locale + a language of the locale. + A column that evaluates to a string. country : :class:`~pyspark.sql.Column` or column name, optional - a country of the locale + a country of the locale. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` arrays of split sentences. + Returns a column that evaluates to an array. See Also -------- @@ -15589,14 +15998,17 @@ def substring( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int starting position in str. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pos` now accepts column and column name. len : :class:`~pyspark.sql.Column` or column name or int length of chars. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `len` now accepts column and column name. @@ -15605,6 +16017,7 @@ def substring( ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -15694,15 +16107,19 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. delim : literal string delimiter of values. + A column that evaluates to a string. count : int number of occurrences. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column that evaluates to a string. See Also -------- @@ -15752,11 +16169,14 @@ def levenshtein( ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. threshold : int, optional if set when the levenshtein distance of the two given strings - less than or equal to a given threshold then return result distance, or -1 + less than or equal to a given threshold then return result distance, or -1. + A column that evaluates to an integer. .. versionadded:: 3.5.0 @@ -15764,6 +16184,7 @@ def levenshtein( ------- :class:`~pyspark.sql.Column` Levenshtein distance as integer value. + Returns a column that evaluates to an integer. Examples -------- @@ -15805,13 +16226,16 @@ def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Colu ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` Jaro-Winkler similarity as a double value. + Returns a column that evaluates to a double. Examples -------- @@ -15840,16 +16264,20 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: Parameters ---------- substr : literal string - a string + a string. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or column name - a Column of :class:`pyspark.sql.types.StringType` + a Column of :class:`pyspark.sql.types.StringType`. + A column that evaluates to a string. pos : int, optional - start position (zero based) + start position (zero based). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` position of the substring. + Returns a column that evaluates to an integer. Notes ----- @@ -15907,14 +16335,17 @@ def lpad( ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15923,6 +16354,7 @@ def lpad( ------- :class:`~pyspark.sql.Column` left padded result. + Returns a column that evaluates to a string. See Also -------- @@ -15977,14 +16409,17 @@ def rpad( ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15993,6 +16428,7 @@ def rpad( ------- :class:`~pyspark.sql.Column` right padded result. + Returns a column that evaluates to a string. See Also -------- @@ -16043,8 +16479,10 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. n : :class:`~pyspark.sql.Column` or column name or int number of times to repeat value. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `n` now accepts column and column name. @@ -16053,6 +16491,7 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ------- :class:`~pyspark.sql.Column` string with repeated values. + Returns a column that evaluates to a string. Examples -------- @@ -16108,10 +16547,12 @@ def split( Parameters ---------- str : :class:`~pyspark.sql.Column` or column name - a string expression to split + a string expression to split. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or literal string a string representing a regular expression. The regex string should be a Java regular expression. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `pattern` now accepts column. Does not accept column name since string type remain @@ -16120,6 +16561,7 @@ def split( limit : :class:`~pyspark.sql.Column` or column name or int an integer which controls the number of times `pattern` is applied. + A column that evaluates to an integer. * ``limit > 0``: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last @@ -16134,6 +16576,7 @@ def split( ------- :class:`~pyspark.sql.Column` array of separated strings. + Returns a column that evaluates to an array. See Also -------- @@ -16207,13 +16650,16 @@ def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16260,13 +16706,16 @@ def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16313,13 +16762,16 @@ def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16368,13 +16820,16 @@ def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = Non ---------- length : :class:`~pyspark.sql.Column` or int Number of characters in the string to generate. + A column that evaluates to an integer. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random string with the specified length. + Returns a column that evaluates to a string. See Also -------- @@ -16421,13 +16876,16 @@ def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the number of times that a Java regex pattern is matched in the string. + Returns a column that evaluates to an integer. Examples -------- @@ -16478,15 +16936,19 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. pattern : str regex pattern to apply. + A column that evaluates to a string. idx : int matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` matched value specified by `idx` group id. + Returns a column that evaluates to a string. See Also -------- @@ -16539,15 +17001,19 @@ def regexp_extract_all( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` all strings in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an array. See Also -------- @@ -16617,18 +17083,23 @@ def regexp_replace( Parameters ---------- string : :class:`~pyspark.sql.Column` or str - column name or column containing the string value + column name or column containing the string value. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str - column object or str containing the regexp pattern + column object or str containing the regexp pattern. + A column that evaluates to a string. replacement : :class:`~pyspark.sql.Column` or str - column object or str containing the replacement + column object or str containing the replacement. + A column that evaluates to a string. position : :class:`~pyspark.sql.Column` or int, optional position to start replacement. The first position is 1. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` string with all substrings replaced. + Returns a column that evaluates to a string. Examples -------- @@ -16697,13 +17168,16 @@ def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the first substring that matches a Java regex within the string `str`. + Returns a column that evaluates to a string. Examples -------- @@ -16766,16 +17240,20 @@ def regexp_instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the position of the first substring in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an integer. Examples -------- @@ -16841,11 +17319,13 @@ def initcap(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string with all first letters are uppercase in each word. + Returns a column that evaluates to a string. Examples -------- @@ -16875,11 +17355,13 @@ def soundex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` SoundEx encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -16909,11 +17391,13 @@ def bin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` binary representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16952,6 +17436,7 @@ def hex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long, binary, or string. See Also -------- @@ -16961,6 +17446,7 @@ def hex(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hexadecimal representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16990,6 +17476,7 @@ def unhex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. See Also -------- @@ -16999,6 +17486,7 @@ def unhex(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` string representation of given hexadecimal value. + Returns a column that evaluates to a binary. Examples -------- @@ -17032,15 +17520,19 @@ def uniform( ---------- min : :class:`~pyspark.sql.Column`, int, or float Minimum value in the range. + A column that evaluates to a numeric. Must be a constant. max : :class:`~pyspark.sql.Column`, int, or float Maximum value in the range. + A column that evaluates to a numeric. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random number within the specified range. + Returns a column of the same type as the input. See Also -------- @@ -17093,11 +17585,13 @@ def length(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` length of the value. + Returns a column that evaluates to an integer. See Also -------- @@ -17130,12 +17624,14 @@ def octet_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Byte length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17165,12 +17661,14 @@ def bit_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Bit length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17202,17 +17700,21 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: Parameters ---------- srcCol : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string. matching : str matching characters. + A column that evaluates to a string. replace : str characters for replacement. If this is shorter than `matching` string then those chars that don't have replacement will be dropped. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` replaced value. + Returns a column that evaluates to a string. Examples -------- @@ -17247,8 +17749,10 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17309,9 +17813,11 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17355,9 +17861,11 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17397,8 +17905,10 @@ def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17426,11 +17936,14 @@ def replace( ---------- src : :class:`~pyspark.sql.Column` or str A column of string to be replaced. + A column that evaluates to a string. search : :class:`~pyspark.sql.Column` or str A column of string, If `search` is not found in `str`, `str` is returned unchanged. + A column that evaluates to a string. replace : :class:`~pyspark.sql.Column` or str, optional A column of string, If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + A column that evaluates to a string. Examples -------- @@ -17462,10 +17975,13 @@ def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnO ---------- src : :class:`~pyspark.sql.Column` or column name A column of string to be split. + A column that evaluates to a string. delimiter : :class:`~pyspark.sql.Column` or column name A column of string, the delimiter used for split. + A column that evaluates to a string. partNum : :class:`~pyspark.sql.Column` or column name A column of string, requested part of the split (1-based). + A column that evaluates to an integer. See Also -------- @@ -17507,15 +18023,19 @@ def substr( ---------- str : :class:`~pyspark.sql.Column` or column name A column of string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name A column of string, the substring of `str` that starts at `pos`. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name, optional A column of string, the substring of `str` is of length `len`. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -17563,15 +18083,19 @@ def try_parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17679,15 +18203,19 @@ def parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17777,7 +18305,8 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: Parameters ---------- format : :class:`~pyspark.sql.Column` or str - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -17815,11 +18344,13 @@ def url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17893,11 +18424,13 @@ def try_url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17938,11 +18471,13 @@ def url_encode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a string to be URL-encoded. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the URL-encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -18018,10 +18553,13 @@ def position( ---------- substr : :class:`~pyspark.sql.Column` or str A column of string, substring. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or str A column of string. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or str, optional A column of string, start position. + A column that evaluates to an integer. Examples -------- @@ -18062,9 +18600,11 @@ def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. suffix : :class:`~pyspark.sql.Column` or str - A column of string, the suffix. + The suffix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18100,9 +18640,11 @@ def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. prefix : :class:`~pyspark.sql.Column` or str - A column of string, the prefix. + The prefix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18138,6 +18680,7 @@ def char(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a long. Examples -------- @@ -18163,8 +18706,10 @@ def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. Examples -------- @@ -18195,6 +18740,7 @@ def char_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18227,6 +18773,7 @@ def character_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18258,6 +18805,7 @@ def chr(n: "ColumnOrName") -> Column: ---------- n : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a long. Examples -------- @@ -18293,8 +18841,10 @@ def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18351,8 +18901,10 @@ def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18402,9 +18954,11 @@ def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or str - The input column or strings to check, may be NULL. + The input to check; may be NULL. + A column that evaluates to a string or binary. right : :class:`~pyspark.sql.Column` or str - The input column or strings to find, may be NULL. + The value to find; may be NULL. + A column that evaluates to a string or binary. Examples -------- @@ -18468,8 +19022,10 @@ def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str The given string to be found. + A column that evaluates to a string. str_array : :class:`~pyspark.sql.Column` or str The comma-delimited list. + A column that evaluates to a string. Examples -------- @@ -18495,6 +19051,7 @@ def like( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18506,6 +19063,7 @@ def like( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18545,6 +19103,7 @@ def ilike( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18556,6 +19115,7 @@ def ilike( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18591,6 +19151,7 @@ def lcase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18622,6 +19183,7 @@ def ucase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18654,8 +19216,10 @@ def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or str Input column or strings, the leftmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18678,8 +19242,10 @@ def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. len : :class:`~pyspark.sql.Column` or str Input column or strings, the rightmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18708,18 +19274,24 @@ def mask( ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. upperChar: :class:`~pyspark.sql.Column` or str, optional character to replace upper-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. lowerChar: :class:`~pyspark.sql.Column` or str, optional character to replace lower-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. digitChar: :class:`~pyspark.sql.Column` or str, optional character to replace digit characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. otherChar: :class:`~pyspark.sql.Column` or str, optional character to replace all other characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` + Returns a column that evaluates to a string. Examples -------- @@ -18780,11 +19352,13 @@ def collation(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Target string column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` collation name of a given expression. + Returns a column that evaluates to a string. Examples -------- @@ -18810,11 +19384,13 @@ def quote(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to be quoted. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` quoted string + Returns a column that evaluates to a string. Examples -------- @@ -18942,13 +19518,16 @@ def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing a set of keys. All elements should not be null. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing a set of values. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A column of map type. + Returns a column that evaluates to a map. Notes ----- @@ -19028,6 +19607,7 @@ def array( :class:`~pyspark.sql.Column` A new Column of array type, where each value is an array containing the corresponding values from the input columns. + Returns a column that evaluates to an array. Examples -------- @@ -19118,14 +19698,17 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The target column containing the arrays. + A column that evaluates to an array. value : The value or column to check for in the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding array from the input column contains the specified value. + Returns a column that evaluates to a boolean. Examples -------- @@ -19205,6 +19788,7 @@ def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding arrays from the input columns contain any common elements. + Returns a column that evaluates to a boolean. Examples -------- @@ -19277,17 +19861,21 @@ def slice( ---------- x : :class:`~pyspark.sql.Column` or str Input array column or column name to be sliced. + A column that evaluates to an array. start : :class:`~pyspark.sql.Column`, str, or int The start index for the slice operation. If negative, starts the index from the end of the array. + A column that evaluates to an integer. length : :class:`~pyspark.sql.Column`, str, or int The length of the slice, representing number of elements in the resulting array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new Column object of Array type, where each value is a slice of the corresponding list from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -19354,16 +19942,20 @@ def array_join( ---------- col : :class:`~pyspark.sql.Column` or str The input column containing the arrays to be joined. + A column that evaluates to an array. delimiter : str The string to be used as the delimiter when joining the array elements. + A column that evaluates to a string. null_replacement : str, optional The string to replace null values within the array. If not set, null values are ignored. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of string type, where each value is the result of joining the corresponding array from the input column. + Returns a column that evaluates to a string. Examples -------- @@ -19461,6 +20053,7 @@ def concat(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` concatenated values. Type of the `Column` depends on input columns' type. + Returns a column of the same type as the input. See Also -------- @@ -19548,8 +20141,10 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to an array. value : Any value or a :class:`~pyspark.sql.Column` expression to look for. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `value` now also accepts a Column type. @@ -19558,6 +20153,7 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ------- :class:`~pyspark.sql.Column` position of the value in the given array if found and 0 otherwise. + Returns a column that evaluates to a long. Examples -------- @@ -19653,14 +20249,17 @@ def element_at(col: "ColumnOrName", extraction: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` value at given position. + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -19749,9 +20348,16 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. + + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -19848,13 +20454,16 @@ def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column containing the array. + A column that evaluates to an array. index : :class:`~pyspark.sql.Column` or str or int Index to check for in the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` Value at the given position. + Returns a column of the element type of the input array. Notes ----- @@ -19941,14 +20550,17 @@ def array_prepend(col: "ColumnOrName", value: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` an array with the given value prepended. + Returns a column that evaluates to an array. Examples -------- @@ -20031,9 +20643,11 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. element : - element or a :class:`~pyspark.sql.Column` expression to be removed from the array + element or a :class:`~pyspark.sql.Column` expression to be removed from the array. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `element` now also accepts a Column type. @@ -20042,6 +20656,7 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the given value from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20132,12 +20747,14 @@ def array_distinct(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array of unique values from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20217,17 +20834,21 @@ def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: An Parameters ---------- arr : :class:`~pyspark.sql.Column` or str - name of column containing an array + name of column containing an array. + A column that evaluates to an array. pos : :class:`~pyspark.sql.Column` or str or int name of Numeric type column indicating position of insertion - (starting at index 1, negative position is a start from the back of the array) + (starting at index 1, negative position is a start from the back of the array). + A column that evaluates to an integer. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` an array of values, including the new specified value + Returns a column that evaluates to an array. Notes ----- @@ -20313,13 +20934,16 @@ def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the intersection of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20406,13 +21030,16 @@ def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the union of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20499,13 +21126,16 @@ def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the elements present in col1 but not in col2. + Returns a column that evaluates to an array. Notes ----- @@ -20587,12 +21217,14 @@ def array_compact(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the null values from the input column. + Returns a column that evaluates to an array. Notes ----- @@ -20678,13 +21310,16 @@ def array_append(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column containing the array. + A column that evaluates to an array. value : A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` A new array column with `value` appended to the original array. + Returns a column that evaluates to an array. Notes ----- @@ -20779,6 +21414,7 @@ def explode(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` One row per array item or map key value. + Returns a column of the element type of the input array, or the key and value columns of the input map. See Also -------- @@ -21093,6 +21729,7 @@ def explode_outer(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` one row per array item or map key value. + Returns a column of the element type of the input array, or the key and value columns of the input map. See Also -------- @@ -21275,14 +21912,17 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - string column in json format + string column in json format. + A column that evaluates to a string. path : str - path to the json object to extract + path to the json object to extract. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string representation of given JSON object value. + Returns a column that evaluates to a string. Examples -------- @@ -21355,6 +21995,7 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column: ------- :class:`~pyspark.sql.Column` a new row for each given field value from json object + Returns a column that evaluates to a string. Examples -------- @@ -21508,12 +22149,14 @@ def try_parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21546,6 +22189,7 @@ def to_variant_object( ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21588,12 +22232,14 @@ def parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21952,6 +22598,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or str name of column containing a struct, an array, a map, or a variant object. + A column that evaluates to a struct, array, map, or variant. options : dict, optional options to control converting. accepts the same options as the JSON datasource. See `Data Source Option `_ @@ -21965,6 +22612,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ------- :class:`~pyspark.sql.Column` JSON object as string column. + Returns a column that evaluates to a string. Examples -------- @@ -22125,11 +22773,13 @@ def json_array_length(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` length of json array. + Returns a column that evaluates to an integer. Examples -------- @@ -22153,11 +22803,13 @@ def json_object_keys(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` all the keys of the outermost JSON object. + Returns a column that evaluates to an array. Examples -------- @@ -22620,12 +23272,14 @@ def size(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22650,11 +23304,13 @@ def array_min(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the minimum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22735,11 +23391,13 @@ def array_max(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the maximum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22818,11 +23476,13 @@ def array_size(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the size of each array. + Returns a column that evaluates to an integer. Examples -------- @@ -22900,11 +23560,13 @@ def cardinality(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22940,14 +23602,17 @@ def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column or expression. + A column that evaluates to an array. asc : bool, optional Whether to sort in ascending or descending order. If `asc` is True (default), then the sorting is in ascending order. If False, then in descending order. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` Sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23034,7 +23699,8 @@ def array_sort( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. comparator : callable, optional A binary ``(Column, Column) -> Column: ...``. The comparator will take two @@ -23046,6 +23712,7 @@ def array_sort( ------- :class:`~pyspark.sql.Column` sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23079,6 +23746,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or expression to be shuffled. + A column that evaluates to an array. seed : :class:`~pyspark.sql.Column` or int, optional Seed value for the random generator. @@ -23088,6 +23756,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A new column that contains an array of elements in random order. + Returns a column that evaluates to an array. Notes ----- @@ -23164,12 +23833,14 @@ def reverse(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be reversed. + A column that evaluates to a string, binary, or array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a reversed string, a binary value with bytes in reverse order, or an array with elements in reverse order. + Returns a column of the same type as the input. Examples -------- @@ -23665,14 +24336,17 @@ def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Colu ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be repeated. + A column of any type. count : :class:`~pyspark.sql.Column` or str or int The name of the column, an expression, or an integer that represents the number of times to repeat the element. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an array of repeated elements. + Returns a column that evaluates to an array. Examples -------- @@ -23746,11 +24420,13 @@ def arrays_zip(*cols: "ColumnOrName") -> Column: ---------- cols : :class:`~pyspark.sql.Column` or str Columns of arrays to be merged. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` Merged array of entries. + Returns a column that evaluates to an array. Examples -------- @@ -23931,6 +24607,7 @@ def sequence( ------- :class:`~pyspark.sql.Column` A new column that contains an array of sequence values. + Returns a column that evaluates to an array. Examples -------- @@ -24218,7 +24895,8 @@ def transform( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function a function that is applied to each element of the input array. Can take one of the following forms: @@ -24236,6 +24914,7 @@ def transform( ------- :class:`~pyspark.sql.Column` a new array of transformed elements. + Returns a column that evaluates to an array. Examples -------- @@ -24273,7 +24952,8 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24286,6 +24966,7 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "any" element of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24314,7 +24995,8 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24327,6 +25009,7 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "all" elements of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24370,7 +25053,8 @@ def filter( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function A function that returns the Boolean expression. Can take one of the following forms: @@ -24389,6 +25073,7 @@ def filter( :class:`~pyspark.sql.Column` filtered array of elements where given function evaluated to True when passed as an argument. + Returns a column that evaluates to an array. Examples -------- @@ -24436,12 +25121,14 @@ def aggregate( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``initialValue`` + of the same type as ``initialValue``. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. @@ -24509,12 +25196,14 @@ def reduce( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``zero`` + of the same type as ``zero``. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. @@ -24579,9 +25268,11 @@ def zip_with( Parameters ---------- left : :class:`~pyspark.sql.Column` or str - name of the first column or expression + name of the first column or expression. + A column that evaluates to an array. right : :class:`~pyspark.sql.Column` or str - name of the second column or expression + name of the second column or expression. + A column that evaluates to an array. f : function a binary function ``(x1: Column, x2: Column) -> Column...`` Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24593,6 +25284,7 @@ def zip_with( ------- :class:`~pyspark.sql.Column` array of calculated values derived by applying given function to each pair of arguments. + Returns a column that evaluates to an array. Examples -------- @@ -24855,15 +25547,19 @@ def str_to_map( ---------- text : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. pairDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split pairs. Default is comma (,). + A column that evaluates to a string. keyValueDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split key/value. Default is colon (:). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of map type where each string in the original column is converted into a map. + Returns a column that evaluates to a map. Examples -------- @@ -25129,15 +25825,19 @@ def convert_timezone( sourceTz : :class:`~pyspark.sql.Column`, optional The time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + A column that evaluates to a string. targetTz : :class:`~pyspark.sql.Column` The time zone to which the input timestamp should be converted. + A column that evaluates to a string. sourceTs : :class:`~pyspark.sql.Column` or column name A timestamp without time zone. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp for converted time zone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -25199,17 +25899,22 @@ def make_dt_interval( ---------- days : :class:`~pyspark.sql.Column` or column name, optional The number of days, positive or negative. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The number of hours, positive or negative. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The number of minutes, positive or negative. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a DayTimeIntervalType duration. + Returns a column that evaluates to an interval. See Also -------- @@ -25300,6 +26005,7 @@ def try_make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25317,6 +26023,7 @@ def try_make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25467,6 +26174,7 @@ def make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25484,6 +26192,7 @@ def make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25616,15 +26325,19 @@ def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrNam ---------- hour : :class:`~pyspark.sql.Column` or column name The hour to represent, from 0 to 23. + A column that evaluates to an integer. minute : :class:`~pyspark.sql.Column` or column name The minute to represent, from 0 to 59. + A column that evaluates to an integer. second : :class:`~pyspark.sql.Column` or column name The second to represent, from 0 to 59.999999. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A column representing the created time. + Returns a column that evaluates to a time. Examples -------- @@ -25651,6 +26364,7 @@ def time_from_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Seconds since midnight (0 to 86399.999999). + A column that evaluates to a numeric. Examples -------- @@ -25677,6 +26391,7 @@ def time_from_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Milliseconds since midnight (0 to 86399999). + A column that evaluates to an integral. Examples -------- @@ -25703,6 +26418,7 @@ def time_from_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Microseconds since midnight (0 to 86399999999). + A column that evaluates to an integral. Examples -------- @@ -25872,22 +26588,27 @@ def make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -25895,6 +26616,7 @@ def make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -25910,6 +26632,7 @@ def make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26097,22 +26820,27 @@ def try_make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26120,6 +26848,7 @@ def try_make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -26135,6 +26864,7 @@ def try_make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26294,20 +27024,26 @@ def make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or str - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or str - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or str - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or str - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or str - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or str The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or str, optional The time zone identifier. For example, CET, UTC and etc. @@ -26315,6 +27051,7 @@ def make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26391,20 +27128,26 @@ def try_make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC and etc. @@ -26412,6 +27155,7 @@ def try_make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26528,22 +27272,27 @@ def make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26551,6 +27300,7 @@ def make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26564,6 +27314,7 @@ def make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26683,22 +27434,27 @@ def try_make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26706,6 +27462,7 @@ def try_make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26719,6 +27476,7 @@ def try_make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26804,14 +27562,17 @@ def make_ym_interval( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative + The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative + The number of months, positive or negative. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a year-month interval. + Returns a column that evaluates to an interval. See Also -------- @@ -26972,6 +27733,7 @@ def st_geogfromwkb(wkb: "ColumnOrName") -> Column: ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOGRAPHY value. + A column that evaluates to a binary. Examples -------- @@ -26995,8 +27757,10 @@ def st_geomfromwkb( ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOMETRY value. + A column that evaluates to a binary. srid : :class:`~pyspark.sql.Column` or int, optional The optional SRID value of the geometry. Default is 0. + A column that evaluates to an integer. Examples -------- @@ -27276,8 +28040,10 @@ def hll_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an integer, long, string, or binary. lgConfigK : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of K, where K is the number of buckets or slots for the HllSketch + The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. + A column that evaluates to an integer. Returns ------- @@ -27482,9 +28248,11 @@ def theta_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an array, binary, double, float, integer, long, or string. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries, where nominal entries is the size of the sketch - (must be between 4 and 26, defaults to 12) + (must be between 4 and 26, defaults to 12). + A column that evaluates to an integer. Returns ------- @@ -27634,9 +28402,11 @@ def tuple_sketch_agg_double( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing double summary values + The column containing double summary values. + A column that evaluates to a double. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27688,9 +28458,11 @@ def tuple_sketch_agg_integer( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing integer summary values + The column containing integer summary values. + A column that evaluates to an integer. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27741,7 +28513,8 @@ def tuple_union_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27794,7 +28567,8 @@ def tuple_union_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27846,7 +28620,8 @@ def tuple_intersection_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27896,7 +28671,8 @@ def tuple_intersection_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27947,7 +28723,8 @@ def kll_sketch_agg_bigint( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing bigint values to aggregate + The column containing bigint values to aggregate. + A column that evaluates to an integral. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28025,7 +28802,8 @@ def kll_sketch_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing double values to aggregate + The column containing double values to aggregate. + A column that evaluates to a float or double. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28191,12 +28969,14 @@ def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28221,12 +29001,14 @@ def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28251,12 +29033,14 @@ def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28281,12 +29065,14 @@ def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28314,12 +29100,14 @@ def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28347,12 +29135,14 @@ def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28380,14 +29170,17 @@ def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL bigint sketch + The first KLL bigint sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL bigint sketch + The second KLL bigint sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28412,14 +29205,17 @@ def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Colum Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL float sketch + The first KLL float sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL float sketch + The second KLL float sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28444,14 +29240,17 @@ def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL double sketch + The first KLL double sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL double sketch + The second KLL double sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28477,14 +29276,17 @@ def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a long. Examples -------- @@ -28513,14 +29315,17 @@ def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a float. Examples -------- @@ -28549,14 +29354,17 @@ def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a double. Examples -------- @@ -28585,14 +29393,17 @@ def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a long or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28621,14 +29432,17 @@ def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a float or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28657,14 +29471,17 @@ def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -29807,7 +30624,9 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- @@ -29842,7 +30661,9 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- @@ -29877,6 +30698,7 @@ def nullifzero(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric. Examples -------- @@ -30031,26 +30853,33 @@ def aes_encrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to encrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to encrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. iv : :class:`~pyspark.sql.Column` or column name, optional Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or "". 16-byte array for CBC mode. 12-byte array for GCM mode. + A column that evaluates to a binary. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an encrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30170,23 +30999,29 @@ def aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30294,23 +31129,29 @@ def try_aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value or a NULL value. + Returns a column that evaluates to a binary. See Also -------- @@ -30423,6 +31264,7 @@ def sha(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a binary. See Also -------- @@ -30760,6 +31602,7 @@ def bitmap_bit_position(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30793,6 +31636,7 @@ def bitmap_bucket_number(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30827,6 +31671,7 @@ def bitmap_construct_agg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column will most likely be bitmap_bit_position(). + A column that evaluates to a long. See Also -------- diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index c8b269a63de5e..cb87005dcc285 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -113,6 +113,8 @@ object functions { * * @group normal_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def lit(literal: Any): Column = { literal match { @@ -135,6 +137,8 @@ object functions { * * @group normal_funcs * @since 2.2.0 + * @return + * Returns a column of the same type as the input. */ def typedLit[T: TypeTag](literal: T): Column = { typedlit(literal) @@ -154,6 +158,8 @@ object functions { * * @group normal_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def typedlit[T: TypeTag](literal: T): Column = { literal match { @@ -246,6 +252,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column): Column = approx_count_distinct(e) @@ -253,6 +261,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) @@ -260,6 +270,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) @@ -267,6 +279,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String, rsd: Double): Column = { @@ -278,6 +292,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) @@ -286,6 +302,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String): Column = approx_count_distinct( column(columnName)) @@ -298,6 +316,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column, rsd: Double): Column = { Column.fn("approx_count_distinct", e, lit(rsd)) @@ -311,6 +331,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String, rsd: Double): Column = { approx_count_distinct(Column(columnName), rsd) @@ -321,6 +343,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(e: Column): Column = Column.fn("avg", e) @@ -329,6 +353,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(columnName: String): Column = avg(Column(columnName)) @@ -341,6 +367,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(e: Column): Column = Column.fn("collect_list", e) @@ -353,6 +381,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(columnName: String): Column = collect_list(Column(columnName)) @@ -365,6 +395,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(e: Column): Column = Column.fn("collect_set", e) @@ -377,6 +409,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(columnName: String): Column = collect_set(Column(columnName)) @@ -388,6 +422,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = Column.fn("count_min_sketch", e, eps, confidence, seed) @@ -400,6 +436,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) @@ -409,6 +447,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) @@ -417,6 +457,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(columnName1: String, columnName2: String): Column = { corr(Column(columnName1), Column(columnName2)) @@ -427,6 +469,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(e: Column): Column = Column.fn("count", e) @@ -436,6 +480,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(columnName: String): TypedColumn[Any, Long] = count(Column(columnName)).as(PrimitiveLongEncoder) @@ -447,6 +493,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) @@ -458,6 +506,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(columnName: String, columnNames: String*): Column = @@ -468,6 +518,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def count_distinct(expr: Column, exprs: Column*): Column = @@ -478,6 +530,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(column1: Column, column2: Column): Column = Column.fn("covar_pop", column1, column2) @@ -487,6 +541,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(columnName1: String, columnName2: String): Column = { covar_pop(Column(columnName1), Column(columnName2)) @@ -497,6 +553,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(column1: Column, column2: Column): Column = Column.fn("covar_samp", column1, column2) @@ -506,6 +564,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(columnName1: String, columnName2: String): Column = { covar_samp(Column(columnName1), Column(columnName2)) @@ -523,6 +583,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column, ignoreNulls: Boolean): Column = Column.fn("first", false, e, lit(ignoreNulls)) @@ -539,6 +601,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String, ignoreNulls: Boolean): Column = { first(Column(columnName), ignoreNulls) @@ -556,6 +620,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column): Column = first(e, ignoreNulls = false) @@ -571,6 +637,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String): Column = first(Column(columnName)) @@ -583,6 +651,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column): Column = Column.fn("first_value", e) @@ -598,6 +668,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column, ignoreNulls: Column): Column = Column.fn("first_value", e, ignoreNulls) @@ -608,6 +680,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(e: Column): Column = Column.fn("grouping", e) @@ -617,6 +691,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(columnName: String): Column = grouping(Column(columnName)) @@ -633,6 +709,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) @@ -649,6 +727,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(colName: String, colNames: String*): Column = { @@ -661,6 +741,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Column): Column = Column.fn("hll_sketch_agg", e, lgConfigK) @@ -671,6 +753,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Int): Column = Column.fn("hll_sketch_agg", e, lit(lgConfigK)) @@ -681,6 +765,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { hll_sketch_agg(Column(columnName), lgConfigK) @@ -692,6 +778,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column): Column = Column.fn("hll_sketch_agg", e) @@ -702,6 +790,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String): Column = { hll_sketch_agg(Column(columnName)) @@ -715,6 +805,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = Column.fn("hll_union_agg", e, allowDifferentLgConfigK) @@ -727,6 +819,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) @@ -739,6 +833,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { hll_union_agg(Column(columnName), allowDifferentLgConfigK) @@ -751,6 +847,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column): Column = Column.fn("hll_union_agg", e) @@ -762,6 +860,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String): Column = { hll_union_agg(Column(columnName)) @@ -772,6 +872,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(e: Column): Column = Column.fn("kurtosis", e) @@ -780,6 +882,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) @@ -795,6 +899,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column, ignoreNulls: Boolean): Column = Column.fn("last", false, e, lit(ignoreNulls)) @@ -811,6 +917,8 @@ object functions { * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String, ignoreNulls: Boolean): Column = { last(Column(columnName), ignoreNulls) @@ -828,6 +936,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column): Column = last(e, ignoreNulls = false) @@ -843,6 +953,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) @@ -855,6 +967,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column): Column = Column.fn("last_value", e) @@ -870,6 +984,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column, ignoreNulls: Column): Column = Column.fn("last_value", e, ignoreNulls) @@ -878,13 +994,15 @@ object functions { * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * * @param hour - * the hour to represent, from 0 to 23 + * the hour to represent, from 0 to 23. A column that evaluates to an integer. * @param minute - * the minute to represent, from 0 to 59 + * the minute to represent, from 0 to 59. A column that evaluates to an integer. * @param second - * the second to represent, from 0 to 59.999999 + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a time. */ def make_time(hour: Column, minute: Column, second: Column): Column = { Column.fn("make_time", hour, minute, second) @@ -895,6 +1013,8 @@ object functions { * * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column): Column = Column.fn("mode", e) @@ -907,6 +1027,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) @@ -915,6 +1037,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(e: Column): Column = Column.fn("max", e) @@ -923,6 +1047,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(columnName: String): Column = max(Column(columnName)) @@ -935,6 +1061,8 @@ object functions { * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) @@ -953,6 +1081,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) @@ -971,6 +1101,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) @@ -979,6 +1111,8 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(e: Column): Column = avg(e) @@ -987,6 +1121,8 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(columnName: String): Column = avg(columnName) @@ -995,6 +1131,8 @@ object functions { * * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a double. */ def median(e: Column): Column = Column.fn("median", e) @@ -1003,6 +1141,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(e: Column): Column = Column.fn("min", e) @@ -1011,6 +1151,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(columnName: String): Column = min(Column(columnName)) @@ -1023,6 +1165,8 @@ object functions { * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) @@ -1042,6 +1186,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) @@ -1061,6 +1207,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) @@ -1070,6 +1218,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) @@ -1079,6 +1229,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column, frequency: Column): Column = Column.fn("percentile", e, percentage, frequency) @@ -1097,6 +1249,8 @@ object functions { * * @group agg_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = Column.fn("percentile_approx", e, percentage, accuracy) @@ -1115,6 +1269,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { Column.fn("approx_percentile", e, percentage, accuracy) @@ -1125,6 +1281,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a double. */ def product(e: Column): Column = Column.internalFn("product", e) @@ -1133,6 +1291,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(e: Column): Column = Column.fn("skewness", e) @@ -1141,6 +1301,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(columnName: String): Column = skewness(Column(columnName)) @@ -1149,6 +1311,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def std(e: Column): Column = Column.fn("std", e) @@ -1157,6 +1321,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(e: Column): Column = Column.fn("stddev", e) @@ -1165,6 +1331,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(columnName: String): Column = stddev(Column(columnName)) @@ -1173,6 +1341,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) @@ -1181,6 +1351,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) @@ -1189,6 +1361,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) @@ -1197,6 +1371,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) @@ -1205,6 +1381,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(e: Column): Column = Column.fn("sum", e) @@ -1213,6 +1391,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(columnName: String): Column = sum(Column(columnName)) @@ -1221,6 +1401,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(e: Column): Column = sum_distinct(e) @@ -1230,6 +1412,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) @@ -1239,6 +1423,8 @@ object functions { * * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) @@ -1249,6 +1435,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(e: Column): Column = Column.fn("theta_intersection_agg", e) @@ -1260,6 +1448,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(columnName: String): Column = theta_intersection_agg(Column(columnName)) @@ -1271,6 +1461,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_sketch_agg", e, lgNomEntries) @@ -1282,6 +1474,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) @@ -1293,6 +1487,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = theta_sketch_agg(Column(columnName), lgNomEntries) @@ -1304,6 +1500,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column): Column = Column.fn("theta_sketch_agg", e) @@ -1315,6 +1513,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String): Column = theta_sketch_agg(Column(columnName)) @@ -1327,6 +1527,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_union_agg", e, lgNomEntries) @@ -1339,6 +1541,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_union_agg", e, lit(lgNomEntries)) @@ -1351,6 +1555,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String, lgNomEntries: Int): Column = theta_union_agg(Column(columnName), lgNomEntries) @@ -1363,6 +1569,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column): Column = Column.fn("theta_union_agg", e) @@ -1375,6 +1583,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String): Column = theta_union_agg(Column(columnName)) @@ -1387,6 +1597,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_double", e, mode) @@ -1399,6 +1611,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_double", e, lit(mode)) @@ -1411,6 +1625,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String, mode: String): Column = tuple_intersection_agg_double(Column(columnName), mode) @@ -1423,6 +1639,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column): Column = Column.fn("tuple_intersection_agg_double", e) @@ -1435,6 +1653,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String): Column = tuple_intersection_agg_double(Column(columnName)) @@ -1448,6 +1668,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_integer", e, mode) @@ -1461,6 +1683,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_integer", e, lit(mode)) @@ -1474,6 +1698,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String, mode: String): Column = tuple_intersection_agg_integer(Column(columnName), mode) @@ -1486,6 +1712,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column): Column = Column.fn("tuple_intersection_agg_integer", e) @@ -1498,6 +1726,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String): Column = tuple_intersection_agg_integer(Column(columnName)) @@ -1510,6 +1740,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1526,6 +1758,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1542,6 +1776,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1557,6 +1793,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) @@ -1568,6 +1806,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1582,6 +1822,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_double", key, summary) @@ -1593,6 +1835,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) @@ -1605,6 +1849,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1621,6 +1867,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1637,6 +1885,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1652,6 +1902,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) @@ -1663,6 +1915,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1677,6 +1931,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_integer", key, summary) @@ -1688,6 +1944,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) @@ -1701,6 +1959,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) @@ -1714,6 +1974,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) @@ -1727,6 +1989,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_double(Column(columnName), lgNomEntries, mode) @@ -1739,6 +2003,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) @@ -1751,6 +2017,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_double(Column(columnName), lgNomEntries) @@ -1763,6 +2031,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column): Column = Column.fn("tuple_union_agg_double", e) @@ -1775,6 +2045,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String): Column = tuple_union_agg_double(Column(columnName)) @@ -1788,6 +2060,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) @@ -1801,6 +2075,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) @@ -1814,6 +2090,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) @@ -1826,6 +2104,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) @@ -1838,6 +2118,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries) @@ -1850,6 +2132,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column): Column = Column.fn("tuple_union_agg_integer", e) @@ -1862,6 +2146,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String): Column = tuple_union_agg_integer(Column(columnName)) @@ -1873,6 +2159,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_bigint", e, k) @@ -1884,6 +2172,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_bigint", e, lit(k)) @@ -1895,6 +2185,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String, k: Int): Column = kll_sketch_agg_bigint(Column(columnName), k) @@ -1905,6 +2197,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column): Column = Column.fn("kll_sketch_agg_bigint", e) @@ -1915,6 +2209,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String): Column = kll_sketch_agg_bigint(Column(columnName)) @@ -1926,6 +2222,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_float", e, k) @@ -1937,6 +2235,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_float", e, lit(k)) @@ -1948,6 +2248,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String, k: Int): Column = kll_sketch_agg_float(Column(columnName), k) @@ -1958,6 +2260,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column): Column = Column.fn("kll_sketch_agg_float", e) @@ -1968,6 +2272,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String): Column = kll_sketch_agg_float(Column(columnName)) @@ -1979,6 +2285,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_double", e, k) @@ -1990,6 +2298,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_double", e, lit(k)) @@ -2001,6 +2311,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String, k: Int): Column = kll_sketch_agg_double(Column(columnName), k) @@ -2011,6 +2323,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column): Column = Column.fn("kll_sketch_agg_double", e) @@ -2021,6 +2335,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String): Column = kll_sketch_agg_double(Column(columnName)) @@ -2033,6 +2349,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_merge_agg_bigint", e, k) @@ -2045,6 +2363,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_merge_agg_bigint", e, lit(k)) @@ -2057,6 +2377,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String, k: Int): Column = kll_merge_agg_bigint(Column(columnName), k) @@ -2068,6 +2390,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column): Column = Column.fn("kll_merge_agg_bigint", e) @@ -2079,6 +2403,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String): Column = kll_merge_agg_bigint(Column(columnName)) @@ -2090,6 +2416,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Column): Column = Column.fn("kll_merge_agg_float", e, k) @@ -2101,6 +2429,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Int): Column = Column.fn("kll_merge_agg_float", e, lit(k)) @@ -2112,6 +2442,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String, k: Int): Column = kll_merge_agg_float(Column(columnName), k) @@ -2122,6 +2454,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column): Column = Column.fn("kll_merge_agg_float", e) @@ -2132,6 +2466,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String): Column = kll_merge_agg_float(Column(columnName)) @@ -2143,6 +2479,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Column): Column = Column.fn("kll_merge_agg_double", e, k) @@ -2154,6 +2492,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Int): Column = Column.fn("kll_merge_agg_double", e, lit(k)) @@ -2165,6 +2505,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String, k: Int): Column = kll_merge_agg_double(Column(columnName), k) @@ -2175,6 +2517,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column): Column = Column.fn("kll_merge_agg_double", e) @@ -2185,6 +2529,8 @@ object functions { * * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String): Column = kll_merge_agg_double(Column(columnName)) @@ -2194,6 +2540,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column): Column = Column.fn("listagg", e) @@ -2203,6 +2551,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) @@ -2211,6 +2561,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) @@ -2220,6 +2572,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column, delimiter: Column): Column = Column.fn("listagg", isDistinct = true, e, delimiter) @@ -2229,6 +2583,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column): Column = Column.fn("string_agg", e) @@ -2238,6 +2594,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) @@ -2247,6 +2605,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) @@ -2256,6 +2616,8 @@ object functions { * * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column, delimiter: Column): Column = Column.fn("string_agg", isDistinct = true, e, delimiter) @@ -2265,6 +2627,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(e: Column): Column = Column.fn("variance", e) @@ -2273,6 +2637,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(columnName: String): Column = variance(Column(columnName)) @@ -2281,6 +2647,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(e: Column): Column = Column.fn("var_samp", e) @@ -2289,6 +2657,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(columnName: String): Column = var_samp(Column(columnName)) @@ -2297,6 +2667,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(e: Column): Column = Column.fn("var_pop", e) @@ -2305,6 +2677,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(columnName: String): Column = var_pop(Column(columnName)) @@ -2314,6 +2688,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) @@ -2323,6 +2699,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) @@ -2332,6 +2710,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) @@ -2342,6 +2722,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) @@ -2351,6 +2733,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) @@ -2360,6 +2744,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) @@ -2369,6 +2755,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) @@ -2378,6 +2766,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) @@ -2387,6 +2777,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) @@ -2395,6 +2787,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column): Column = Column.fn("any_value", e) @@ -2404,6 +2798,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column, ignoreNulls: Column): Column = Column.fn("any_value", e, ignoreNulls) @@ -2413,6 +2809,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def count_if(e: Column): Column = Column.fn("count_if", e) @@ -2421,8 +2819,7 @@ object functions { * 6 fractional digits of seconds. * * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2437,8 +2834,7 @@ object functions { * An integer literal in the range [0..6], indicating how many fractional digits of seconds to * include in the result. * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2459,6 +2855,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def histogram_numeric(e: Column, nBins: Column): Column = Column.fn("histogram_numeric", e, nBins) @@ -2468,6 +2866,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def every(e: Column): Column = Column.fn("every", e) @@ -2476,6 +2876,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_and(e: Column): Column = Column.fn("bool_and", e) @@ -2484,6 +2886,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def some(e: Column): Column = Column.fn("some", e) @@ -2492,6 +2896,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def any(e: Column): Column = Column.fn("any", e) @@ -2500,6 +2906,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_or(e: Column): Column = Column.fn("bool_or", e) @@ -2508,6 +2916,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_and(e: Column): Column = Column.fn("bit_and", e) @@ -2516,6 +2926,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_or(e: Column): Column = Column.fn("bit_or", e) @@ -2524,6 +2936,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_xor(e: Column): Column = Column.fn("bit_xor", e) @@ -2550,8 +2964,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the same + * type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2580,8 +2994,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the same + * type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2599,6 +3013,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def cume_dist(): Column = Column.fn("cume_dist") @@ -2615,6 +3031,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def dense_rank(): Column = Column.fn("dense_rank") @@ -2627,6 +3045,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int): Column = lag(e, offset, null) @@ -2639,6 +3059,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) @@ -2651,6 +3073,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int, defaultValue: Any): Column = { lag(Column(columnName), offset, defaultValue) @@ -2665,6 +3089,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any): Column = { lag(e, offset, defaultValue, false) @@ -2681,6 +3107,8 @@ object functions { * * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2694,6 +3122,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } @@ -2706,6 +3136,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } @@ -2718,6 +3150,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int, defaultValue: Any): Column = { lead(Column(columnName), offset, defaultValue) @@ -2732,6 +3166,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any): Column = { lead(e, offset, defaultValue, false) @@ -2748,6 +3184,8 @@ object functions { * * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2763,6 +3201,8 @@ object functions { * * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) @@ -2775,6 +3215,8 @@ object functions { * * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) @@ -2787,6 +3229,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def ntile(n: Int): Column = Column.fn("ntile", lit(n)) @@ -2803,6 +3247,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def percent_rank(): Column = Column.fn("percent_rank") @@ -2819,6 +3265,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def rank(): Column = Column.fn("rank") @@ -2827,6 +3275,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def row_number(): Column = Column.fn("row_number") @@ -2839,6 +3289,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(cols: Column*): Column = Column.fn("array", cols: _*) @@ -2848,6 +3300,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(colName: String, colNames: String*): Column = { @@ -2861,6 +3315,8 @@ object functions { * * @group map_funcs * @since 2.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map(cols: Column*): Column = Column.fn("map", cols: _*) @@ -2870,6 +3326,8 @@ object functions { * * @group struct_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) @@ -2880,6 +3338,8 @@ object functions { * * @group map_funcs * @since 2.4 + * @return + * Returns a column that evaluates to a map. */ def map_from_arrays(keys: Column, values: Column): Column = Column.fn("map_from_arrays", keys, values) @@ -2890,6 +3350,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = Column.fn("str_to_map", text, pairDelim, keyValueDelim) @@ -2900,6 +3362,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column): Column = Column.fn("str_to_map", text, pairDelim) @@ -2909,6 +3373,8 @@ object functions { * * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column): Column = Column.fn("str_to_map", text) @@ -2936,6 +3402,8 @@ object functions { * * @group conditional_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) @@ -2945,6 +3413,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def input_file_name(): Column = Column.fn("input_file_name") @@ -2953,6 +3423,8 @@ object functions { * * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnan(e: Column): Column = e.isNaN @@ -2961,6 +3433,8 @@ object functions { * * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnull(e: Column): Column = e.isNull @@ -2981,6 +3455,8 @@ object functions { * * @group misc_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use monotonically_increasing_id()", "2.0.0") def monotonicallyIncreasingId(): Column = monotonically_increasing_id() @@ -3002,6 +3478,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a long. */ def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") @@ -3012,6 +3490,8 @@ object functions { * * @group conditional_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) @@ -3028,6 +3508,8 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def negate(e: Column): Column = -e @@ -3043,6 +3525,8 @@ object functions { * * @group predicate_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def not(e: Column): Column = !e @@ -3055,6 +3539,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(seed: Long): Column = Column.fn("rand", lit(seed)) @@ -3067,6 +3553,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(): Column = rand(SparkClassUtils.random.nextLong) @@ -3079,6 +3567,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(seed: Long): Column = Column.fn("randn", lit(seed)) @@ -3091,6 +3581,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(): Column = randn(SparkClassUtils.random.nextLong) @@ -3101,6 +3593,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column): Column = randstr(length, lit(SparkClassUtils.random.nextLong)) @@ -3112,6 +3606,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) @@ -3123,6 +3619,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def spark_partition_id(): Column = Column.fn("spark_partition_id") @@ -3131,6 +3629,8 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(e: Column): Column = Column.fn("sqrt", e) @@ -3139,6 +3639,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(colName: String): Column = sqrt(Column(colName)) @@ -3148,6 +3650,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) @@ -3156,6 +3660,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def try_avg(e: Column): Column = Column.fn("try_avg", e) @@ -3165,6 +3671,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) @@ -3173,6 +3681,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) @@ -3182,6 +3692,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) @@ -3191,6 +3703,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) @@ -3199,6 +3713,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a numeric. */ def try_sum(e: Column): Column = Column.fn("try_sum", e) @@ -3210,6 +3726,8 @@ object functions { * * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(cols: Column*): Column = Column.fn("struct", cols: _*) @@ -3219,6 +3737,8 @@ object functions { * * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(colName: String, colNames: String*): Column = { @@ -3245,6 +3765,8 @@ object functions { * * @group conditional_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def when(condition: Column, value: Any): Column = Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) @@ -3254,6 +3776,8 @@ object functions { * * @group bitwise_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use bitwise_not", "3.2.0") def bitwiseNOT(e: Column): Column = bitwise_not(e) @@ -3263,6 +3787,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def bitwise_not(e: Column): Column = Column.fn("~", e) @@ -3272,6 +3798,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_count(e: Column): Column = Column.fn("bit_count", e) @@ -3281,6 +3809,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos) @@ -3290,6 +3820,8 @@ object functions { * * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) @@ -3314,12 +3846,15 @@ object functions { * * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def abs(e: Column): Column = Column.fn("abs", e) /** * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3328,8 +3863,8 @@ object functions { /** * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos` - * + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3337,7 +3872,7 @@ object functions { /** * @return - * inverse hyperbolic cosine of `e` + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3346,8 +3881,7 @@ object functions { /** * @return - * inverse hyperbolic cosine of `columnName` - * + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3355,7 +3889,8 @@ object functions { /** * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3364,8 +3899,8 @@ object functions { /** * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin` - * + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3373,7 +3908,7 @@ object functions { /** * @return - * inverse hyperbolic sine of `e` + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3382,8 +3917,7 @@ object functions { /** * @return - * inverse hyperbolic sine of `columnName` - * + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3391,7 +3925,8 @@ object functions { /** * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan` + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3400,8 +3935,8 @@ object functions { /** * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan` - * + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3409,13 +3944,13 @@ object functions { /** * @param y - * coordinate on y-axis + * coordinate on y-axis. A column that evaluates to a double. * @param x - * coordinate on x-axis + * coordinate on x-axis. A column that evaluates to a double. * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3430,8 +3965,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3445,8 +3979,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3460,8 +3993,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3476,8 +4008,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3491,8 +4022,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3506,8 +4036,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3521,8 +4050,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3530,7 +4058,7 @@ object functions { /** * @return - * inverse hyperbolic tangent of `e` + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3539,8 +4067,7 @@ object functions { /** * @return - * inverse hyperbolic tangent of `columnName` - * + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3552,6 +4079,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(e: Column): Column = Column.fn("bin", e) @@ -3561,6 +4090,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(columnName: String): Column = bin(Column(columnName)) @@ -3569,6 +4100,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(e: Column): Column = Column.fn("cbrt", e) @@ -3577,6 +4110,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(columnName: String): Column = cbrt(Column(columnName)) @@ -3585,6 +4120,8 @@ object functions { * * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) @@ -3593,6 +4130,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column): Column = Column.fn("ceil", e) @@ -3601,6 +4140,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(columnName: String): Column = ceil(Column(columnName)) @@ -3609,6 +4150,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) @@ -3617,6 +4160,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column): Column = Column.fn("ceiling", e) @@ -3625,15 +4170,18 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def conv(num: Column, fromBase: Int, toBase: Int): Column = Column.fn("conv", num, lit(fromBase), lit(toBase)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -3644,8 +4192,8 @@ object functions { * @param columnName * angle in radians * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` - * + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that evaluates + * to a double. * @group math_funcs * @since 1.4.0 */ @@ -3653,9 +4201,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3666,8 +4215,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` - * + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3675,9 +4224,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cotangent of the angle + * cotangent of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3686,9 +4235,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosecant of the angle + * cosecant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3700,6 +4249,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def e(): Column = Column.fn("e") @@ -3708,6 +4259,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(e: Column): Column = Column.fn("exp", e) @@ -3716,6 +4269,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(columnName: String): Column = exp(Column(columnName)) @@ -3724,6 +4279,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(e: Column): Column = Column.fn("expm1", e) @@ -3732,6 +4289,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(columnName: String): Column = expm1(Column(columnName)) @@ -3740,6 +4299,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def factorial(e: Column): Column = Column.fn("factorial", e) @@ -3748,6 +4309,8 @@ object functions { * * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) @@ -3756,6 +4319,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column): Column = Column.fn("floor", e) @@ -3764,6 +4329,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(columnName: String): Column = floor(Column(columnName)) @@ -3773,6 +4340,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) @@ -3783,6 +4352,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(columnName: String, columnNames: String*): Column = { @@ -3794,6 +4365,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def hex(column: Column): Column = Column.fn("hex", column) @@ -3803,6 +4376,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unhex(column: Column): Column = Column.fn("unhex", column) @@ -3811,6 +4386,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) @@ -3819,6 +4396,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) @@ -3827,6 +4406,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) @@ -3835,6 +4416,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, rightName: String): Column = hypot(Column(leftName), Column(rightName)) @@ -3844,6 +4427,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) @@ -3852,6 +4437,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) @@ -3860,6 +4447,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, r: Column): Column = hypot(lit(l), r) @@ -3868,6 +4457,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) @@ -3877,6 +4468,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(exprs: Column*): Column = Column.fn("least", exprs: _*) @@ -3887,6 +4480,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(columnName: String, columnNames: String*): Column = { @@ -3898,6 +4493,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def ln(e: Column): Column = Column.fn("ln", e) @@ -3906,6 +4503,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(e: Column): Column = ln(e) @@ -3914,6 +4513,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(columnName: String): Column = log(Column(columnName)) @@ -3922,6 +4523,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) @@ -3930,6 +4533,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, columnName: String): Column = log(base, Column(columnName)) @@ -3938,6 +4543,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(e: Column): Column = Column.fn("log10", e) @@ -3946,6 +4553,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(columnName: String): Column = log10(Column(columnName)) @@ -3954,6 +4563,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(e: Column): Column = Column.fn("log1p", e) @@ -3962,6 +4573,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(columnName: String): Column = log1p(Column(columnName)) @@ -3970,6 +4583,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(expr: Column): Column = Column.fn("log2", expr) @@ -3978,6 +4593,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(columnName: String): Column = log2(Column(columnName)) @@ -3986,6 +4603,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def negative(e: Column): Column = Column.fn("negative", e) @@ -3994,6 +4613,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def pi(): Column = Column.fn("pi") @@ -4002,6 +4623,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def positive(e: Column): Column = Column.fn("positive", e) @@ -4010,6 +4633,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Column): Column = Column.fn("power", l, r) @@ -4018,6 +4643,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) @@ -4026,6 +4653,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) @@ -4034,6 +4663,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) @@ -4042,6 +4673,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Double): Column = pow(l, lit(r)) @@ -4050,6 +4683,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) @@ -4058,6 +4693,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, r: Column): Column = pow(lit(l), r) @@ -4066,6 +4703,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) @@ -4074,6 +4713,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def power(l: Column, r: Column): Column = Column.fn("power", l, r) @@ -4082,6 +4723,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) @@ -4091,6 +4734,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(e: Column): Column = Column.fn("rint", e) @@ -4100,6 +4745,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(columnName: String): Column = rint(Column(columnName)) @@ -4108,6 +4755,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column): Column = round(e, 0) @@ -4117,6 +4766,8 @@ object functions { * * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) @@ -4126,6 +4777,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) @@ -4134,6 +4787,8 @@ object functions { * * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column): Column = bround(e, 0) @@ -4143,6 +4798,8 @@ object functions { * * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) @@ -4152,14 +4809,16 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * secant of the angle + * secant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4172,6 +4831,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftleft", "3.2.0") def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) @@ -4182,6 +4843,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) @@ -4191,6 +4854,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftright", "3.2.0") def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) @@ -4201,6 +4866,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) @@ -4210,6 +4877,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftrightunsigned", "3.2.0") def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) @@ -4220,6 +4889,8 @@ object functions { * * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftrightunsigned(e: Column, numBits: Int): Column = Column.fn("shiftrightunsigned", e, lit(numBits)) @@ -4229,6 +4900,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def sign(e: Column): Column = Column.fn("sign", e) @@ -4237,6 +4910,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(e: Column): Column = Column.fn("signum", e) @@ -4245,14 +4920,17 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(columnName: String): Column = signum(Column(columnName)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -4263,8 +4941,8 @@ object functions { * @param columnName * angle in radians * @return - * sine of the angle, as if computed by `java.lang.Math.sin` - * + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates to + * a double. * @group math_funcs * @since 1.4.0 */ @@ -4272,10 +4950,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4285,8 +4963,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4294,9 +4972,10 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4307,8 +4986,8 @@ object functions { * @param columnName * angle in radians * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` - * + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4316,10 +4995,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4329,8 +5008,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4339,6 +5018,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(e: Column): Column = degrees(e) @@ -4346,6 +5027,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(columnName: String): Column = degrees(Column(columnName)) @@ -4355,9 +5038,10 @@ object functions { * degrees. * * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4371,8 +5055,8 @@ object functions { * @param columnName * angle in radians * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` - * + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4381,6 +5065,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(e: Column): Column = radians(e) @@ -4388,6 +5074,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(columnName: String): Column = radians(Column(columnName)) @@ -4397,9 +5085,10 @@ object functions { * radians. * * @param e - * angle in degrees + * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4413,8 +5102,8 @@ object functions { * @param columnName * angle in degrees * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` - * + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4426,15 +5115,17 @@ object functions { * method will return null. * * @param v - * value to compute a bucket number in the histogram + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. * @param min - * minimum value of the histogram + * minimum value of the histogram. A column that evaluates to a double or interval. * @param max - * maximum value of the histogram + * maximum value of the histogram. A column that evaluates to a double or interval. * @param numBucket - * the number of buckets + * the number of buckets. A column that evaluates to a long. * @return - * the bucket number into which the value would fall after being evaluated + * the bucket number into which the value would fall after being evaluated. Returns a column + * that evaluates to a long. * @group math_funcs * @since 3.5.0 */ @@ -4450,6 +5141,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_catalog(): Column = Column.fn("current_catalog") @@ -4458,6 +5151,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_database(): Column = Column.fn("current_database") @@ -4466,6 +5161,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_schema(): Column = Column.fn("current_schema") @@ -4474,6 +5171,8 @@ object functions { * * @group misc_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a string. */ def current_path(): Column = Column.fn("current_path") @@ -4482,6 +5181,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_user(): Column = Column.fn("current_user") @@ -4491,6 +5192,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def md5(e: Column): Column = Column.fn("md5", e) @@ -4500,6 +5203,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha1(e: Column): Column = Column.fn("sha1", e) @@ -4508,12 +5213,14 @@ object functions { * hex string. * * @param e - * column to compute SHA-2 on. + * column to compute SHA-2 on. A column that evaluates to a binary. * @param numBits - * one of 224, 256, 384, or 512. + * one of 224, 256, 384, or 512. A column that evaluates to an integer. * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha2(e: Column, numBits: Int): Column = { require( @@ -4528,6 +5235,8 @@ object functions { * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def crc32(e: Column): Column = Column.fn("crc32", e) @@ -4536,6 +5245,8 @@ object functions { * * @group hash_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to an integer. */ @scala.annotation.varargs def hash(cols: Column*): Column = Column.fn("hash", cols: _*) @@ -4546,6 +5257,8 @@ object functions { * * @group hash_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) @@ -4555,6 +5268,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column): Column = Column.fn("assert_true", c) @@ -4563,6 +5278,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) @@ -4571,6 +5288,8 @@ object functions { * * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def raise_error(c: Column): Column = Column.fn("raise_error", c) @@ -4579,6 +5298,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def user(): Column = Column.fn("user") @@ -4587,6 +5308,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def session_user(): Column = Column.fn("session_user") @@ -4596,6 +5319,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) @@ -4605,6 +5330,8 @@ object functions { * * @group misc_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(seed: Column): Column = Column.fn("uuid", seed) @@ -4619,25 +5346,29 @@ object functions { * GCM. * * @param input - * The binary value to encrypt. + * The binary value to encrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to encrypt the data. + * The passphrase to use to encrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param iv * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt( input: Column, @@ -4656,6 +5387,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv) @@ -4669,6 +5402,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding) @@ -4682,6 +5417,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_encrypt", input, key, mode) @@ -4695,6 +5432,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column): Column = Column.fn("aes_encrypt", input, key) @@ -4707,22 +5446,25 @@ object functions { * for decryption. The default mode is GCM. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt( input: Column, @@ -4740,6 +5482,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_decrypt", input, key, mode, padding) @@ -4752,6 +5496,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_decrypt", input, key, mode) @@ -4764,6 +5510,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column): Column = Column.fn("aes_decrypt", input, key) @@ -4773,22 +5521,25 @@ object functions { * NULL value instead of raising an error if the decryption cannot be performed. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt( input: Column, @@ -4806,6 +5557,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("try_aes_decrypt", input, key, mode, padding) @@ -4818,6 +5571,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("try_aes_decrypt", input, key, mode) @@ -4830,6 +5585,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column): Column = Column.fn("try_aes_decrypt", input, key) @@ -4839,6 +5596,8 @@ object functions { * * @group hash_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha(col: Column): Column = Column.fn("sha", col) @@ -4847,6 +5606,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_length(): Column = Column.fn("input_file_block_length") @@ -4855,6 +5616,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_start(): Column = Column.fn("input_file_block_start") @@ -4863,6 +5626,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) @@ -4872,6 +5637,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) @@ -4882,6 +5649,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) @@ -4892,6 +5661,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def version(): Column = Column.fn("version") @@ -4900,6 +5671,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def typeof(col: Column): Column = Column.fn("typeof", col) @@ -4909,6 +5682,8 @@ object functions { * * @group generator_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def stack(cols: Column*): Column = Column.fn("stack", cols: _*) @@ -4922,6 +5697,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column): Column = uniform(min, max, lit(SparkClassUtils.random.nextLong)) @@ -4935,6 +5712,8 @@ object functions { * * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column, seed: Column): Column = Column.fn("uniform", min, max, seed) @@ -4945,6 +5724,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(seed: Column): Column = Column.fn("random", seed) @@ -4954,6 +5735,8 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(): Column = random(lit(SparkClassUtils.random.nextLong)) @@ -4962,6 +5745,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bit_position(col: Column): Column = Column.fn("bitmap_bit_position", col) @@ -4971,6 +5756,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bucket_number(col: Column): Column = Column.fn("bitmap_bucket_number", col) @@ -4981,6 +5768,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_construct_agg(col: Column): Column = Column.fn("bitmap_construct_agg", col) @@ -4990,6 +5779,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) @@ -4999,6 +5790,8 @@ object functions { * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) @@ -5008,6 +5801,8 @@ object functions { * * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) @@ -5021,6 +5816,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def ascii(e: Column): Column = Column.fn("ascii", e) @@ -5030,6 +5827,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def base64(e: Column): Column = Column.fn("base64", e) @@ -5038,6 +5837,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_length(e: Column): Column = Column.fn("bit_length", e) @@ -5050,6 +5851,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def concat_ws(sep: String, exprs: Column*): Column = @@ -5062,6 +5865,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def decode(value: Column, charset: String): Column = Column.fn("decode", value, lit(charset)) @@ -5073,6 +5878,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def encode(value: Column, charset: String): Column = Column.fn("encode", value, lit(charset)) @@ -5082,6 +5889,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_utf8(str: Column): Column = Column.fn("is_valid_utf8", str) @@ -5092,6 +5901,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def make_valid_utf8(str: Column): Column = Column.fn("make_valid_utf8", str) @@ -5102,6 +5913,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def validate_utf8(str: Column): Column = Column.fn("validate_utf8", str) @@ -5111,6 +5924,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_validate_utf8(str: Column): Column = Column.fn("try_validate_utf8", str) @@ -5124,6 +5939,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) @@ -5132,6 +5949,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def format_string(format: String, arguments: Column*): Column = @@ -5145,6 +5964,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def initcap(e: Column): Column = Column.fn("initcap", e) @@ -5158,6 +5979,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: String): Column = instr(str, lit(substring)) @@ -5171,6 +5994,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) @@ -5187,6 +6012,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int): Column = Column.fn("instr", str, substring, lit(start)) @@ -5204,6 +6031,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column): Column = Column.fn("instr", str, substring, start) @@ -5223,6 +6052,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = Column.fn("instr", str, substring, lit(start), lit(occurrence)) @@ -5242,6 +6073,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = Column.fn("instr", str, substring, start, occurrence) @@ -5253,6 +6086,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def length(e: Column): Column = Column.fn("length", e) @@ -5263,6 +6098,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def len(e: Column): Column = Column.fn("len", e) @@ -5271,6 +6108,8 @@ object functions { * * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def lower(e: Column): Column = Column.fn("lower", e) @@ -5278,7 +6117,7 @@ object functions { * Computes the Levenshtein distance of the two given string columns if it's less than or equal * to a given threshold. * @return - * result distance, or -1 + * result distance, or -1. Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -5289,6 +6128,8 @@ object functions { * Computes the Levenshtein distance of the two given string columns. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) @@ -5297,6 +6138,8 @@ object functions { * double between 0.0 (no similarity) and 1.0 (identical). * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a double. */ def jaro_winkler_similarity(l: Column, r: Column): Column = Column.fn("jaro_winkler_similarity", l, r) @@ -5310,6 +6153,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) @@ -5322,6 +6167,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column, pos: Int): Column = Column.fn("locate", lit(substr), str, lit(pos)) @@ -5332,6 +6179,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) @@ -5341,6 +6190,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) @@ -5350,6 +6201,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) @@ -5358,6 +6211,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column): Column = Column.fn("ltrim", e) @@ -5365,6 +6220,8 @@ object functions { * Trim the specified character string from left end for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) @@ -5372,6 +6229,8 @@ object functions { * Trim the specified character string from left end for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) @@ -5380,6 +6239,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def octet_length(e: Column): Column = Column.fn("octet_length", e) @@ -5388,6 +6249,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) @@ -5396,6 +6259,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collation(e: Column): Column = Column.fn("collation", e) @@ -5404,6 +6269,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) @@ -5412,6 +6279,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) @@ -5420,6 +6289,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) @@ -5429,6 +6300,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) @@ -5440,6 +6313,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) @@ -5450,6 +6325,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column): Column = Column.fn("regexp_extract_all", str, regexp) @@ -5460,6 +6337,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_extract_all", str, regexp, idx) @@ -5469,6 +6348,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String): Column = regexp_replace(e, lit(pattern), lit(replacement)) @@ -5479,6 +6360,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) @@ -5488,6 +6371,8 @@ object functions { * * @group string_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = Column.fn("regexp_replace", e, pattern, replacement) @@ -5498,6 +6383,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = Column.fn("regexp_replace", e, pattern, replacement, pos) @@ -5508,6 +6395,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) @@ -5518,6 +6407,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) @@ -5528,6 +6419,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_instr", str, regexp, idx) @@ -5538,6 +6431,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unbase64(e: Column): Column = Column.fn("unbase64", e) @@ -5547,6 +6442,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) @@ -5556,6 +6453,8 @@ object functions { * * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) @@ -5565,6 +6464,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) @@ -5573,6 +6474,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) @@ -5581,6 +6484,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) @@ -5589,6 +6494,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column): Column = Column.fn("rtrim", e) @@ -5596,6 +6503,8 @@ object functions { * Trim the specified character string from right end for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) @@ -5603,6 +6512,8 @@ object functions { * Trim the specified character string from right end for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) @@ -5611,6 +6522,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def soundex(e: Column): Column = Column.fn("soundex", e) @@ -5625,6 +6538,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) @@ -5639,6 +6554,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) @@ -5659,6 +6576,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String, limit: Int): Column = Column.fn("split", str, lit(pattern), lit(limit)) @@ -5680,6 +6599,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column, limit: Column): Column = Column.fn("split", str, pattern, limit) @@ -5693,6 +6614,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Int, len: Int): Column = Column.fn("substring", str, lit(pos), lit(len)) @@ -5706,6 +6629,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Column, len: Column): Column = Column.fn("substring", str, pos, len) @@ -5717,6 +6642,8 @@ object functions { * right) is returned. substring_index performs a case-sensitive match when searching for delim. * * @group string_funcs + * @return + * Returns a column that evaluates to a string. */ def substring_index(str: Column, delim: String, count: Int): Column = Column.fn("substring_index", str, lit(delim), lit(count)) @@ -5727,6 +6654,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the input. */ def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = Column.fn("overlay", src, replace, pos, len) @@ -5737,6 +6666,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def overlay(src: Column, replace: Column, pos: Column): Column = Column.fn("overlay", src, replace, pos) @@ -5745,6 +6676,8 @@ object functions { * Splits a string into arrays of sentences, where each sentence is an array of words. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column, country: Column): Column = Column.fn("sentences", string, language, country) @@ -5754,6 +6687,8 @@ object functions { * default `country`('') is used. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column): Column = Column.fn("sentences", string, language) @@ -5763,6 +6698,8 @@ object functions { * default locale is used. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column): Column = Column.fn("sentences", string) @@ -5773,6 +6710,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def translate(src: Column, matchingString: String, replaceString: String): Column = Column.fn("translate", src, lit(matchingString), lit(replaceString)) @@ -5782,6 +6721,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column): Column = Column.fn("trim", e) @@ -5789,6 +6730,8 @@ object functions { * Trim the specified character from both ends for the specified string column. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) @@ -5796,6 +6739,8 @@ object functions { * Trim the specified character from both ends for the specified string column. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) @@ -5804,6 +6749,8 @@ object functions { * * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def upper(e: Column): Column = Column.fn("upper", e) @@ -5815,6 +6762,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) @@ -5824,6 +6773,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column): Column = Column.fn("to_binary", e) @@ -5853,6 +6804,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) @@ -5883,6 +6836,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) @@ -5906,6 +6861,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) @@ -5913,15 +6870,18 @@ object functions { * Replaces all occurrences of `search` with `replace`. * * @param src - * A column of string to be replaced + * A column of string to be replaced. A column that evaluates to a string. * @param search - * A column of string, If `search` is not found in `str`, `str` is returned unchanged. + * A column of string, If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. * @param replace * A column of string, If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. + * the string that is removed from `str`. A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column, replace: Column): Column = Column.fn("replace", src, search, replace) @@ -5936,6 +6896,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) @@ -5947,6 +6909,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def split_part(str: Column, delimiter: Column, partNum: Column): Column = Column.fn("split_part", str, delimiter, partNum) @@ -5957,6 +6921,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column, len: Column): Column = Column.fn("substr", str, pos, len) @@ -5967,6 +6933,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) @@ -5975,6 +6943,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("try_parse_url", url, partToExtract, key) @@ -5984,6 +6954,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column): Column = Column.fn("try_parse_url", url, partToExtract) @@ -5993,6 +6965,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("parse_url", url, partToExtract, key) @@ -6002,6 +6976,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column): Column = Column.fn("parse_url", url, partToExtract) @@ -6011,6 +6987,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def printf(format: Column, arguments: Column*): Column = @@ -6022,6 +7000,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_decode(str: Column): Column = Column.fn("url_decode", str) @@ -6031,6 +7011,8 @@ object functions { * * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) @@ -6040,6 +7022,8 @@ object functions { * * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_encode(str: Column): Column = Column.fn("url_encode", str) @@ -6049,6 +7033,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column, start: Column): Column = Column.fn("position", substr, str, start) @@ -6059,6 +7045,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column): Column = Column.fn("position", substr, str) @@ -6070,6 +7058,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def endswith(str: Column, suffix: Column): Column = Column.fn("endswith", str, suffix) @@ -6081,6 +7071,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def startswith(str: Column, prefix: Column): Column = Column.fn("startswith", str, prefix) @@ -6091,6 +7083,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def char(n: Column): Column = Column.fn("char", n) @@ -6099,6 +7093,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column): Column = Column.fn("btrim", str) @@ -6107,6 +7103,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) @@ -6116,6 +7114,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) @@ -6125,6 +7125,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) @@ -6135,6 +7137,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) @@ -6144,6 +7148,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def char_length(str: Column): Column = Column.fn("char_length", str) @@ -6153,6 +7159,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def character_length(str: Column): Column = Column.fn("character_length", str) @@ -6162,6 +7170,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def chr(n: Column): Column = Column.fn("chr", n) @@ -6172,6 +7182,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) @@ -6183,6 +7195,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) @@ -6194,6 +7208,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) @@ -6203,6 +7219,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("like", str, pattern, escapeChar) @@ -6213,6 +7231,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) @@ -6222,6 +7242,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("ilike", str, pattern, escapeChar) @@ -6232,6 +7254,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) @@ -6240,6 +7264,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def lcase(str: Column): Column = Column.fn("lcase", str) @@ -6248,6 +7274,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def ucase(str: Column): Column = Column.fn("ucase", str) @@ -6257,6 +7285,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def left(str: Column, len: Column): Column = Column.fn("left", str, len) @@ -6266,6 +7296,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def right(str: Column, len: Column): Column = Column.fn("right", str, len) @@ -6275,6 +7307,8 @@ object functions { * * @group string_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def quote(str: Column): Column = Column.fn("quote", str) @@ -6288,6 +7322,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) @@ -6297,6 +7333,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(columnName: String): Column = { hll_sketch_estimate(Column(columnName)) @@ -6308,6 +7346,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column): Column = Column.fn("hll_union", c1, c2) @@ -6318,6 +7358,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(columnName1: String, columnName2: String): Column = { hll_union(Column(columnName1), Column(columnName2)) @@ -6330,6 +7372,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) @@ -6341,6 +7385,8 @@ object functions { * * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union( columnName1: String, @@ -6355,6 +7401,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(c1: Column, c2: Column): Column = Column.fn("theta_difference", c1, c2) @@ -6365,6 +7413,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(columnName1: String, columnName2: String): Column = { theta_difference(Column(columnName1), Column(columnName2)) @@ -6376,6 +7426,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(c1: Column, c2: Column): Column = Column.fn("theta_intersection", c1, c2) @@ -6386,6 +7438,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(columnName1: String, columnName2: String): Column = { theta_intersection(Column(columnName1), Column(columnName2)) @@ -6397,6 +7451,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) @@ -6406,6 +7462,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(columnName: String): Column = { theta_sketch_estimate(Column(columnName)) @@ -6418,6 +7476,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column): Column = Column.fn("theta_union", c1, c2) @@ -6429,6 +7489,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String): Column = { theta_union(Column(columnName1), Column(columnName2)) @@ -6441,6 +7503,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("theta_union", c1, c2, lit(lgNomEntries)) @@ -6452,6 +7516,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { theta_union(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6464,6 +7530,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = Column.fn("theta_union", c1, c2, lgNomEntries) @@ -6475,6 +7543,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_double", c1, c2) @@ -6486,6 +7556,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(columnName1: String, columnName2: String): Column = tuple_difference_double(Column(columnName1), Column(columnName2)) @@ -6497,6 +7569,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_integer", c1, c2) @@ -6508,6 +7582,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(columnName1: String, columnName2: String): Column = tuple_difference_integer(Column(columnName1), Column(columnName2)) @@ -6520,6 +7596,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_double", c1, c2) @@ -6532,6 +7610,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2)) @@ -6544,6 +7624,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_double", c1, c2, lit(mode)) @@ -6556,6 +7638,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2), mode) @@ -6568,6 +7652,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_double", c1, c2, mode) @@ -6580,6 +7666,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_integer", c1, c2) @@ -6592,6 +7680,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2)) @@ -6604,6 +7694,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) @@ -6616,6 +7708,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) @@ -6628,6 +7722,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_integer", c1, c2, mode) @@ -6638,6 +7734,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(c: Column): Column = Column.fn("tuple_sketch_estimate_double", c) @@ -6648,6 +7746,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(columnName: String): Column = tuple_sketch_estimate_double(Column(columnName)) @@ -6658,6 +7758,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(c: Column): Column = Column.fn("tuple_sketch_estimate_integer", c) @@ -6668,6 +7770,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(columnName: String): Column = tuple_sketch_estimate_integer(Column(columnName)) @@ -6679,6 +7783,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column): Column = Column.fn("tuple_sketch_summary_double", c) @@ -6690,6 +7796,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String): Column = tuple_sketch_summary_double(Column(columnName)) @@ -6700,6 +7808,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_double", c, lit(mode)) @@ -6710,6 +7820,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String, mode: String): Column = tuple_sketch_summary_double(Column(columnName), mode) @@ -6720,6 +7832,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_double", c, mode) @@ -6731,6 +7845,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column): Column = Column.fn("tuple_sketch_summary_integer", c) @@ -6742,6 +7858,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String): Column = tuple_sketch_summary_integer(Column(columnName)) @@ -6752,6 +7870,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_integer", c, lit(mode)) @@ -6762,6 +7882,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String, mode: String): Column = tuple_sketch_summary_integer(Column(columnName), mode) @@ -6772,6 +7894,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_integer", c, mode) @@ -6783,6 +7907,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(c: Column): Column = Column.fn("tuple_sketch_theta_double", c) @@ -6794,6 +7920,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(columnName: String): Column = tuple_sketch_theta_double(Column(columnName)) @@ -6805,6 +7933,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(c: Column): Column = Column.fn("tuple_sketch_theta_integer", c) @@ -6816,6 +7946,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(columnName: String): Column = tuple_sketch_theta_integer(Column(columnName)) @@ -6827,6 +7959,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_double", c1, c2) @@ -6838,6 +7972,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String): Column = tuple_union_double(Column(columnName1), Column(columnName2)) @@ -6850,6 +7986,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) @@ -6862,6 +8000,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6874,6 +8014,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6886,6 +8028,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double( columnName1: String, @@ -6902,6 +8046,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) @@ -6913,6 +8059,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_integer", c1, c2) @@ -6924,6 +8072,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String): Column = tuple_union_integer(Column(columnName1), Column(columnName2)) @@ -6936,6 +8086,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) @@ -6948,6 +8100,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6960,6 +8114,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6972,6 +8128,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer( columnName1: String, @@ -6988,6 +8146,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) @@ -6999,6 +8159,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_double", c1, c2) @@ -7010,6 +8172,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = tuple_difference_theta_double(Column(columnName1), Column(columnName2)) @@ -7021,6 +8185,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_integer", c1, c2) @@ -7032,6 +8198,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) @@ -7044,6 +8212,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2) @@ -7056,6 +8226,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) @@ -7068,6 +8240,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) @@ -7080,6 +8254,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double( columnName1: String, @@ -7095,6 +8271,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2, mode) @@ -7107,6 +8285,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2) @@ -7119,6 +8299,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) @@ -7131,6 +8313,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) @@ -7143,6 +8327,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer( columnName1: String, @@ -7158,6 +8344,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, mode) @@ -7169,6 +8357,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_double", c1, c2) @@ -7180,6 +8370,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(columnName1: String, columnName2: String): Column = tuple_union_theta_double(Column(columnName1), Column(columnName2)) @@ -7192,6 +8384,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) @@ -7204,6 +8398,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7219,6 +8415,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7231,6 +8429,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7247,6 +8447,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( c1: Column, @@ -7262,6 +8464,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_integer", c1, c2) @@ -7273,6 +8477,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = tuple_union_theta_integer(Column(columnName1), Column(columnName2)) @@ -7285,6 +8491,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) @@ -7297,6 +8505,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7312,6 +8522,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7324,6 +8536,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7340,6 +8554,8 @@ object functions { * * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( c1: Column, @@ -7353,6 +8569,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_bigint(e: Column): Column = Column.fn("kll_sketch_to_string_bigint", e) @@ -7362,6 +8580,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_float(e: Column): Column = Column.fn("kll_sketch_to_string_float", e) @@ -7371,6 +8591,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_double(e: Column): Column = Column.fn("kll_sketch_to_string_double", e) @@ -7380,6 +8602,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_bigint(e: Column): Column = Column.fn("kll_sketch_get_n_bigint", e) @@ -7389,6 +8613,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_float(e: Column): Column = Column.fn("kll_sketch_get_n_float", e) @@ -7398,6 +8624,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_double(e: Column): Column = Column.fn("kll_sketch_get_n_double", e) @@ -7407,6 +8635,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_bigint(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_bigint", left, right) @@ -7416,6 +8646,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_float(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_float", left, right) @@ -7425,6 +8657,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_double(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_double", left, right) @@ -7435,6 +8669,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) @@ -7445,6 +8681,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a float. */ def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_float", sketch, rank) @@ -7455,6 +8693,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_double", sketch, rank) @@ -7465,6 +8705,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) @@ -7475,6 +8717,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_float", sketch, quantile) @@ -7485,6 +8729,8 @@ object functions { * * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_double", sketch, quantile) @@ -7498,11 +8744,14 @@ object functions { * * @param startDate * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7518,7 +8767,8 @@ object functions { * @param numMonths * A column of the number of months to add to `startDate`, can be negative to subtract months * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7531,6 +8781,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def curdate(): Column = Column.fn("curdate") @@ -7540,6 +8792,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def current_date(): Column = Column.fn("current_date") @@ -7548,6 +8802,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_timezone(): Column = Column.fn("current_timezone") @@ -7557,6 +8813,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def current_timestamp(): Column = Column.fn("current_timestamp") @@ -7565,6 +8823,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def now(): Column = Column.fn("now") @@ -7575,6 +8835,8 @@ object functions { * * @group datetime_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def localtimestamp(): Column = Column.fn("localtimestamp") @@ -7587,11 +8849,14 @@ object functions { * * @param dateExpr * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993` + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns + * a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a * specialized implementation. @@ -7608,11 +8873,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to add to `start`, can be negative to subtract days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7627,7 +8895,8 @@ object functions { * @param days * A column of the number of days to add to `start`, can be negative to subtract days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7638,11 +8907,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * A column of the number of days to add to `start`, can be negative to subtract days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.5.0 */ @@ -7653,11 +8925,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to subtract from `start`, can be negative to add days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7672,7 +8947,8 @@ object functions { * @param days * A column of the number of days to subtract from `start`, can be negative to add days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7690,13 +8966,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7713,13 +8991,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7730,13 +9010,16 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) /** * Extracts the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7745,7 +9028,8 @@ object functions { /** * Extracts the quarter as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7754,7 +9038,8 @@ object functions { /** * Extracts the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7764,7 +9049,8 @@ object functions { * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 * for a Sunday through to 7 for a Saturday * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 */ @@ -7773,7 +9059,8 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7782,7 +9069,8 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7791,7 +9079,8 @@ object functions { /** * Extracts the day of the year as an integer from a given date/timestamp/string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7802,7 +9091,8 @@ object functions { * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7816,7 +9106,8 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -7833,7 +9124,8 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -7850,7 +9142,8 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 3.5.0 */ @@ -7864,9 +9157,11 @@ object functions { * * @param e * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * A date, or null if the input was a string that could not be cast to a date + * A date, or null if the input was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7877,7 +9172,8 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7888,12 +9184,14 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def weekday(e: Column): Column = Column.fn("weekday", e) /** * @return - * A date created from year, month and day fields. + * A date created from year, month and day fields. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.3.0 */ @@ -7922,7 +9220,7 @@ object functions { * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` * @return * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start` + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a double. * @group datetime_funcs * @since 1.5.0 */ @@ -7934,6 +9232,8 @@ object functions { * result is rounded off to 8 digits; it is not rounded otherwise. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a double. */ def months_between(end: Column, start: Column, roundOff: Boolean): Column = Column.fn("months_between", end, start, lit(roundOff)) @@ -7947,12 +9247,14 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7973,7 +9275,7 @@ object functions { * "Fri", "Sat", "Sun" * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.2.0 */ @@ -7985,7 +9287,8 @@ object functions { * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @return - * An integer, or null if the input was a string that could not be cast to a timestamp + * An integer, or null if the input was a string that could not be cast to a timestamp. Returns + * a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7998,7 +9301,8 @@ object functions { * as defined by ISO 8601 * * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8013,7 +9317,8 @@ object functions { * A number of a type that is castable to a long, such as string or integer. Can be negative * for timestamps before the unix epoch * @return - * A string, or null if the input was a string that could not be cast to a long + * A string, or null if the input was a string that could not be cast to a long. Returns a + * column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8034,7 +9339,7 @@ object functions { * A date time pattern that the input will be formatted to * @return * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern + * invalid date time pattern. Returns a column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8050,6 +9355,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_timestamp(): Column = unix_timestamp(current_timestamp()) @@ -8061,7 +9368,8 @@ object functions { * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` * format * @return - * A long, or null if the input was a string not of the correct format + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8080,7 +9388,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format + * format. Returns a column that evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8093,7 +9401,8 @@ object functions { * @param str * A string to be parsed to time. * @return - * A time, or raises an error if the input is malformed. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * * @group datetime_funcs * @since 4.1.0 @@ -8113,8 +9422,8 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or raises an error if the input is malformed. - * + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * @group datetime_funcs * @since 4.1.0 */ @@ -8127,9 +9436,11 @@ object functions { * * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp + * A timestamp, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8148,7 +9459,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format + * an invalid format. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8160,7 +9471,7 @@ object functions { * @param str * A string to be parsed to time. * @return - * A time, or null if the input is malformed. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * * @group datetime_funcs * @since 4.1.0 @@ -8180,8 +9491,7 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or null if the input is malformed. - * + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -8196,6 +9506,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column, format: Column): Column = Column.fn("try_to_timestamp", s, format) @@ -8207,6 +9519,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) @@ -8215,6 +9529,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def to_date(e: Column): Column = Column.fn("to_date", e) @@ -8231,7 +9547,7 @@ object functions { * A date time pattern detailing the format of `e` when `e`is a string * @return * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format + * invalid format. Returns a column that evaluates to a date. * @group datetime_funcs * @since 2.2.0 */ @@ -8243,6 +9559,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column): Column = Column.fn("try_to_date", e) @@ -8252,6 +9570,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) @@ -8260,6 +9580,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def unix_date(e: Column): Column = Column.fn("unix_date", e) @@ -8268,6 +9590,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_micros(e: Column): Column = Column.fn("unix_micros", e) @@ -8278,6 +9602,8 @@ object functions { * * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a decimal. */ def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) @@ -8287,6 +9613,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_millis(e: Column): Column = Column.fn("unix_millis", e) @@ -8296,6 +9624,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) @@ -8306,14 +9636,15 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param format: * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter' + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value + * invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8327,13 +9658,14 @@ object functions { * @param format: * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter' + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. * @param timestamp * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value + * `format` was an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8347,16 +9679,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8368,6 +9702,8 @@ object functions { * 03:40:00.0'. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def from_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("from_utc_timestamp", ts, tz) @@ -8379,16 +9715,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8400,6 +9738,8 @@ object functions { * '2017-07-14 01:40:00.0'. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) @@ -8447,6 +9787,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window( timeColumn: Column, @@ -8496,6 +9838,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { window(timeColumn, windowDuration, slideDuration, "0 second") @@ -8534,6 +9878,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String): Column = { window(timeColumn, windowDuration, windowDuration, "0 second") @@ -8552,6 +9898,8 @@ object functions { * * @group datetime_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) @@ -8578,6 +9926,8 @@ object functions { * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: String): Column = session_window(timeColumn, lit(gapDuration)) @@ -8614,6 +9964,8 @@ object functions { * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: Column): Column = Column.fn("session_window", timeColumn, gapDuration) @@ -8622,6 +9974,8 @@ object functions { * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. * @group datetime_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) @@ -8630,6 +9984,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) @@ -8638,6 +9994,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) @@ -8647,6 +10005,8 @@ object functions { * * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) @@ -8656,6 +10016,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a long. */ def timestamp_diff(unit: String, start: Column, end: Column): Column = Column.internalFn("timestampdiff", lit(unit), start, end) @@ -8665,6 +10027,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def timestamp_add(unit: String, quantity: Column, ts: Column): Column = Column.internalFn("timestampadd", lit(unit), quantity, ts) @@ -8681,6 +10045,8 @@ object functions { * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column): Column = Column.fn("time_bucket", bucketSize, ts) @@ -8699,6 +10065,8 @@ object functions { * Alignment anchor. Must be the same type as `ts` and must be foldable. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = Column.fn("time_bucket", bucketSize, ts, origin) @@ -8709,13 +10077,15 @@ object functions { * * @param unit * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param start - * A starting TIME. + * A starting TIME. A column that evaluates to a time. * @param end - * An ending TIME. + * An ending TIME. A column that evaluates to a time. * @return - * The difference between `end` and `start` times, measured in specified units. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @group datetime_funcs @@ -8730,11 +10100,12 @@ object functions { * * @param unit * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param time - * A TIME to truncate. + * A TIME to truncate. A column that evaluates to a time. * @return - * A TIME truncated to the specified unit. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @throws IllegalArgumentException @@ -8751,6 +10122,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) @@ -8759,6 +10132,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) @@ -8767,6 +10142,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) @@ -8776,6 +10153,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a decimal. */ def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) @@ -8784,6 +10163,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) @@ -8792,6 +10173,8 @@ object functions { * * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) @@ -8801,6 +10184,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ltz", timestamp, format) @@ -8811,6 +10196,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column): Column = Column.fn("to_timestamp_ltz", timestamp) @@ -8821,6 +10208,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ntz", timestamp, format) @@ -8831,6 +10220,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column): Column = Column.fn("to_timestamp_ntz", timestamp) @@ -8840,6 +10231,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column, format: Column): Column = Column.fn("to_unix_timestamp", timeExp, format) @@ -8849,6 +10242,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column): Column = Column.fn("to_unix_timestamp", timeExp) @@ -8858,6 +10253,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def monthname(timeExp: Column): Column = Column.fn("monthname", timeExp) @@ -8867,6 +10264,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def dayname(timeExp: Column): Column = Column.fn("dayname", timeExp) @@ -8879,6 +10278,8 @@ object functions { * Returns null if the array is null, true if the array contains `value`, and false otherwise. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def array_contains(column: Column, value: Any): Column = Column.fn("array_contains", column, lit(value)) @@ -8889,6 +10290,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_append(column: Column, element: Any): Column = Column.fn("array_append", column, lit(element)) @@ -8899,6 +10302,8 @@ object functions { * `false` otherwise. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a boolean. */ def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) @@ -8907,14 +10312,16 @@ object functions { * end if `start` is negative) with the specified `length`. * * @param x - * the array column to be sliced + * the array column to be sliced. A column that evaluates to an array. * @param start - * the starting index + * the starting index. A column that evaluates to an integer. * @param length - * the length of the slice + * the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Int, length: Int): Column = slice(x, lit(start), lit(length)) @@ -8932,6 +10339,8 @@ object functions { * * @group array_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Column, length: Column): Column = Column.fn("slice", x, start, length) @@ -8941,6 +10350,8 @@ object functions { * `nullReplacement`. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String, nullReplacement: String): Column = Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) @@ -8949,6 +10360,8 @@ object functions { * Concatenates the elements of `column` using the `delimiter`. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String): Column = Column.fn("array_join", column, lit(delimiter)) @@ -8962,6 +10375,8 @@ object functions { * * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) @@ -8976,6 +10391,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a long. */ def array_position(column: Column, value: Any): Column = Column.fn("array_position", column, lit(value)) @@ -8986,6 +10403,8 @@ object functions { * * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input map. */ def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) @@ -8999,6 +10418,8 @@ object functions { * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input map. */ def try_element_at(column: Column, value: Column): Column = Column.fn("try_element_at", column, value) @@ -9009,6 +10430,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column of the element type of the input array. */ def get(column: Column, index: Column): Column = Column.fn("get", column, index) @@ -9023,6 +10446,8 @@ object functions { * * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column): Column = Column.fn("array_sort", e) @@ -9034,6 +10459,8 @@ object functions { * * @group collection_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column, comparator: (Column, Column) => Column): Column = Column.fn("array_sort", e, createLambda(comparator)) @@ -9043,6 +10470,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_remove(column: Column, element: Any): Column = Column.fn("array_remove", column, lit(element)) @@ -9052,6 +10481,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_compact(column: Column): Column = Column.fn("array_compact", column) @@ -9061,6 +10492,8 @@ object functions { * * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_prepend(column: Column, element: Any): Column = Column.fn("array_prepend", column, lit(element)) @@ -9069,6 +10502,8 @@ object functions { * Removes duplicate values from the array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_distinct(e: Column): Column = Column.fn("array_distinct", e) @@ -9078,6 +10513,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_intersect(col1: Column, col2: Column): Column = Column.fn("array_intersect", col1, col2) @@ -9087,6 +10524,8 @@ object functions { * * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_insert(arr: Column, pos: Column, value: Column): Column = Column.fn("array_insert", arr, pos, value) @@ -9096,6 +10535,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_union(col1: Column, col2: Column): Column = Column.fn("array_union", col1, col2) @@ -9106,6 +10547,8 @@ object functions { * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_except(col1: Column, col2: Column): Column = Column.fn("array_except", col1, col2) @@ -9139,12 +10582,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => transformed_col, the lambda function to transform the input column + * col => transformed_col, the lambda function to transform the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: Column => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9164,6 +10609,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: (Column, Column) => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9175,12 +10622,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def exists(column: Column, f: Column => Column): Column = Column.fn("exists", column, createLambda(f)) @@ -9192,12 +10641,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def forall(column: Column, f: Column => Column): Column = Column.fn("forall", column, createLambda(f)) @@ -9209,12 +10660,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to filter the input column + * col => predicate, the Boolean predicate to filter the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: Column => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9233,6 +10686,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: (Column, Column) => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9246,18 +10701,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate( expr: Column, @@ -9282,6 +10739,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = aggregate(expr, initialValue, merge, c => c) @@ -9295,18 +10754,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce( expr: Column, @@ -9331,6 +10792,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = reduce(expr, initialValue, merge, c => c) @@ -9344,14 +10807,16 @@ object functions { * }}} * * @param left - * the left input array column + * the left input array column. A column that evaluates to an array. * @param right - * the right input array column + * the right input array column. A column that evaluates to an array. * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = Column.fn("zip_with", left, right, createLambda(f)) @@ -9370,6 +10835,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_keys(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_keys", expr, createLambda(f)) @@ -9388,6 +10855,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_values(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_values", expr, createLambda(f)) @@ -9405,6 +10874,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_filter(expr: Column, f: (Column, Column) => Column): Column = Column.fn("map_filter", expr, createLambda(f)) @@ -9424,6 +10895,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = Column.fn("map_zip_with", left, right, createLambda(f)) @@ -9435,6 +10908,9 @@ object functions { * * @group generator_funcs * @since 1.3.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of the + * input map. */ def explode(e: Column): Column = Column.fn("explode", e) @@ -9445,6 +10921,9 @@ object functions { * * @group generator_funcs * @since 2.2.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of the + * input map. */ def explode_outer(e: Column): Column = Column.fn("explode_outer", e) @@ -9474,6 +10953,8 @@ object functions { * * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline(e: Column): Column = Column.fn("inline", e) @@ -9483,6 +10964,8 @@ object functions { * * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline_outer(e: Column): Column = Column.fn("inline_outer", e) @@ -9492,6 +10975,8 @@ object functions { * * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def get_json_object(e: Column, path: String): Column = Column.fn("get_json_object", e, lit(path)) @@ -9501,6 +10986,8 @@ object functions { * * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def json_tuple(json: Column, fields: String*): Column = { @@ -9525,6 +11012,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -9548,6 +11037,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { @@ -9571,6 +11062,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -9594,6 +11087,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { @@ -9611,6 +11106,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: StructType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9627,6 +11124,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: DataType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9649,6 +11148,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -9673,6 +11174,8 @@ object functions { * * @group json_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: Map[String, String]): Column = { @@ -9691,6 +11194,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: Column): Column = { from_json(e, schema, Map.empty[String, String].asJava) @@ -9714,6 +11219,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { @@ -9736,6 +11243,8 @@ object functions { * * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) @@ -9743,9 +11252,11 @@ object functions { * Parses a JSON string and constructs a Variant value. * * @param json - * a string column that contains JSON data. + * a string column that contains JSON data. A column that evaluates to a string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def parse_json(json: Column): Column = Column.fn("parse_json", json) @@ -9758,6 +11269,8 @@ object functions { * a column with a nested schema or column name. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) @@ -9769,6 +11282,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) @@ -9780,6 +11295,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) @@ -9797,6 +11314,8 @@ object functions { * additional JSONPath arguments, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: Column, paths: Column*): Column = @@ -9816,6 +11335,8 @@ object functions { * additional JSONPath strings, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: String, paths: String*): Column = @@ -9836,6 +11357,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: Column, value: Column): Column = Column.fn("variant_insert", v, path, value) @@ -9855,6 +11378,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: String, value: Column): Column = Column.fn("variant_insert", v, lit(path), value) @@ -9872,6 +11397,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: String, targetType: String): Column = Column.fn("variant_get", v, lit(path), lit(targetType)) @@ -9890,6 +11417,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("variant_get", v, path, lit(targetType)) @@ -9907,6 +11436,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: String, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -9925,6 +11456,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -9936,6 +11469,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) @@ -9946,6 +11481,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) @@ -9957,6 +11494,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: String): Column = schema_of_json(lit(json)) @@ -9968,6 +11507,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) @@ -9983,7 +11524,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-json.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * * @group json_funcs * @since 3.0.0 @@ -9998,6 +11540,8 @@ object functions { * * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def json_array_length(e: Column): Column = Column.fn("json_array_length", e) @@ -10008,6 +11552,8 @@ object functions { * * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) @@ -10018,7 +11564,7 @@ object functions { * type. * * @param e - * a column containing a struct, an array or a map. + * a column containing a struct, an array, a map, or a variant. * @param options * options to control how the struct column is converted into a json string. accepts the same * options and the json data source. See Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. - * + * a column with string literal containing schema in DDL format. Returns a column that evaluates + * to a string. * @group csv_funcs * @since 3.0.0 */ @@ -10474,6 +12090,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_csv(e: Column, options: java.util.Map[String, String]): Column = @@ -10488,6 +12106,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) @@ -10507,6 +12127,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -10528,6 +12150,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -10545,6 +12169,8 @@ object functions { * the schema to use when parsing the XML string * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column): Column = { @@ -10567,6 +12193,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -10583,6 +12211,8 @@ object functions { * * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ def from_xml(e: Column, schema: StructType): Column = from_xml(e, schema, Map.empty[String, String].asJava) @@ -10598,6 +12228,8 @@ object functions { * a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) @@ -10608,6 +12240,8 @@ object functions { * a foldable string column containing a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) @@ -10624,7 +12258,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-xml.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that evaluates + * to a string. * @group xml_funcs * @since 4.0.0 */ @@ -10647,6 +12282,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_xml(e: Column, options: java.util.Map[String, String]): Column = @@ -10660,6 +12297,8 @@ object functions { * a column containing a struct. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) @@ -10692,6 +12331,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def xpath(xml: Column, path: Column): Column = Column.fn("xpath", xml, path) @@ -10701,6 +12342,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def xpath_boolean(xml: Column, path: Column): Column = Column.fn("xpath_boolean", xml, path) @@ -10711,6 +12354,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_double(xml: Column, path: Column): Column = Column.fn("xpath_double", xml, path) @@ -10721,6 +12366,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_number(xml: Column, path: Column): Column = Column.fn("xpath_number", xml, path) @@ -10731,6 +12378,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a float. */ def xpath_float(xml: Column, path: Column): Column = Column.fn("xpath_float", xml, path) @@ -10741,6 +12390,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def xpath_int(xml: Column, path: Column): Column = Column.fn("xpath_int", xml, path) @@ -10751,6 +12402,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def xpath_long(xml: Column, path: Column): Column = Column.fn("xpath_long", xml, path) @@ -10761,6 +12414,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a short. */ def xpath_short(xml: Column, path: Column): Column = Column.fn("xpath_short", xml, path) @@ -10770,6 +12425,8 @@ object functions { * * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def xpath_string(xml: Column, path: Column): Column = Column.fn("xpath_string", xml, path) @@ -10788,13 +12445,16 @@ object functions { * * @param sourceTz * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. + * used as the source time zone. A column that evaluates to a string. * @param targetTz - * the time zone to which the input timestamp should be converted. + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. * @param sourceTs - * a timestamp without time zone. + * a timestamp without time zone. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) @@ -10808,6 +12468,8 @@ object functions { * a timestamp without time zone. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", targetTz, sourceTs) @@ -10817,6 +12479,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = Column.fn("make_dt_interval", days, hours, mins, secs) @@ -10826,6 +12490,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column): Column = Column.fn("make_dt_interval", days, hours, mins) @@ -10835,6 +12501,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column): Column = Column.fn("make_dt_interval", days, hours) @@ -10844,6 +12512,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column): Column = Column.fn("make_dt_interval", days) @@ -10853,6 +12523,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(): Column = Column.fn("make_dt_interval") @@ -10863,6 +12535,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10879,6 +12553,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10896,6 +12572,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10911,6 +12589,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10927,6 +12607,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10941,6 +12623,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10956,6 +12640,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("try_make_interval", years, months, weeks, days) @@ -10965,6 +12651,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("make_interval", years, months, weeks, days) @@ -10975,6 +12663,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("try_make_interval", years, months, weeks) @@ -10984,6 +12674,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("make_interval", years, months, weeks) @@ -10994,6 +12686,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column): Column = Column.fn("try_make_interval", years, months) @@ -11003,6 +12697,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column): Column = Column.fn("make_interval", years, months) @@ -11013,6 +12709,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column): Column = Column.fn("try_make_interval", years) @@ -11022,6 +12720,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column): Column = Column.fn("make_interval", years) @@ -11031,6 +12731,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(): Column = Column.fn("make_interval") @@ -11043,6 +12745,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11062,6 +12766,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11077,6 +12783,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("make_timestamp", date, time, timezone) @@ -11086,6 +12794,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column): Column = Column.fn("make_timestamp", date, time) @@ -11097,6 +12807,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11115,6 +12827,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11130,6 +12844,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("try_make_timestamp", date, time, timezone) @@ -11139,6 +12855,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column): Column = Column.fn("try_make_timestamp", date, time) @@ -11150,6 +12868,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11168,6 +12888,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11184,6 +12906,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11201,6 +12925,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11218,6 +12944,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz( years: Column, @@ -11233,6 +12961,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("make_timestamp_ntz", date, time) @@ -11243,6 +12973,8 @@ object functions { * * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz( years: Column, @@ -11258,6 +12990,8 @@ object functions { * * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("try_make_timestamp_ntz", date, time) @@ -11267,6 +13001,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column, months: Column): Column = Column.fn("make_ym_interval", years, months) @@ -11276,6 +13012,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) @@ -11284,6 +13022,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(): Column = Column.fn("make_ym_interval") @@ -11312,6 +13052,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) @@ -11320,6 +13062,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnotnull(col: Column): Column = Column.fn("isnotnull", col) @@ -11329,6 +13073,8 @@ object functions { * * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) @@ -11337,6 +13083,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) @@ -11345,6 +13093,8 @@ object functions { * * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def nullifzero(col: Column): Column = Column.fn("nullifzero", col) @@ -11353,6 +13103,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) @@ -11361,6 +13113,8 @@ object functions { * * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) @@ -11369,6 +13123,8 @@ object functions { * * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) @@ -11425,6 +13181,8 @@ object functions { * * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column): Column = Column.fn("st_asbinary", geo) @@ -11434,6 +13192,8 @@ object functions { * * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: Column): Column = Column.fn("st_asbinary", geo, endianness) @@ -11443,6 +13203,8 @@ object functions { * * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: String): Column = Column.fn("st_asbinary", geo, lit(endianness)) @@ -11506,6 +13268,8 @@ object functions { * * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to an integer. */ def st_srid(geo: Column): Column = Column.fn("st_srid", geo) @@ -12093,6 +13857,8 @@ object functions { * Returns the cosine similarity between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_cosine_similarity(left: Column, right: Column): Column = Column.fn("vector_cosine_similarity", left, right) @@ -12101,6 +13867,8 @@ object functions { * Returns the inner product (dot product) between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_inner_product(left: Column, right: Column): Column = Column.fn("vector_inner_product", left, right) @@ -12109,6 +13877,8 @@ object functions { * Returns the Euclidean (L2) distance between two float vectors. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_l2_distance(left: Column, right: Column): Column = Column.fn("vector_l2_distance", left, right) @@ -12117,6 +13887,8 @@ object functions { * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column, degree: Column): Column = Column.fn("vector_norm", vector, degree) @@ -12125,6 +13897,8 @@ object functions { * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column): Column = Column.fn("vector_norm", vector) @@ -12133,6 +13907,8 @@ object functions { * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column, degree: Column): Column = Column.fn("vector_normalize", vector, degree) @@ -12141,6 +13917,8 @@ object functions { * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column): Column = Column.fn("vector_normalize", vector) @@ -12149,6 +13927,8 @@ object functions { * Aggregate function: returns the element-wise mean of float vectors in a group. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_avg(col: Column): Column = Column.fn("vector_avg", col) @@ -12156,6 +13936,8 @@ object functions { * Aggregate function: returns the element-wise sum of float vectors in a group. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_sum(col: Column): Column = Column.fn("vector_sum", col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala index c226e48c6be5e..adaa340ffd402 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala @@ -30,7 +30,11 @@ import org.apache.spark.sql.internal.SQLConf arguments = """ Arguments: * input - An expression that is being compared with lower and upper bound. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * lower - Lower bound of the between check. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * upper - Upper bound of the between check. """, since = "1.0.0", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala index b167ff71449d0..289b102c1f6f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala @@ -58,6 +58,13 @@ case class TryEval(child: Expression) extends UnaryExpression { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sum of `expr1`and `expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `+` operator.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(1, 2); @@ -102,6 +109,13 @@ case class TryAdd(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns `dividend`/`divisor`. It always performs floating point division. Its result is always null if `expr2` is 0. " + "`dividend` must be a numeric or an interval. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a numeric or interval. + * divisor - The expression to divide by. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -141,6 +155,13 @@ case class TryDivide(left: Expression, right: Expression, replacement: Expressio @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns the remainder after `expr1`/`expr2`. " + "`dividend` must be a numeric. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a numeric. + * divisor - The expression to divide by. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -177,6 +198,13 @@ case class TryMod(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`-`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `-` operator.", + arguments = """ + Arguments: + * expr1 - The minuend to subtract from. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The subtrahend to subtract. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(2, 1); @@ -219,6 +247,13 @@ case class TrySubtract(left: Expression, right: Expression, replacement: Express @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`*`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `*` operator.", + arguments = """ + Arguments: + * expr1 - The first factor. + An expression that evaluates to a numeric or interval. + * expr2 - The second factor. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -251,6 +286,13 @@ case class TryMultiply(left: Expression, right: Expression, replacement: Express // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str[, fmt]) - This is a special version of `to_binary` that performs the same operation, but returns a NULL value instead of raising an error if the conversion cannot be performed.", + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format of the input string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala index 9fbca1629c95f..86192a955b889 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala @@ -34,6 +34,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns some value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return some value of. + An expression of any type. + * isIgnoreNull - Whether to return only non-null values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala index 7ae542f190d56..0fcd8aec16186 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala @@ -59,6 +59,15 @@ import org.apache.spark.unsafe.types.UTF8String `k` An optional INTEGER literal greater than 0. If k is not specified, it defaults to 5. `maxItemsTracked` An optional INTEGER literal greater than or equal to k and has upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression to compute the top k most frequent items of. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. + * k - The number of top items to return. + An expression that evaluates to an integer. Must be a constant. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(expr) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); @@ -488,6 +497,13 @@ object ApproxTopKAggregateBuffer { _FUNC_(expr, maxItemsTracked) - Accumulates items into a sketch. `maxItemsTracked` An optional positive INTEGER literal with upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression whose values are accumulated into the sketch. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT approx_top_k_estimate(_FUNC_(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index fe30e2ea6f3ff..85325068297a8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -145,6 +150,11 @@ case class Average( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 99fb512ea047d..eb4f2c1191523 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -147,6 +147,11 @@ abstract class CentralMomentAgg(child: Expression, nullOnDivideByZero: Boolean) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population standard deviation of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -178,6 +183,11 @@ case class StddevPop( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample standard deviation of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -210,6 +220,11 @@ case class StddevSamp( // Compute the population variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population variance of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -239,6 +254,11 @@ case class VariancePop( // Compute the sample variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample variance of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -281,6 +301,11 @@ case class RegrReplacement(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the skewness of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); @@ -312,6 +337,11 @@ case class Skewness( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the kurtosis of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index ea154b9bce88c..72f682c00b4ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -105,6 +105,13 @@ abstract class PearsonCorrelation(x: Expression, y: Expression, nullOnDivideByZe // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala index 248ade05ab1d3..f389405ec495f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala @@ -25,6 +25,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType} usage = """ _FUNC_(expr) - Returns the number of `TRUE` values for the expression. """, + arguments = """ + Arguments: + * expr - The boolean expression whose TRUE values are counted. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 8e03daee9b659..7d047aca06068 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -90,6 +90,13 @@ abstract class Covariance(val left: Expression, val right: Expression, nullOnDiv @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); @@ -119,6 +126,13 @@ case class CovPopulation( @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 9a39a6fe98796..eb39f96ed2ee9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -36,6 +36,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return the first value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the first non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 5840c783cb8be..a0f11ffcb7142 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -35,6 +35,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values""", + arguments = """ + Arguments: + * expr - The expression to return the last value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the last non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala index e2d85cafa52f2..21959a66b2b34 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType} // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - this function is used and can only be used to calculate a measure defined in a metric view.", + arguments = """ + Arguments: + * expr - The measure defined in a metric view to calculate. + An expression of any type. + """, examples = """ Examples: > SELECT dimension_col, _FUNC_(measure_col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala index 30e00ac68004b..b87c0ede1ab81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala @@ -233,6 +233,14 @@ case class Mode( _FUNC_() WITHIN GROUP (ORDER BY col) - Returns the most frequent value for the values within `col` (specified in ORDER BY clause). NULL values are ignored. If all the values are NULL, or there are 0 rows, returns NULL. When multiple values have the same greatest frequency only one value will be returned. The value will be chosen based on sort direction. Return the smallest value if sort direction is asc or the largest value if sort direction is desc from multiple values with the same frequency.""", + arguments = """ + Arguments: + * col - The column to compute the most frequent value of. + An expression of any type. + * deterministic - Whether to return a deterministic result when there are multiple most + frequent values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index d066a87fc7919..949a107928df7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); @@ -201,6 +206,11 @@ case class Sum( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala index 86a16ad389b5d..b1d9a46a237ec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala @@ -55,6 +55,11 @@ abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise AND of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise AND of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -76,6 +81,11 @@ case class BitAndAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise OR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise OR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -97,6 +107,11 @@ case class BitOrAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise XOR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise XOR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala index ae759abf8a4f5..3eba7352e3477 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala @@ -23,6 +23,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if all values of `expr` are true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (true), (true) AS tab(col); @@ -45,6 +50,11 @@ case class BoolAnd(child: Expression) extends AggregateFunction with RuntimeRepl @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if at least one value of `expr` is true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (false), (false) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index 8a370e9a9dc78..17e3bf3ffc3d4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -387,8 +387,10 @@ private[aggregate] object CollectTopK { arguments = """ Arguments: * expr - a string or binary expression to be concatenated. + An expression that evaluates to a string or binary. * delimiter - an optional string or binary foldable expression used to separate the input values. If NULL, the concatenation will be performed without a delimiter. Default is NULL. + An expression that evaluates to a string, binary, or null. Must be a constant. * key - an optional expression for ordering the input values. Multiple keys can be specified. If none are specified, the order of the rows in the result is non-deterministic. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala index 8fb1bf51319cc..952b331b7227d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala @@ -48,6 +48,13 @@ import org.apache.spark.unsafe.types.UTF8String _FUNC_(expr, lgConfigK) - Returns the HllSketch's updatable binary representation. `lgConfigK` (optional) the log-base-2 of K, with K is the number of buckets or slots for the HllSketch. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the HLL sketch. + An expression that evaluates to an integer, long, string, or binary. + * lgConfigK - The log-base-2 of K, where K is the number of buckets for the sketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT hll_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); @@ -233,7 +240,7 @@ object HllSketchAgg { // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(expr, allowDifferentLgConfigK) - Returns the estimated number of unique values. + _FUNC_(expr, allowDifferentLgConfigK) - Returns the merged HllSketch's updatable binary representation. `allowDifferentLgConfigK` (optional) Allow sketches with different lgConfigK values to be unioned (defaults to false).""", examples = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala index 80ac45d273afa..d9a517be86d06 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala @@ -54,6 +54,12 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, ByteType, DataT The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to an integral. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -337,6 +343,12 @@ case class KllSketchAggFloat( The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to a float or double. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala index 08cc7827b4779..b397b7fdb0d2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala @@ -25,6 +25,13 @@ import org.apache.spark.sql.types.{AbstractDataType, DataType, DoubleType, Numer // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the number of non-null number pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -57,6 +64,13 @@ case class RegrCount(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the independent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable to average. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -92,6 +106,13 @@ case class RegrAvgX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the dependent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable to average. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -127,6 +148,13 @@ case class RegrAvgY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the coefficient of determination for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -160,6 +188,13 @@ case class RegrR2(y: Expression, x: Expression) extends PearsonCorrelation(y, x, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -195,6 +230,13 @@ case class RegrSXX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -224,6 +266,13 @@ case class RegrSXY(y: Expression, x: Expression) extends Covariance(y, x, true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -259,6 +308,13 @@ case class RegrSYY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the slope of the linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); @@ -322,6 +378,13 @@ case class RegrSlope(left: Expression, right: Expression) extends DeclarativeAgg // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the intercept of the univariate linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala index ac351db9e4710..d382182221b72 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala @@ -331,6 +331,11 @@ case class Percentile( @ExpressionDescription( usage = "_FUNC_(col) - Returns the median of numeric, ANSI interval or TIME column `col`.", + arguments = """ + Arguments: + * col - The column to compute the median of. + An expression that evaluates to a numeric, interval, or time. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala index 0f148d03cd70b..d8cd93129ec81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala @@ -74,6 +74,14 @@ case class FinalizedSketch(sketch: CompactSketch) extends ThetaSketchState { _FUNC_(expr, lgNomEntries) - Returns the ThetaSketch compact binary representation. `lgNomEntries` (optional) is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the ThetaSketch. """, + arguments = """ + Arguments: + * expr - The expression whose distinct values are aggregated into the sketch. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + ThetaSketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT theta_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala index 8cd864c914041..4439358ac8e11 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala @@ -54,6 +54,14 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with a double type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary)); @@ -143,6 +151,14 @@ case class TupleIntersectionAggDouble( _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with an integer type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala index 7bf9240456857..bc6cb9de57759 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala @@ -478,6 +478,16 @@ abstract class TupleSketchAggBase[U, S <: UpdatableSummary[U]] `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The double value to be aggregated as the summary. + An expression that evaluates to a double. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary); @@ -510,6 +520,16 @@ object TupleSketchAggDoubleExpressionBuilder extends ExpressionBuilder { `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The integer value to be aggregated as the summary. + An expression that evaluates to an integer. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala index 0baff38674762..ff565baaa38d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala @@ -357,6 +357,13 @@ abstract class TupleUnionAggBase[S <: Summary] `child` should be a binary TupleSketch representation created with a double type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary)); @@ -386,6 +393,14 @@ object TupleUnionAggDoubleExpressionBuilder extends ExpressionBuilder { `child` should be a binary TupleSketch representation created with an integer type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 9c72f822143f6..03bb84c2623ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -36,6 +36,11 @@ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( usage = "_FUNC_(expr) - Returns the negated value of `expr`.", + arguments = """ + Arguments: + * expr - The expression to negate. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -109,6 +114,11 @@ case class UnaryMinus( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the value of `expr`.", + arguments = """ + Arguments: + * expr - The input expression whose value is returned. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -140,6 +150,11 @@ case class UnaryPositive(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the absolute value of the numeric or interval value.", + arguments = """ + Arguments: + * expr - The numeric or interval value to take the absolute value of. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(-1); @@ -388,6 +403,13 @@ object BinaryArithmetic { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`+`expr2`.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT 1 _FUNC_ 2; @@ -562,6 +584,13 @@ object Subtract { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`*`expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand. + An expression that evaluates to a numeric or interval. + * expr2 - The second operand. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT 2 _FUNC_ 3; @@ -811,6 +840,13 @@ trait DivModLike extends BinaryArithmetic { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`/`expr2`. It always performs floating point division.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric or interval. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -893,6 +929,13 @@ object Divide { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long.", + arguments = """ + Arguments: + * expr1 - The dividend to be divided. + An expression that evaluates to an integral, decimal, or interval. + * expr2 - The divisor to divide by. + An expression that evaluates to an integral, decimal, or interval. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -976,6 +1019,13 @@ object IntegralDivide { @ExpressionDescription( usage = "expr1 % expr2, or mod(expr1, expr2) - Returns the remainder after `expr1`/`expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT 2 % 1.8; @@ -1064,6 +1114,13 @@ object Remainder { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the positive value of `expr1` mod `expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(10, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala index 5a73fd830e037..1f943ed7edb08 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType, LongT @ExpressionDescription( usage = "_FUNC_(child) - Returns the bucket number for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bucket number for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(123); @@ -63,6 +68,11 @@ case class BitmapBucketNumber(child: Expression) @ExpressionDescription( usage = "_FUNC_(child) - Returns the bit position for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bit position for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -148,6 +158,11 @@ case class BitmapCount(child: Expression) _FUNC_(child) - Returns a bitmap with the positions of the bits set from all the values from the child expression. The child expression will most likely be bitmap_bit_position(). """, + arguments = """ + Arguments: + * child - The expression whose values set the bit positions in the bitmap. + An expression that evaluates to a long. + """, // scalastyle:off line.size.limit examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 5fac0a93bf9bf..7fcc150cf05c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -29,6 +29,13 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise AND. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise AND. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -78,6 +85,13 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise OR. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise OR. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -127,6 +141,13 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -174,6 +195,11 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "_FUNC_ expr - Returns the result of bitwise NOT of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise NOT of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_ 0; @@ -216,6 +242,11 @@ case class BitwiseNot(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the number of bits that are set in the argument expr as an" + " unsigned 64-bit integer, or NULL if the argument is NULL.", + arguments = """ + Arguments: + * expr - The argument whose set bits are counted. + An expression that evaluates to an integral or boolean. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -266,6 +297,13 @@ object BitwiseGetUtil { The positions are numbered from right to left, starting at zero. The position argument cannot be negative. """, + arguments = """ + Arguments: + * expr - The expression to read the bit from. + An expression that evaluates to an integral. + * pos - The position of the bit to return, numbered from right to left starting at zero. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(11, 0); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala index c3db6fca6a861..bf1bc66ca8b19 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala @@ -145,6 +145,7 @@ case class ResolvedCollation(collationName: String) extends LeafExpression { arguments = """ Arguments: * expr - String expression to perform collation on. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ab56386869a02..195113695f1ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -101,6 +101,11 @@ trait BinaryArrayExpressionWithImplicitCast spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the default settings, the function returns null for null input. """, + arguments = """ + Arguments: + * expr - The array or map whose size is returned. + An expression that evaluates to an array or map. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -157,6 +162,11 @@ object Size { */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the size of an array. The function returns null for null input.", + arguments = """ + Arguments: + * expr - The array to return the size of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -279,6 +289,12 @@ case class MapContainsKey(left: Expression, right: Expression) _FUNC_(a1, a2, ...) - Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays. """, + arguments = """ + Arguments: + * a1 - The first array to merge. + An expression that evaluates to an array. + * a2 - The second array to merge. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4)); @@ -1044,6 +1060,13 @@ case class MapSort(base: Expression) elements for double/float type. Null elements will be placed at the beginning of the returned array in ascending order or at the end of the returned array in descending order. """, + arguments = """ + Arguments: + * array - The array to sort. + An expression that evaluates to an array. + * ascendingOrder - Whether to sort in ascending order; false sorts in descending order. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true); @@ -1244,6 +1267,11 @@ case class SortArray(base: Expression, ascendingOrder: Expression) */ @ExpressionDescription( usage = "_FUNC_(array) - Returns a random permutation of the given array.", + arguments = """ + Arguments: + * array - The array to return a random permutation of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, 3, 5)); @@ -1344,6 +1372,11 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None) extends U @ExpressionDescription( usage = """_FUNC_(expr) - Returns a reversed string, a binary value with bytes in reverse order, or an array with reverse order of elements.""", + arguments = """ + Arguments: + * expr - The string, binary value, or array to reverse. + An expression that evaluates to a string, binary, or array. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -1451,6 +1484,13 @@ case class Reverse(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, value) - Returns true if the array contains the value.", + arguments = """ + Arguments: + * array - The array to search. + An expression that evaluates to an array. + * value - The value to check for membership in the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -1731,6 +1771,13 @@ trait ArrayPendBase extends RuntimeReplaceable Null element is also prepended to the array. But if the array passed is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to prepend the element to. + An expression that evaluates to an array. + * element - The element to add at the beginning of the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1766,6 +1813,13 @@ case class ArrayPrepend(left: Expression, right: Expression) extends ArrayPendBa Null element is also appended into the array. But if the array passed, is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to append the element to. + An expression that evaluates to an array. + * element - The element to add at the end of the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1794,6 +1848,13 @@ case class ArrayAppend(left: Expression, right: Expression) extends ArrayPendBas // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(a1, a2) - Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.", + arguments = """ + Arguments: + * a1 - The first array to compare. + An expression that evaluates to an array. + * a2 - The second array to compare. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5)); @@ -2022,6 +2083,15 @@ case class ArraysOverlap(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(x, start, length) - Subsets array x starting from index start (array indices start at 1, or starting from the end if start is negative) with the specified length.", + arguments = """ + Arguments: + * x - The array to take a subset of. + An expression that evaluates to an array. + * start - The 1-based start index, or from the end if negative. + An expression that evaluates to an integer. + * length - The number of elements to take. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 2, 2); @@ -2121,6 +2191,15 @@ case class Slice(x: Expression, start: Expression, length: Expression) _FUNC_(array, delimiter[, nullReplacement]) - Concatenates the elements of the given array using the delimiter and an optional string to replace nulls. If no value is set for nullReplacement, any null value is filtered.""", + arguments = """ + Arguments: + * array - The array whose elements are concatenated. + An expression that evaluates to an array. + * delimiter - The delimiter placed between concatenated elements. + An expression that evaluates to a string. + * nullReplacement - The string used to replace null elements. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(array('hello', 'world'), ' '); @@ -2316,6 +2395,11 @@ case class ArrayJoin( usage = """ _FUNC_(array) - Returns the minimum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the minimum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2389,6 +2473,11 @@ case class ArrayMin(child: Expression) usage = """ _FUNC_(array) - Returns the maximum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the maximum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2469,6 +2558,13 @@ case class ArrayMax(child: Expression) _FUNC_(array, element) - Returns the (1-based) index of the first matching element of the array as long, or 0 if no match is found. """, + arguments = """ + Arguments: + * array - The array to search within. + An expression that evaluates to an array. + * element - The element to find the position of. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(312, 773, 708, 708), 708); @@ -2571,6 +2667,13 @@ case class ArrayPosition(left: Expression, right: Expression) _FUNC_(array, index) - Returns element of array at given (0-based) index. If the index points outside of the array boundaries, then this function returns NULL. """, + arguments = """ + Arguments: + * array - The array to retrieve an element from. + An expression that evaluates to an array. + * index - The 0-based index of the element to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0); @@ -2619,6 +2722,13 @@ case class Get(left: Expression, right: Expression) _FUNC_(map, key) - Returns value for given key. The function returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array to retrieve the element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -2872,6 +2982,13 @@ case class ElementAt( _FUNC_(map, key) - Returns value for given key. The function always returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array or map to retrieve an element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -3883,6 +4000,13 @@ object Sequence { */ @ExpressionDescription( usage = "_FUNC_(element, count) - Returns the array containing element count times.", + arguments = """ + Arguments: + * element - The element to repeat. + An expression of any type. + * count - The number of times to repeat the element. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -4007,6 +4131,13 @@ case class ArrayRepeat(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, element) - Remove all elements that equal to element from array.", + arguments = """ + Arguments: + * array - The array to remove elements from. + An expression that evaluates to an array. + * element - The element to remove from the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3), 3); @@ -4219,6 +4350,11 @@ trait ArraySetLike { */ @ExpressionDescription( usage = "_FUNC_(array) - Removes duplicate values from the array.", + arguments = """ + Arguments: + * array - The array to remove duplicate values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3)); @@ -4417,6 +4553,13 @@ trait ArrayBinaryLike _FUNC_(array1, array2) - Returns an array of the elements in the union of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to union. + An expression that evaluates to an array. + * array2 - The second array to union. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4596,6 +4739,13 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi _FUNC_(array1, array2) - Returns an array of the elements in the intersection of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to intersect. + An expression that evaluates to an array. + * array2 - The second array to intersect. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4830,6 +4980,13 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina _FUNC_(array1, array2) - Returns an array of the elements in array1 but not in array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The array to take elements from. + An expression that evaluates to an array. + * array2 - The array of elements to exclude. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -5043,6 +5200,15 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL Index above array size appends the array, or prepends the array if index is negative, with 'null' elements. """, + arguments = """ + Arguments: + * x - The array to insert the value into. + An expression that evaluates to an array. + * pos - The 1-based index at which to insert the value. + An expression that evaluates to an integer. + * val - The value to insert into the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 5, 5); @@ -5391,6 +5557,11 @@ case class ArrayInsert( @ExpressionDescription( usage = "_FUNC_(array) - Removes null values from the array.", + arguments = """ + Arguments: + * array - The array to remove null values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 1c601b29002e3..122272ba43339 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -299,6 +299,13 @@ object CreateMap { usage = """ _FUNC_(keys, values) - Creates a map with a pair of the given key/value arrays. All elements in keys should not be null""", + arguments = """ + Arguments: + * keys - The array providing the keys of the map. + An expression that evaluates to an array. + * values - The array providing the values of the map. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1.0, 3.0), array('2', '4')); @@ -566,6 +573,15 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression with // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for `pairDelim` and ':' for `keyValueDelim`. Both `pairDelim` and `keyValueDelim` are treated as regular expressions.", + arguments = """ + Arguments: + * text - The text to split into key/value pairs. + An expression that evaluates to a string. + * pairDelim - The delimiter regular expression separating pairs, defaults to ','. + An expression that evaluates to a string. + * keyValueDelim - The delimiter regular expression separating key and value, defaults to ':'. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index e7c6a1091861b..f966b04bec263 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -316,6 +316,13 @@ case class CurrentBatchTimestamp( */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to add to the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -356,6 +363,13 @@ case class DateAdd(startDate: Expression, days: Expression) */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` before `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to subtract from the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -548,6 +562,11 @@ trait GetDateField extends UnaryExpression with ImplicitCastInputTypes { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of year of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-04-09'); @@ -569,6 +588,11 @@ case class DayOfYear(child: Expression) extends GetDateField { @ExpressionDescription( usage = "_FUNC_(days) - Create date from the number of days since 1970-01-01.", + arguments = """ + Arguments: + * days - The number of days since 1970-01-01. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -597,6 +621,11 @@ case class DateFromUnixDate(child: Expression) extends UnaryExpression @ExpressionDescription( usage = "_FUNC_(date) - Returns the number of days since 1970-01-01.", + arguments = """ + Arguments: + * date - The date to convert to the number of days since 1970-01-01. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE("1970-01-02")); @@ -666,6 +695,11 @@ abstract class IntegralToTimestampBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds (can be fractional) since UTC epoch.", + arguments = """ + Arguments: + * seconds - The number of seconds since the UTC epoch. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(1230219000); @@ -802,6 +836,11 @@ case class SecondsToTimestamp(child: Expression) extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch.", + arguments = """ + Arguments: + * milliseconds - The number of milliseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123); @@ -826,6 +865,11 @@ case class MillisToTimestamp(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(microseconds) - Creates timestamp from the number of microseconds since UTC epoch.", + arguments = """ + Arguments: + * microseconds - The number of microseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123123); @@ -850,6 +894,11 @@ case class MicrosToTimestamp(child: Expression) // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(nanoseconds) - Creates timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from the number of nanoseconds since UTC epoch.", + arguments = """ + Arguments: + * nanoseconds - The number of nanoseconds since the UTC epoch. + An expression that evaluates to an integral or decimal. + """, examples = """ Examples: > SET spark.sql.timestampNanosTypes.enabled=true; @@ -965,6 +1014,11 @@ abstract class TimestampToLongBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to seconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -998,6 +1052,11 @@ case class CastTimestampNTZToLong(child: Expression) extends TimestampToLongBase // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to milliseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1018,6 +1077,11 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to microseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1089,6 +1153,11 @@ case class UnixNanos(child: Expression) // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1118,6 +1187,11 @@ case class YearOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the quarter from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31'); @@ -1140,6 +1214,11 @@ case class Quarter(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the month component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1161,6 +1240,11 @@ case class Month(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of month of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1183,6 +1267,11 @@ case class DayOfMonth(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (1 = Sunday, 2 = Monday, ..., 7 = Saturday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1205,6 +1294,11 @@ case class DayOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1227,6 +1321,11 @@ case class WeekDay(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the week of the year of the given date. A week is considered to start on a Monday and week 1 is the first week with >3 days.", + arguments = """ + Arguments: + * date - The date to extract the week of the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1249,6 +1348,11 @@ case class WeekOfYear(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated month name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated month name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1271,6 +1375,11 @@ case class MonthName(child: Expression) extends GetDateField with DefaultStringP // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated day name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated day name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE('2008-02-20')); @@ -1298,8 +1407,10 @@ case class DayName(child: Expression) extends GetDateField with DefaultStringPro arguments = """ Arguments: * timestamp - A date, time, timestamp or string to be converted to the given format. + An expression that evaluates to a timestamp or time. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1439,9 +1550,11 @@ object DateFormatClass { arguments = """ Arguments: * timeExp - A date/timestamp or string which is returned as a UNIX timestamp. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1493,9 +1606,11 @@ case class ToUnixTimestamp( arguments = """ Arguments: * timeExp - A date/timestamp or string. If not provided, this defaults to current time. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1578,8 +1693,10 @@ case class GetTimestamp( arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp without time zone. + An expression that evaluates to a string, date, or timestamp. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1615,6 +1732,7 @@ object ParseToTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp with local time zone. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1653,6 +1771,7 @@ object ParseToTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1874,8 +1993,10 @@ abstract class UnixTime extends ToTimestamp { arguments = """ Arguments: * unix_time - UNIX Timestamp to be converted to the provided format. + An expression that evaluates to a long. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1953,6 +2074,11 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ */ @ExpressionDescription( usage = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", + arguments = """ + Arguments: + * date - The date whose month's last day is returned. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-01-12'); @@ -2000,6 +2126,13 @@ case class LastDay(startDate: Expression) When both of the input parameters are not NULL and day_of_week is an invalid input, the function throws SparkIllegalArgumentException if `spark.sql.ansi.enabled` is set to true, otherwise NULL. """, + arguments = """ + Arguments: + * start_date - The date after which to find the next occurrence of the given day of week. + An expression that evaluates to a date. + * day_of_week - The name of the day of week to find (e.g. "Mon", "Tuesday"). + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2015-01-14', 'TU'); @@ -2353,6 +2486,13 @@ sealed trait UTCTimestamp extends BinaryExpression with ImplicitCastInputTypes { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in UTC. + An expression that evaluates to a timestamp. + * timezone - The target time zone to render the timestamp in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2387,6 +2527,13 @@ case class FromUTCTimestamp(left: Expression, right: Expression) extends UTCTime // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in the given time zone. + An expression that evaluates to a timestamp. + * timezone - The time zone the timestamp is interpreted as being in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2426,6 +2573,13 @@ abstract class AddMonthsBase extends BinaryExpression with ImplicitCastInputType // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(start_date, num_months) - Returns the date that is `num_months` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date to add months to. + An expression that evaluates to a date. + * num_months - The number of months to add to the start date. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 1); @@ -2519,6 +2673,15 @@ case class TimestampAddYMInterval( are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false. """, + arguments = """ + Arguments: + * timestamp1 - The first timestamp to compare. + An expression that evaluates to a timestamp. + * timestamp2 - The second timestamp to compare. + An expression that evaluates to a timestamp. + * roundOff - Whether to round off the result to 8 decimal places. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); @@ -2586,8 +2749,10 @@ case class MonthsBetween( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2662,8 +2827,10 @@ case class ParseToDate( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2705,6 +2872,7 @@ object TryToDateExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -2864,11 +3032,13 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes { arguments = """ Arguments: * date - date value or valid date string + An expression that evaluates to a date. * fmt - the format representing the unit to be truncated to - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in - "QUARTER" - truncate to the first date of the quarter that the `date` falls in - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in - "WEEK" - truncate to the Monday of the week that the `date` falls in + An expression that evaluates to a string. """, examples = """ Examples: @@ -2933,7 +3103,9 @@ case class TruncDate(date: Expression, format: Expression) - "SECOND" - zero out the second fraction part - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - everything remains + An expression that evaluates to a string. * ts - datetime value or valid timestamp string + An expression that evaluates to a timestamp. """, examples = """ Examples: @@ -2993,6 +3165,13 @@ case class TruncTimestamp( */ @ExpressionDescription( usage = "_FUNC_(endDate, startDate) - Returns the number of days from `startDate` to `endDate`.", + arguments = """ + Arguments: + * endDate - The end date to count days up to. + An expression that evaluates to a date. + * startDate - The start date to count days from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-31', '2009-07-30'); @@ -3031,8 +3210,11 @@ case class DateDiff(endDate: Expression, startDate: Expression) arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. """, examples = """ Examples: @@ -3252,13 +3434,19 @@ case class MakeTimestampLTZNanos( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3306,13 +3494,19 @@ object MakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3363,13 +3557,19 @@ object TryMakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3430,13 +3630,19 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3702,14 +3908,20 @@ case class TryMakeTimestamp( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -3852,13 +4064,19 @@ case class MakeTimestampFromDateTime( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -4250,8 +4468,11 @@ object SubtractDates { Arguments: * sourceTz - the time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + An expression that evaluates to a string. * targetTz - the time zone to which the input timestamp should be converted + An expression that evaluates to a string. * sourceTs - a timestamp without time zone + An expression that evaluates to a timestamp. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala index 3e6f6dbe22e07..2a9a140ef2a99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala @@ -52,6 +52,11 @@ import org.apache.spark.util.ArrayImplicits._ */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the MD5 checksum of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -94,6 +99,13 @@ case class Md5(child: Expression) _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. """, + arguments = """ + Arguments: + * expr - The expression to compute the SHA-2 checksum of. + An expression that evaluates to a binary. + * bitLength - The bit length of the SHA-2 result (224, 256, 384, or 512). + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark', 256); @@ -160,6 +172,11 @@ case class Sha2(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the SHA-1 hash of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -195,6 +212,11 @@ case class Sha1(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint.", + arguments = """ + Arguments: + * expr - The expression to compute the cyclic redundancy check value of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index f87b4f70d298a..524e27321f4fa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -393,6 +393,13 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction { */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Transforms elements in an array using the function.", + arguments = """ + Arguments: + * expr - The array whose elements are transformed. + An expression that evaluates to an array. + * func - The function applied to each element. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x + 1); @@ -513,6 +520,13 @@ case class ArrayTransform( equal to, or greater than the second element. If the comparator function returns null, the function will fail and raise an error. """, + arguments = """ + Arguments: + * expr - The input array to sort. + An expression that evaluates to an array. + * func - The comparator function used to determine the sort order. + A lambda function returning an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); @@ -700,6 +714,13 @@ case class MapFilter( */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Filters the input array using the given predicate.", + arguments = """ + Arguments: + * expr - The array to filter. + An expression that evaluates to an array. + * func - The predicate used to filter the array. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1); @@ -838,6 +859,13 @@ case class ArrayFilter( */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for one or more elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -972,6 +1000,13 @@ object ArrayExists { */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for all elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -1093,6 +1128,17 @@ case class ArrayForAll( elements in the array, and reduces this to a single state. The final state is converted into the final result by applying a finish function. """, + arguments = """ + Arguments: + * expr - The input array to reduce. + An expression that evaluates to an array. + * start - The initial state to start the reduction from. + An expression of any type. + * merge - The binary operator applied to the current state and each array element. + A lambda function. + * finish - The function that converts the final state into the result. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0, (acc, x) -> acc + x); @@ -1652,6 +1698,15 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(left, right, func) - Merges the two given arrays, element-wise, into a single array using function. If one array is shorter, nulls are appended at the end to match the length of the longer array, before applying function.", + arguments = """ + Arguments: + * left - The first array to merge. + An expression that evaluates to an array. + * right - The second array to merge. + An expression that evaluates to an array. + * func - The function applied element-wise to merge the arrays. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 3e4d6772c4fc4..29522cbcc15c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -233,6 +233,7 @@ case class DivideInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -316,6 +317,7 @@ case class TryMakeInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -474,9 +476,13 @@ case class MakeInterval( arguments = """ Arguments: * days - the number of days, positive or negative + An expression that evaluates to an integer. * hours - the number of hours, positive or negative + An expression that evaluates to an integer. * mins - the number of minutes, positive or negative + An expression that evaluates to an integer. * secs - the number of seconds with the fractional part in microsecond precision. + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -557,7 +563,9 @@ case class MakeDTInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index b4f41fda42f00..a6792d80ef341 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -42,6 +42,13 @@ import org.apache.spark.unsafe.types.UTF8String */ @ExpressionDescription( usage = "_FUNC_(json_txt, path) - Extracts a json object from `path`.", + arguments = """ + Arguments: + * json_txt - The JSON text to extract from. + An expression that evaluates to a string. + * path - The path identifying the JSON object to extract. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":"b"}', '$.a'); @@ -449,6 +456,12 @@ object JsonToStructs { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, options]) - Returns a JSON string with a given struct value", + arguments = """ + Arguments: + * expr - The struct value to convert to a JSON string. + An expression that evaluates to a struct, array, map, or variant. + * options - Options controlling how the JSON string is produced. + """, examples = """ Examples: > SELECT _FUNC_(named_struct('a', 1, 'b', 2)); @@ -611,6 +624,7 @@ case class SchemaOfJson( Arguments: * jsonArray - A JSON array. `NULL` is returned in case of any other valid JSON string, `NULL` or an invalid JSON. + An expression that evaluates to a string. """, examples = """ Examples: @@ -657,6 +671,7 @@ case class LengthOfJsonArray(child: Expression) * json_object - A JSON object. If a valid JSON object is given, all the keys of the outermost object will be returned as an array. If it is any other valid JSON string, an invalid JSON string or an empty string, the function returns null. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala index 01481050f2805..ba6c68ab95fcc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala @@ -33,6 +33,11 @@ import org.apache.spark.unsafe.types.UTF8String usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -61,6 +66,11 @@ case class KllSketchToStringBigint(child: Expression) extends KllSketchToStringB usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -89,6 +99,11 @@ case class KllSketchToStringFloat(child: Expression) extends KllSketchToStringBa usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -127,6 +142,11 @@ abstract class KllSketchToStringBase usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -155,6 +175,11 @@ case class KllSketchGetNBigint(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -183,6 +208,11 @@ case class KllSketchGetNFloat(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -221,6 +251,13 @@ abstract class KllSketchGetNBase usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -252,6 +289,13 @@ case class KllSketchMergeBigint(left: Expression, right: Expression) extends Kll usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_float(_FUNC_(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -283,6 +327,13 @@ case class KllSketchMergeFloat(left: Expression, right: Expression) extends KllS usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -327,6 +378,13 @@ abstract class KllSketchMergeBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -359,6 +417,13 @@ case class KllSketchGetQuantileBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -391,6 +456,13 @@ case class KllSketchGetQuantileFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -523,6 +595,13 @@ abstract class KllSketchGetQuantileBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a long or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -551,6 +630,13 @@ case class KllSketchGetRankBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a float or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -579,6 +665,13 @@ case class KllSketchGetRankFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala index 5bba89b3dad51..9613bc25d65f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala @@ -38,10 +38,15 @@ import org.apache.spark.unsafe.types.UTF8String arguments = """ Arguments: * input - string value to mask. Supported types: STRING, VARCHAR, CHAR + An expression that evaluates to a string. * upperChar - character to replace upper-case characters with. Specify NULL to retain original character. Default value: 'X' + An expression that evaluates to a string. Must be a constant. * lowerChar - character to replace lower-case characters with. Specify NULL to retain original character. Default value: 'x' + An expression that evaluates to a string. Must be a constant. * digitChar - character to replace digit characters with. Specify NULL to retain original character. Default value: 'n' + An expression that evaluates to a string. Must be a constant. * otherChar - character to replace all other characters with. Specify NULL to retain original character. Default value: NULL + An expression that evaluates to a string. Must be a constant. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 7bd166250db01..5fecba3fc54f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -192,6 +192,11 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") _FUNC_(expr) - Returns the inverse cosine (a.k.a. arc cosine) of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -210,6 +215,11 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS" _FUNC_(expr) - Returns the inverse sine (a.k.a. arc sine) the arc sin of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -228,6 +238,11 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN" _FUNC_(expr) - Returns the inverse tangent (a.k.a. arc tangent) of `expr`, as if computed by `java.lang.Math._FUNC_` """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -241,6 +256,11 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN" @ExpressionDescription( usage = "_FUNC_(expr) - Returns the cube root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the cube root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(27.0); @@ -308,6 +328,13 @@ trait CeilFloorExpressionBuilderBase extends ExpressionBuilder { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, scale]) - Returns the smallest number after rounding up that is not smaller than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round up. + An expression that evaluates to a numeric. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -349,6 +376,7 @@ case class RoundCeil(child: Expression, scale: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -368,6 +396,7 @@ case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -392,6 +421,7 @@ case class Sec(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -408,6 +438,11 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic cosine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -436,6 +471,15 @@ case class Acosh(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`.", + arguments = """ + Arguments: + * num - The number to convert. + An expression that evaluates to a string. + * from_base - The base the number is currently in. + An expression that evaluates to an integer. + * to_base - The base to convert the number to. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('100', 2, 10); @@ -501,6 +545,11 @@ case class Conv( @ExpressionDescription( usage = "_FUNC_(expr) - Returns e to the power of `expr`.", + arguments = """ + Arguments: + * expr - The exponent to raise e to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -517,6 +566,11 @@ case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "E @ExpressionDescription( usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.", + arguments = """ + Arguments: + * expr - The exponent used to compute exp(expr) - 1. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -564,6 +618,13 @@ case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLO // scalastyle:off line.size.limit @ExpressionDescription( usage = " _FUNC_(expr[, scale]) - Returns the largest number after rounding down that is not greater than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round down. + An expression that evaluates to a numeric. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -630,6 +691,11 @@ object Factorial { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null.", + arguments = """ + Arguments: + * expr - The integer between 0 and 20 to compute the factorial of. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(5); @@ -676,6 +742,11 @@ case class Factorial(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the natural logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -690,6 +761,11 @@ case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LO @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.", + arguments = """ + Arguments: + * expr - The expression to compute the base-2 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2); @@ -715,6 +791,11 @@ case class Log2(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 10.", + arguments = """ + Arguments: + * expr - The expression to compute the base-10 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10); @@ -728,6 +809,11 @@ case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, @ExpressionDescription( usage = "_FUNC_(expr) - Returns log(1 + `expr`).", + arguments = """ + Arguments: + * expr - The expression used to compute log(1 + expr). + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -743,6 +829,11 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.", + arguments = """ + Arguments: + * expr - The value to round to the closest mathematical integer. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(12.3456); @@ -759,6 +850,11 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND @ExpressionDescription( usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive.", + arguments = """ + Arguments: + * expr - The expression whose sign is returned. + An expression that evaluates to a double or interval. + """, examples = """ Examples: > SELECT _FUNC_(40); @@ -782,6 +878,7 @@ case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "S arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -801,6 +898,7 @@ case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -824,6 +922,7 @@ case class Csc(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -840,6 +939,11 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic sine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -859,6 +963,11 @@ case class Asinh(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the square root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the square root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(4); @@ -877,6 +986,7 @@ case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT" arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -896,6 +1006,7 @@ case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -920,6 +1031,7 @@ case class Cot(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -936,6 +1048,11 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic tangent of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -961,6 +1078,7 @@ case class Atanh(child: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -980,6 +1098,7 @@ case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegre arguments = """ Arguments: * expr - angle in degrees + An expression that evaluates to a double. """, examples = """ Examples: @@ -997,6 +1116,11 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the string representation of the long value `expr` represented in binary.", + arguments = """ + Arguments: + * expr - The long value to represent in binary. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(13); @@ -1112,6 +1236,11 @@ object Hex { */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts `expr` to hexadecimal.", + arguments = """ + Arguments: + * expr - The expression to convert to hexadecimal. + An expression that evaluates to a long, binary, or string. + """, examples = """ Examples: > SELECT _FUNC_(17); @@ -1162,6 +1291,11 @@ case class Hex(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts hexadecimal `expr` to binary.", + arguments = """ + Arguments: + * expr - The hexadecimal expression to convert to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); @@ -1226,7 +1360,9 @@ case class Unhex(child: Expression, failOnError: Boolean = false) arguments = """ Arguments: * exprY - coordinate on y-axis + An expression that evaluates to a double. * exprX - coordinate on x-axis + An expression that evaluates to a double. """, examples = """ Examples: @@ -1253,6 +1389,13 @@ case class Atan2(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Raises `expr1` to the power of `expr2`.", + arguments = """ + Arguments: + * expr1 - The base to be raised to a power. + An expression that evaluates to a double. + * expr2 - The exponent to raise the base to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -1314,6 +1457,13 @@ sealed trait BitShiftOperation */ @ExpressionDescription( usage = "base _FUNC_ exp - Bitwise left shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted left. + An expression that evaluates to an integer or long. + * exp - The number of positions to shift left by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftleft(2, 1); @@ -1343,6 +1493,13 @@ case class ShiftLeft(left: Expression, right: Expression) extends BitShiftOperat */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise (signed) right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftright(4, 1); @@ -1371,6 +1528,13 @@ case class ShiftRight(left: Expression, right: Expression) extends BitShiftOpera */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise unsigned right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right without sign extension. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftrightunsigned(4, 1); @@ -1395,6 +1559,13 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) extends BitSh // scalastyle:off nonascii @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns sqrt(`expr1`\u00B2 + `expr2`\u00B2).", + arguments = """ + Arguments: + * expr1 - The first value whose square is summed. + An expression that evaluates to a double. + * expr2 - The second value whose square is summed. + An expression that evaluates to a double. + """, // scalastyle:on nonascii examples = """ Examples: @@ -1418,6 +1589,13 @@ case class Hypot(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(base, expr) - Returns the logarithm of `expr` with `base`.", + arguments = """ + Arguments: + * base - The base of the logarithm. + An expression that evaluates to a double. + * expr - The expression to compute the logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10, 100); @@ -1701,6 +1879,13 @@ abstract class RoundBase(child: Expression, scale: Expression, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1732,6 +1917,13 @@ case class Round( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1826,6 +2018,17 @@ object WidthBucket { `value` would be assigned in an equiwidth histogram with `num_bucket` buckets, in the range `min_value` to `max_value`." """, + arguments = """ + Arguments: + * value - The value to assign to a bucket. + An expression that evaluates to a double or interval. + * min_value - The minimum value of the histogram range. + An expression that evaluates to a double or interval. + * max_value - The maximum value of the histogram range. + An expression that evaluates to a double or interval. + * num_bucket - The number of equiwidth buckets in the histogram. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(5.3, 0.2, 10.6, 5); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 23e8b343e431e..651b1a231279d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -154,6 +154,11 @@ object RaiseError { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_( expr ) - Throws a USER_RAISED_EXCEPTION with `expr` as message.", + arguments = """ + Arguments: + * expr - The message for the raised exception. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('custom error message'); @@ -177,6 +182,13 @@ object RaiseErrorExpressionBuilder extends ExpressionBuilder { */ @ExpressionDescription( usage = "_FUNC_(expr [, message]) - Throws an exception if `expr` is not true.", + arguments = """ + Arguments: + * expr - The expression that is expected to be true. + An expression that evaluates to a boolean. + * message - The message for the exception thrown when the expression is not true. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(0 < 1); @@ -412,15 +424,21 @@ case class CurrentUser() arguments = """ Arguments: * expr - The binary value to encrypt. + An expression that evaluates to a binary. * key - The passphrase to use to encrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * iv - Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or ''. 16-byte array for CBC mode. 12-byte array for GCM mode. + An expression that evaluates to a binary. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -495,13 +513,18 @@ case class AesEncrypt( arguments = """ Arguments: * expr - The binary value to decrypt. + An expression that evaluates to a binary. * key - The passphrase to use to decrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -561,6 +584,19 @@ case class AesDecrypt( @ExpressionDescription( usage = "_FUNC_(expr, key[, mode[, padding[, aad]]]) - This is a special version of `aes_decrypt` that performs the same operation, but returns a NULL value instead of raising an error if the decryption cannot be performed.", + arguments = """ + Arguments: + * expr - The encrypted value to decrypt. + An expression that evaluates to a binary. + * key - The key used to decrypt the value. + An expression that evaluates to a binary. + * mode - The block cipher mode used for decryption. + An expression that evaluates to a string. + * padding - The padding scheme used for decryption. + An expression that evaluates to a string. + * aad - The additional authenticated data. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 1d50d8987ccee..eb8e67eca3f14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -188,6 +188,13 @@ private case class TypedNullLiteral(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not equal to the other expression. + An expression of any orderable type. + * expr2 - The value compared against the first expression. + An expression of any orderable type. + """, examples = """ Examples: > SELECT _FUNC_(2, 2); @@ -219,6 +226,11 @@ case class NullIf(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns null if `expr` is equal to zero, or `expr` otherwise.", + arguments = """ + Arguments: + * expr - The expression that returns null when equal to zero. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -313,6 +325,11 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, replace */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if `expr` is NaN, or false otherwise.", + arguments = """ + Arguments: + * expr - The expression to test for NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double)); @@ -361,6 +378,13 @@ case class IsNaN(child: Expression) extends UnaryExpression */ @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not NaN. + An expression that evaluates to a double or float. + * expr2 - The value returned when the first expression is NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double), 123); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala index 0a6d23977d2f6..6389a9e0b5660 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala @@ -127,6 +127,13 @@ abstract class ToNumberBase(left: Expression, right: Expression, errorOnFail: Bo negative number with wrapping angled brackets. ('<1>'). """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -162,6 +169,13 @@ case class ToNumber(left: Expression, right: Expression) Returns NULL if the string 'expr' does not match the expected format. The format follows the same semantics as the to_number function. """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -223,6 +237,13 @@ case class TryToNumber(left: Expression, right: Expression) 'hex': a string in the hexadecimal format. 'utf-8': the input binary is decoded to UTF-8 string. """, + arguments = """ + Arguments: + * expr - The expression to convert to a string. + An expression that evaluates to a numeric, date, timestamp, time, or binary. + * format - The format that specifies how to render the value as a string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(454, '999'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 7767e97d34ac5..d9f9c7fec6dab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -306,6 +306,11 @@ trait PredicateHelper extends AliasHelper with Logging { @ExpressionDescription( usage = "_FUNC_ expr - Logical not.", + arguments = """ + Arguments: + * expr - The boolean expression to negate. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_ true; @@ -826,6 +831,13 @@ object InSet { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical AND.", + arguments = """ + Arguments: + * expr1 - The first boolean operand. + An expression that evaluates to a boolean. + * expr2 - The second boolean operand. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ true; @@ -922,6 +934,13 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical OR.", + arguments = """ + Arguments: + * expr1 - The left boolean operand of the logical OR. + An expression that evaluates to a boolean. + * expr2 - The right boolean operand of the logical OR. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ false; @@ -1091,6 +1110,7 @@ object Equality { * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1139,6 +1159,7 @@ case class EqualTo(left: Expression, right: Expression) * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1202,6 +1223,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1238,6 +1260,7 @@ case class EqualNull(left: Expression, right: Expression, replacement: Expressio and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1273,6 +1296,7 @@ case class LessThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1308,6 +1332,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1344,6 +1369,7 @@ case class GreaterThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index adaaceb2e4396..b52d09fc9c709 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -92,6 +92,11 @@ private[catalyst] object ExpressionWithRandomSeed { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1).", + arguments = """ + Arguments: + * seed - The seed for the random number generator. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -146,6 +151,11 @@ object Rand { // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.""", + arguments = """ + Arguments: + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -205,6 +215,15 @@ object Randn { one or both of these are floating-point numbers, then the result will also be a floating-point number. """, + arguments = """ + Arguments: + * min - The lower bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * max - The upper bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(10, 20, 0) > 0 AS result; @@ -329,6 +348,13 @@ object Uniform { optional. The string length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). """, + arguments = """ + Arguments: + * length - The length of the random string to generate. + An expression that evaluates to an integer. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 3bd0352f6f07c..9f24c512bd335 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -127,6 +127,7 @@ private[catalyst] object StringRegexExpression { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)\ @@ -140,6 +141,7 @@ private[catalyst] object StringRegexExpression { enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -251,6 +253,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally and case-insensitively, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)

@@ -264,6 +267,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -481,7 +485,9 @@ case class NotLikeAny(child: Expression, patterns: Seq[UTF8String]) extends Like arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * regexp - a string expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL parser, see the unescaping rules at String Literal. @@ -573,14 +579,17 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress arguments = """ Arguments: * str - a string expression to split. + An expression that evaluates to a string. * regex - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. * limit - an integer expression which controls the number of times the regex is applied. * limit > 0: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last matched regex. * limit <= 0: `regex` will be applied as many times as possible, and the resulting array can be of any size. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -654,6 +663,7 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) arguments = """ Arguments: * str - a string expression to search for a regular expression pattern match. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -664,9 +674,12 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * rep - a string expression to replace matched substrings. + An expression that evaluates to a string. * position - a positive integer literal that indicates the position within `str` to begin searching. The default is 1. If position is greater than the number of characters in `str`, the result is `str`. + An expression that evaluates to an integer. Must be a constant. """, examples = """ Examples: @@ -876,6 +889,7 @@ abstract class RegExpExtractBase arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -886,11 +900,13 @@ abstract class RegExpExtractBase if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex maybe contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -949,6 +965,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -959,11 +976,13 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex may contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -1018,8 +1037,10 @@ case class RegExpExtractAll(subject: Expression, regexp: Expression, idx: Expres arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1057,7 +1078,9 @@ case class RegExpCount(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1097,6 +1120,7 @@ case class RegExpSubStr(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -1107,6 +1131,10 @@ case class RegExpSubStr(left: Expression, right: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. + * idx - the group index of the regular expression to search for. The default value is 0 + if not specified. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala index fe774a62fcc6d..556b9603e9785 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala @@ -134,6 +134,7 @@ case class ST_AsBinary(geo: Expression, endianness: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOGRAPHY value. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -179,7 +180,9 @@ case class ST_GeogFromWKB(wkb: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOMETRY value. + An expression that evaluates to a binary. * srid - The optional SRID value of the geometry. Default is 0. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 66c4a39ce8233..2aed0d9c3abe4 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -60,6 +60,11 @@ import org.apache.spark.util.ArrayImplicits._ // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values.", + arguments = """ + Arguments: + * sep - The separator placed between the concatenated values. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(' ', 'Spark', 'SQL'); @@ -452,6 +457,11 @@ trait String2StringExpression extends ImplicitCastInputTypes { */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to uppercase.", + arguments = """ + Arguments: + * str - The string to convert to uppercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -485,6 +495,11 @@ case class Upper(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to lowercase.", + arguments = """ + Arguments: + * str - The string to convert to lowercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -593,6 +608,13 @@ case class BinaryPredicate(override val prettyName: String, left: Expression, ri Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to search within. + An expression that evaluates to a string or binary. + * right - The expression to search for inside the left expression. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -635,6 +657,13 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The string to test. + An expression that evaluates to a string or binary. + * right - The prefix to check for at the start of the string. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -684,6 +713,13 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to test. + An expression that evaluates to a string or binary. + * right - The suffix expression to check the left expression ends with. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'SQL'); @@ -735,6 +771,7 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -783,6 +820,7 @@ case class IsValidUTF8(input: Expression) extends RuntimeReplaceable with Implic arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -829,6 +867,7 @@ case class MakeValidUTF8(input: Expression) extends RuntimeReplaceable with Impl arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -877,6 +916,7 @@ case class ValidateUTF8(input: Expression) extends RuntimeReplaceable with Impli arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -928,9 +968,12 @@ case class TryValidateUTF8(input: Expression) extends RuntimeReplaceable with Im arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * search - a string expression. If `search` is not found in `str`, `str` is returned unchanged. + An expression that evaluates to a string. * replace - a string expression. If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1013,6 +1056,17 @@ object Overlay { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, replace, pos[, len]) - Replace `input` with `replace` that starts at `pos` and is of length `len`.", + arguments = """ + Arguments: + * input - The string to have part of its content replaced. + An expression that evaluates to a string or binary. + * replace - The replacement content to insert. + An expression that evaluates to a string or binary. + * pos - The 1-based start position of the replacement. + An expression that evaluates to an integer. + * len - The number of characters to replace. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL' PLACING '_' FROM 6); @@ -1167,6 +1221,15 @@ object StringTranslate { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, from, to) - Translates the `input` string by replacing the characters present in the `from` string with the corresponding characters in the `to` string.", + arguments = """ + Arguments: + * input - The string to translate. + An expression that evaluates to a string. + * from - The characters to be replaced. + An expression that evaluates to a string. + * to - The corresponding replacement characters. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('AaBbCc', 'abc', '123'); @@ -1247,6 +1310,13 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac _FUNC_(str, str_array) - Returns the index (1-based) of the given string (`str`) in the comma-delimited list (`str_array`). Returns 0, if the string was not found or if the given string (`str`) contains a comma. """, + arguments = """ + Arguments: + * str - The string to search for. + An expression that evaluates to a string. + * str_array - The comma-delimited list to search within. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('ab','abc,b,ab,c,def'); @@ -1413,7 +1483,9 @@ object StringTrim { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. * BOTH, FROM - these are keywords to specify trimming string characters from both ends of the string * LEADING, FROM - these are keywords to specify trimming string characters from the left @@ -1487,7 +1559,9 @@ case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1546,7 +1620,9 @@ object StringTrimLeft { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1611,7 +1687,9 @@ object StringTrimRight { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1662,6 +1740,17 @@ case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = Non If `start` is not specified, it defaults to 1. `occurrence` must be a positive integer and defaults to 1. """, + arguments = """ + Arguments: + * str - The string to search within. + An expression that evaluates to a string. + * substr - The substring to search for. + An expression that evaluates to a string. + * start - The 1-based position from which to start the search. + An expression that evaluates to an integer. + * occurrence - Which occurrence of the substring to find. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('SparkSQL', 'SQL'); @@ -1809,6 +1898,15 @@ case class StringInstrWithOccurrence( (counting from the right) is returned. The function substring_index performs a case-sensitive match when searching for `delim`. """, + arguments = """ + Arguments: + * str - The string to take the substring from. + An expression that evaluates to a string. + * delim - The delimiter to count occurrences of. + An expression that evaluates to a string. + * count - The number of delimiter occurrences that bound the returned substring. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('www.apache.org', '.', 2); @@ -1860,6 +1958,15 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`. The given `pos` and return value are 1-based. """, + arguments = """ + Arguments: + * substr - The substring to search for. + An expression that evaluates to a string. + * str - The string to search within. + An expression that evaluates to a string. + * pos - The 1-based position from which to start the search. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('bar', 'foobarbar'); @@ -1988,6 +2095,15 @@ trait PadExpressionBuilderBase extends ExpressionBuilder { If `pad` is not specified, `str` will be padded to the left with space characters if it is a character string, and with zeros if it is a byte sequence. """, + arguments = """ + Arguments: + * str - The string to be left-padded. + An expression that evaluates to a string or binary. + * len - The target length of the resulting string. + An expression that evaluates to an integer. + * pad - The string to pad with on the left. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2073,6 +2189,15 @@ case class BinaryPad(funcName: String, str: Expression, len: Expression, pad: Ex If `pad` is not specified, `str` will be padded to the right with space characters if it is a character string, and with zeros if it is a binary string. """, + arguments = """ + Arguments: + * str - The string to right-pad. + An expression that evaluates to a string or binary. + * len - The length to pad the string to. + An expression that evaluates to an integer. + * pad - The string used to pad on the right. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2135,6 +2260,12 @@ case class StringRPad( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.", + arguments = """ + Arguments: + * strfmt - The printf-style format string. + An expression that evaluates to a string. + * obj - The value to be formatted into the string. + """, examples = """ Examples: > SELECT _FUNC_("Hello World %d %s", 100, "days"); @@ -2253,6 +2384,11 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC _FUNC_(str) - Returns `str` with the first letter of each word in uppercase. All other letters are in lowercase. Words are delimited by white space. """, + arguments = """ + Arguments: + * str - The string to capitalize the first letter of each word in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('sPark sql'); @@ -2288,6 +2424,13 @@ case class InitCap(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.", + arguments = """ + Arguments: + * str - The string to repeat. + An expression that evaluates to a string. + * n - The number of times to repeat the string. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -2328,6 +2471,11 @@ case class StringRepeat(str: Expression, times: Expression) */ @ExpressionDescription( usage = "_FUNC_(n) - Returns a string consisting of `n` spaces.", + arguments = """ + Arguments: + * n - The number of spaces to produce. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT concat(_FUNC_(2), '1'); @@ -2370,6 +2518,15 @@ case class StringSpace(child: Expression) _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`. """, + arguments = """ + Arguments: + * str - The string or byte array to take the substring from. + An expression that evaluates to a string or binary. + * pos - The 1-based starting position of the substring. + An expression that evaluates to an integer. + * len - The length of the substring to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 5); @@ -2444,6 +2601,13 @@ case class Substring(str: Expression, pos: Expression, len: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the rightmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the rightmost characters from. + An expression that evaluates to a string. + * len - The number of characters to take from the right. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2484,6 +2648,13 @@ case class Right(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the leftmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the leftmost characters from. + An expression that evaluates to a string or binary. + * len - The number of characters to take from the left. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2523,6 +2694,11 @@ case class Left(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL '); @@ -2570,6 +2746,11 @@ case class Length(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bit length of string data or number of bits of binary data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the bit length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2615,6 +2796,11 @@ case class BitLength(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the byte length of string data or number of bytes of binary " + "data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the byte length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2662,6 +2848,15 @@ case class OctetLength(child: Expression) @ExpressionDescription( usage = """ _FUNC_(str1, str2[, threshold]) - Returns the Levenshtein distance between the two given strings. If threshold is set and distance more than it, return -1.""", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + * threshold - The maximum distance to compute; if exceeded, -1 is returned. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('kitten', 'sitting'); @@ -2817,6 +3012,13 @@ case class Levenshtein( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str1, str2) - Returns the Jaro-Winkler similarity between the two given strings. The result is a double between 0 and 1, where 1 means identical.", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('MARTHA', 'MARHTA'); @@ -2855,6 +3057,11 @@ case class JaroWinkler(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns Soundex code of the string.", + arguments = """ + Arguments: + * str - The string to compute the Soundex code of. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Miller'); @@ -2886,6 +3093,11 @@ case class SoundEx(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns the numeric value of the first character of `str`.", + arguments = """ + Arguments: + * str - The string whose first character's numeric value is returned. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('222'); @@ -2919,6 +3131,11 @@ case class Ascii(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the ASCII character having the binary equivalent to `expr`. If n is larger than 256 the result is equivalent to chr(n % 256)", + arguments = """ + Arguments: + * expr - The number to convert to its ASCII character. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(65); @@ -2952,6 +3169,11 @@ case class Chr(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(bin) - Converts the argument from a binary `bin` to a base 64 string.", + arguments = """ + Arguments: + * bin - The binary value to encode as a base 64 string. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -3007,6 +3229,11 @@ object Base64 { */ @ExpressionDescription( usage = "_FUNC_(str) - Converts the argument from a base 64 string `str` to a binary.", + arguments = """ + Arguments: + * str - The base 64 string to decode to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('U3BhcmsgU1FM'); @@ -3156,7 +3383,9 @@ object Decode { arguments = """ Arguments: * bin - a binary expression to decode + An expression that evaluates to a binary. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to decode `bin` into a STRING. It is case insensitive. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3252,7 +3481,9 @@ object StringDecode { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to encode `str` into a BINARY. It is case insensitive. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3332,6 +3563,13 @@ object Encode { By default, the binary format for conversion is "hex" if `fmt` is omitted. The function returns NULL if at least one of the input parameters is NULL. """, + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format describing how to interpret the string (e.g. "hex", "utf-8", "base64"). + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); @@ -3445,6 +3683,13 @@ case class ToBinary( `expr2` also accept a user specified format. This is supposed to function like MySQL's FORMAT. """, + arguments = """ + Arguments: + * expr1 - The number to format. + An expression that evaluates to a numeric. + * expr2 - The number of decimal places to round to, or a format string. + An expression that evaluates to an integer or string. + """, examples = """ Examples: > SELECT _FUNC_(12332.123456, 4); @@ -3611,10 +3856,13 @@ case class FormatNumber(x: Expression, d: Expression) arguments = """ Arguments: * str - A STRING expression to be parsed. + An expression that evaluates to a string. * lang - An optional STRING expression with a language code from ISO 639 Alpha-2 (e.g. 'DE'), Alpha-3, or a language subtag of up to 8 characters. + An expression that evaluates to a string. * country - An optional STRING expression with a country code from ISO 3166 alpha-2 country code or a UN M.49 numeric-3 area code. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3717,6 +3965,15 @@ case class StringSplitSQL( throws an error. If `partNum` is negative, the parts are counted backward from the end of the string. If the `delimiter` is an empty string, the `str` is not split. """, + arguments = """ + Arguments: + * str - The string to split. + An expression that evaluates to a string. + * delimiter - The string used to split the input into parts. + An expression that evaluates to a string. + * partNum - The 1-based index of the split part to return. + An expression that evaluates to an integer. + """, examples = """ Examples: @@ -3781,6 +4038,11 @@ case class Empty2Null(child: Expression) extends UnaryExpression with String2Str This checksum function is widely applied on credit card numbers and government identification numbers to distinguish valid numbers from mistyped, incorrect numbers. """, + arguments = """ + Arguments: + * str - The string of digits to validate against the Luhn algorithm. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('8112189876'); @@ -3818,6 +4080,11 @@ case class Luhncheck(input: Expression) extends RuntimeReplaceable with Implicit // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` enclosed by single quotes and each instance of single quote in it is preceded by a backslash.", + arguments = """ + Arguments: + * str - The string to enclose in single quotes and escape. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Don\'t'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 1c9d6b335e8d9..827cfbda957d8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -61,8 +61,10 @@ trait TimeExpression extends Expression { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -188,8 +190,10 @@ private[expressions] object NanosTimestampCast { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -263,6 +267,11 @@ case class MinutesOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the minute of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the minute component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -330,6 +339,11 @@ case class HoursOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the hour of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the hour component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -422,6 +436,11 @@ case class SecondsOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the second of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the second component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -583,8 +602,11 @@ case class CurrentTime( arguments = """ Arguments: * hour - the hour to represent, from 0 to 23 + An expression that evaluates to an integer. * minute - the minute to represent, from 0 to 59 + An expression that evaluates to an integer. * second - the second to represent, from 0 to 59.999999 + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -706,8 +728,11 @@ case class SubtractTimes(left: Expression, right: Expression) - "SECOND" - "MILLISECOND" - "MICROSECOND" + An expression that evaluates to a string. * start - a starting TIME expression + An expression that evaluates to a time. * end - an ending TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -770,7 +795,9 @@ case class TimeDiff( - "SECOND" - zero out the fraction part of seconds - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - zero out the nanoseconds + An expression that evaluates to a string. * time - a TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -834,6 +861,7 @@ abstract class TimeFromBase extends UnaryExpression with RuntimeReplaceable with Arguments: * seconds - seconds since midnight (0 to 86399.999999). Supports decimals for fractional seconds. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -864,6 +892,7 @@ case class TimeFromSeconds(child: Expression) extends TimeFromBase { arguments = """ Arguments: * millis - milliseconds since midnight (0 to 86399999) + An expression that evaluates to an integral. """, examples = """ Examples: @@ -893,6 +922,7 @@ case class TimeFromMillis(child: Expression) extends TimeFromBase { arguments = """ Arguments: * micros - microseconds since midnight (0 to 86399999999) + An expression that evaluates to an integral. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala index b51cb74a5a8fa..b0bd67701be85 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala @@ -38,7 +38,8 @@ import org.apache.spark.unsafe.types.UTF8String """, arguments = """ Arguments: - str - a string expression to be translated + * str - a string expression to be translated + An expression that evaluates to a string. """, examples = """ Examples: @@ -80,6 +81,7 @@ case class UrlEncode(child: Expression) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -123,6 +125,7 @@ case class UrlDecode(child: Expression, failOnError: Boolean = true) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -165,6 +168,15 @@ object UrlCodec { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - This is a special version of `parse_url` that performs the same operation, but returns a NULL value instead of raising an error if the parsing cannot be performed.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); @@ -197,6 +209,15 @@ case class TryParseUrl(params: Seq[Expression], replacement: Expression) */ @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index 0f8995ce829fe..d06b7a22bd2e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -576,6 +576,11 @@ abstract class ParseJsonExpressionBuilderBase(failOnError: Boolean) extends Expr // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Throw an exception when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); @@ -590,6 +595,11 @@ object ParseJsonExpressionBuilder extends ParseJsonExpressionBuilderBase(true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Return NULL when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 800b38ea32223..0e1cafbb24c01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -78,6 +78,13 @@ abstract class XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('1','a/b'); @@ -99,6 +106,13 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -121,6 +135,13 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -143,6 +164,13 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -165,6 +193,13 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -187,6 +222,13 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -210,6 +252,13 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the node whose text is returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('bcc','a/c'); @@ -233,6 +282,13 @@ case class XPathString(xml: Expression, path: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the nodes whose values are returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()');