diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 1e348cfd42874..46bd99404fd4c 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -517,11 +517,13 @@ def sqrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -553,7 +555,9 @@ def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -634,6 +638,7 @@ def try_avg(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -691,9 +696,11 @@ def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a numeric. Examples -------- @@ -754,9 +761,11 @@ def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - dividend + dividend. + A column that evaluates to a numeric. right : :class:`~pyspark.sql.Column` or column name - divisor + divisor. + A column that evaluates to a numeric. Examples -------- @@ -803,9 +812,11 @@ def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - multiplicand + multiplicand. + A column that evaluates to a numeric or interval. right : :class:`~pyspark.sql.Column` or column name - multiplier + multiplier. + A column that evaluates to a numeric or interval. Examples -------- @@ -867,7 +878,9 @@ def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- @@ -948,6 +961,7 @@ def try_sum(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- @@ -1017,11 +1031,13 @@ def abs(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the absolute value on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the absolute value of the input. + Returns a column of the same type as the input. Examples -------- @@ -1083,8 +1099,10 @@ def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column of any type. deterministic : bool, optional if there are multiple equally-frequent results then return the lowest (defaults to false). + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -1358,13 +1376,16 @@ def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values to be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be maximized. This could be the column instance or the column name as string. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the top `k` maximum ordering values, sorted in descending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1469,13 +1490,16 @@ def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> col : :class:`~pyspark.sql.Column` or column name The column representing the values that will be returned. This could be the column instance or the column name as string. + A column of any type. ord : :class:`~pyspark.sql.Column` or column name The column that needs to be minimized. This could be the column instance or the column name as string. + A column of any orderable type. k : int, optional If specified, returns an array of up to `k` values associated with the bottom `k` minimum ordering values, sorted in ascending order by the ordering column. Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- @@ -1638,6 +1662,7 @@ def sum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1702,6 +1727,7 @@ def avg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1756,6 +1782,7 @@ def mean(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- @@ -1800,6 +1827,7 @@ def median(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric, interval, or time. Returns ------- @@ -1939,8 +1967,10 @@ def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2094,8 +2124,10 @@ def string_agg( ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a string or binary. delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- @@ -2286,11 +2318,13 @@ def acos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2341,11 +2375,13 @@ def acosh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The target column or expression to compute the inverse hyperbolic cosine on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` A new column object representing the inverse hyperbolic cosine of the input. + Returns a column that evaluates to a double. Examples -------- @@ -2392,11 +2428,13 @@ def asin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse sine of `col`, as if computed by `java.lang.Math.asin()` + Returns a column that evaluates to a double. Examples -------- @@ -2444,11 +2482,13 @@ def asinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2495,11 +2535,13 @@ def atan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` inverse tangent of `col`, as if computed by `java.lang.Math.atan()` + Returns a column that evaluates to a double. Examples -------- @@ -2546,11 +2588,13 @@ def atanh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2599,11 +2643,13 @@ def cbrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -2650,8 +2696,10 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2659,6 +2707,7 @@ def ceil(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Col ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2708,8 +2757,10 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the ceiling on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -2717,6 +2768,7 @@ def ceiling(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> ------- :class:`~pyspark.sql.Column` A column for the computed results. + Returns a column that evaluates to a long or decimal. See Also -------- @@ -2765,12 +2817,14 @@ def cos(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosine of the angle, as if computed by `java.lang.Math.cos()`. + Returns a column that evaluates to a double. Examples -------- @@ -2817,12 +2871,14 @@ def cosh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh()` + Returns a column that evaluates to a double. Examples -------- @@ -2869,11 +2925,13 @@ def cot(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cotangent of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2921,11 +2979,13 @@ def csc(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` cosecant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -2992,11 +3052,13 @@ def exp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential of the given value. + Returns a column that evaluates to a double. Examples -------- @@ -3045,11 +3107,13 @@ def expm1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate exponential for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` exponential less one. + Returns a column that evaluates to a double. Examples -------- @@ -3098,8 +3162,10 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the floor on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionadded:: 4.0.0 @@ -3108,6 +3174,7 @@ def floor(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` nearest integer that is less than or equal to given value. + Returns a column that evaluates to a long or decimal. Examples -------- @@ -3153,11 +3220,13 @@ def log10(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logarithm of the given value in Base 10. + Returns a column that evaluates to a double. Examples -------- @@ -3206,11 +3275,13 @@ def log1p(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate natural logarithm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of the "given value plus one". + Returns a column that evaluates to a double. Examples -------- @@ -3245,11 +3316,13 @@ def negative(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to calculate negative value for. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` negative value. + Returns a column of the same type as the input. Examples -------- @@ -3300,11 +3373,13 @@ def positive(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input value column. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` value. + Returns a column of the same type as the input. Examples -------- @@ -3337,11 +3412,13 @@ def rint(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. Examples -------- @@ -3376,12 +3453,14 @@ def sec(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Angle in radians + Angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` Secant of the angle. + Returns a column that evaluates to a double. Examples -------- @@ -3428,11 +3507,13 @@ def signum(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3470,11 +3551,13 @@ def sign(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or interval. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a double. See Also -------- @@ -3512,11 +3595,13 @@ def sin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` sine of the angle, as if computed by `java.lang.Math.sin()` + Returns a column that evaluates to a double. Examples -------- @@ -3564,12 +3649,14 @@ def sinh(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh()` + Returns a column that evaluates to a double. Examples -------- @@ -3615,12 +3702,14 @@ def tan(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` tangent of the given value, as if computed by `java.lang.Math.tan()` + Returns a column that evaluates to a double. Examples -------- @@ -3667,13 +3756,15 @@ def tanh(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - hyperbolic angle + hyperbolic angle. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` hyperbolic tangent of the given value as if computed by `java.lang.Math.tanh()` + Returns a column that evaluates to a double. Examples -------- @@ -3810,12 +3901,14 @@ def bit_count(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral or boolean. Returns ------- :class:`~pyspark.sql.Column` the number of bits that are set in the argument expr as an unsigned 64-bit integer, or NULL if the argument is NULL. + Returns a column that evaluates to an integer. See Also -------- @@ -3853,13 +3946,16 @@ def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -3912,13 +4008,16 @@ def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. pos : :class:`~pyspark.sql.Column` or column name The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- @@ -4271,6 +4370,7 @@ def stddev(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. See Also -------- @@ -4310,6 +4410,7 @@ def std(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4353,6 +4454,7 @@ def stddev_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4394,6 +4496,7 @@ def stddev_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4434,6 +4537,7 @@ def variance(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4478,6 +4582,7 @@ def var_samp(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4518,6 +4623,7 @@ def var_pop(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -4556,8 +4662,10 @@ def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4646,8 +4754,10 @@ def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4736,8 +4846,10 @@ def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. See Also -------- @@ -4827,8 +4939,10 @@ def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -4917,8 +5031,10 @@ def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5007,8 +5123,10 @@ def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5097,8 +5215,10 @@ def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5187,8 +5307,10 @@ def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. See Also -------- @@ -5277,8 +5399,10 @@ def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: ---------- y : :class:`~pyspark.sql.Column` or column name the dependent variable. + A column that evaluates to a numeric. x : :class:`~pyspark.sql.Column` or column name the independent variable. + A column that evaluates to a numeric. Returns ------- @@ -5366,6 +5490,7 @@ def every(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. See Also -------- @@ -5422,6 +5547,7 @@ def bool_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if all values are true. + A column that evaluates to a boolean. Returns ------- @@ -5475,6 +5601,7 @@ def some(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5531,6 +5658,7 @@ def bool_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- @@ -5579,6 +5707,7 @@ def bit_and(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5652,6 +5781,7 @@ def bit_or(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5725,6 +5855,7 @@ def bit_xor(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to an integral. Returns ------- @@ -5801,6 +5932,7 @@ def skewness(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. See Also -------- @@ -5842,6 +5974,7 @@ def kurtosis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a numeric. Returns ------- @@ -6096,12 +6229,14 @@ def degrees(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in degrees, as if computed by `java.lang.Math.toDegrees()` + Returns a column that evaluates to a double. Examples -------- @@ -6135,12 +6270,14 @@ def radians(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in degrees + angle in degrees. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` angle in radians, as if computed by `java.lang.Math.toRadians()` + Returns a column that evaluates to a double. Examples -------- @@ -6174,9 +6311,11 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] Parameters ---------- col1 : :class:`~pyspark.sql.Column`, column name or float - coordinate on y-axis + coordinate on y-axis. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float - coordinate on x-axis + coordinate on x-axis. + A column that evaluates to a double. Returns ------- @@ -6186,6 +6325,7 @@ def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] in polar coordinates that corresponds to the point (`x`, `y`) in Cartesian coordinates, as if computed by `java.lang.Math.atan2()` + Returns a column that evaluates to a double. Examples -------- @@ -6214,13 +6354,16 @@ def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float] ---------- col1 : :class:`~pyspark.sql.Column`, column name or float a leg. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float b leg. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` length of the hypotenuse. + Returns a column that evaluates to a double. Examples -------- @@ -6249,13 +6392,16 @@ def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) ---------- col1 : :class:`~pyspark.sql.Column`, column name or float the base number. + A column that evaluates to a double. col2 : :class:`~pyspark.sql.Column`, column name or float the exponent number. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` the base rased to the power the argument. + Returns a column that evaluates to a double. Examples -------- @@ -6287,14 +6433,17 @@ def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", Parameters ---------- dividend : :class:`~pyspark.sql.Column`, column name or float - the column that contains dividend, or the specified dividend value + the column that contains dividend, or the specified dividend value. + A column that evaluates to a numeric. divisor : :class:`~pyspark.sql.Column`, column name or float - the column that contains divisor, or the specified divisor value + the column that contains divisor, or the specified divisor value. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` positive value of dividend mod divisor. + Returns a column of the same type as the input. Notes ----- @@ -6343,18 +6492,23 @@ def width_bucket( Parameters ---------- v : :class:`~pyspark.sql.Column` or column name - value to compute a bucket number in the histogram + value to compute a bucket number in the histogram. + A column that evaluates to a double or interval. min : :class:`~pyspark.sql.Column` or column name - minimum value of the histogram + minimum value of the histogram. + A column that evaluates to a double or interval. max : :class:`~pyspark.sql.Column` or column name - maximum value of the histogram + maximum value of the histogram. + A column that evaluates to a double or interval. numBucket : :class:`~pyspark.sql.Column`, column name or int - the number of buckets + the number of buckets. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` the bucket number into which the value would fall after being evaluated + Returns a column that evaluates to a long. Examples -------- @@ -6809,6 +6963,7 @@ def coalesce(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` value of the first column that is not null. + Returns a column of the same type as the input. Examples -------- @@ -6858,8 +7013,10 @@ def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate correlation. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate correlation. + A column that evaluates to a numeric. Returns ------- @@ -6896,8 +7053,10 @@ def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- @@ -6938,8 +7097,10 @@ def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to calculate covariance. + A column that evaluates to a numeric. col2 : :class:`~pyspark.sql.Column` or column name second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- @@ -7088,8 +7249,10 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch first value for. + A column of any type. ignorenulls : bool if first value is null then look for first non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7354,11 +7517,13 @@ def isnan(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` True if value is NaN and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7393,6 +7558,7 @@ def isnull(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` True if value is null and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -7430,8 +7596,10 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column to fetch last value for. + A column of any type. ignorenulls : bool if last value is null then look for non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -7533,13 +7701,16 @@ def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or column name first column to check. + A column that evaluates to a double or float. col2 : :class:`~pyspark.sql.Column` or column name second column to return if first is NaN. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` value from first column or second if first is NaN . + Returns a column of the same type as the first input. Examples -------- @@ -7772,11 +7943,13 @@ def rand(seed: Optional[int] = None) -> Column: ---------- seed : int, optional Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7834,11 +8007,13 @@ def randn(seed: Optional[int] = None) -> Column: ---------- seed : int (default: None) Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` A column of random values. + Returns a column that evaluates to a double. See Also -------- @@ -7890,8 +8065,10 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7900,6 +8077,7 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -7946,8 +8124,10 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or column name The target column or column name to compute the round on. + A column that evaluates to a numeric. scale : :class:`~pyspark.sql.Column` or int, optional An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. .. versionchanged:: 4.0.0 Support Column type. @@ -7956,6 +8136,7 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A column for the rounded value. + Returns a column of the same type as the input. Examples -------- @@ -8016,13 +8197,16 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column of the same type as the input. Examples -------- @@ -8071,13 +8255,16 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted values. + Returns a column of the same type as the input. Examples -------- @@ -8126,13 +8313,16 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to shift. + A column that evaluates to an integer or long. numBits : int number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` shifted value. + Returns a column of the same type as the input. Examples -------- @@ -8254,6 +8444,7 @@ def struct( ------- :class:`~pyspark.sql.Column` a struct type column of given columns. + Returns a column that evaluates to a struct. See Also -------- @@ -8330,6 +8521,7 @@ def greatest(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` greatest value. + Returns a column of the same type as the input. See Also -------- @@ -8374,6 +8566,7 @@ def least(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` least value. + Returns a column of the same type as the input. See Also -------- @@ -8420,6 +8613,7 @@ def when(condition: Column, value: Any) -> Column: ------- :class:`~pyspark.sql.Column` column representing when expression. + Returns a column of the same type as the input. See Also -------- @@ -8486,14 +8680,17 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non Parameters ---------- arg1 : :class:`~pyspark.sql.Column`, str or float - base number or actual number (in this case base is `e`) + base number or actual number (in this case base is `e`). + A column that evaluates to a double. arg2 : :class:`~pyspark.sql.Column`, str or float, optional number to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8560,11 +8757,13 @@ def ln(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` natural logarithm of given value. + Returns a column that evaluates to a double. See Also -------- @@ -8605,11 +8804,13 @@ def log2(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a double. Examples -------- @@ -8647,15 +8848,19 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to convert base for. + A column that evaluates to a string. fromBase: int from base number. + A column that evaluates to an integer. toBase: int to base number. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` logariphm of given value. + Returns a column that evaluates to a string. Examples -------- @@ -8691,11 +8896,13 @@ def factorial(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str a column to calculate factorial for. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` factorial of given value. + Returns a column that evaluates to a long. Examples -------- @@ -8990,8 +9197,10 @@ def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9036,8 +9245,10 @@ def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9089,8 +9300,10 @@ def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column of any type. ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- @@ -9142,6 +9355,7 @@ def count_if(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a boolean. Returns ------- @@ -9604,8 +9818,10 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to format. + A column that evaluates to a timestamp or time. format: literal string format to use to represent datetime values. + A column that evaluates to a string. See Also -------- @@ -9620,6 +9836,7 @@ def date_format(date: "ColumnOrName", format: str) -> Column: ------- :class:`~pyspark.sql.Column` string value representing formatted datetime. + Returns a column that evaluates to a string. Examples -------- @@ -9696,11 +9913,13 @@ def year(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` year part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9787,11 +10006,13 @@ def quarter(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` quarter of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9878,11 +10099,13 @@ def month(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` month part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -9971,11 +10194,13 @@ def dayofweek(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the week for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10057,6 +10282,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. See Also -------- @@ -10069,6 +10295,7 @@ def dayofmonth(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10140,11 +10367,13 @@ def day(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10236,11 +10465,13 @@ def dayofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` day of the year for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10326,11 +10557,13 @@ def hour(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` hour part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10408,6 +10641,7 @@ def minute(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. See Also -------- @@ -10425,6 +10659,7 @@ def minute(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` minutes part of the timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- @@ -10490,11 +10725,13 @@ def second(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` `seconds` part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10571,11 +10808,13 @@ def weekofyear(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` `week` of the year for given date as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10654,11 +10893,13 @@ def weekday(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + Returns a column that evaluates to an integer. See Also -------- @@ -10738,11 +10979,13 @@ def monthname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of month name for date/timestamp (Jan, Feb, Mar...) + Returns a column that evaluates to a string. See Also -------- @@ -10819,11 +11062,13 @@ def dayname(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target date/timestamp column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the three-letter abbreviation of day name for date/timestamp (Mon, Tue, Wed...) + Returns a column that evaluates to a string. See Also -------- @@ -10901,12 +11146,14 @@ def extract(field: Column, source: "ColumnOrName") -> Column: field : :class:`~pyspark.sql.Column` selects which part of the source should be extracted. source : :class:`~pyspark.sql.Column` or column name - a date/timestamp or interval column from where `field` should be extracted. + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. See Also -------- @@ -10956,12 +11203,14 @@ def date_part(field: Column, source: "ColumnOrName") -> Column: selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `extract`. source : :class:`~pyspark.sql.Column` or column name - a date/timestamp or interval column from where `field` should be extracted. + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. See Also -------- @@ -11011,12 +11260,14 @@ def datepart(field: Column, source: "ColumnOrName") -> Column: selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `extract`. source : :class:`~pyspark.sql.Column` or column name - a date/timestamp or interval column from where `field` should be extracted. + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. See Also -------- @@ -11066,16 +11317,20 @@ def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") Parameters ---------- year : :class:`~pyspark.sql.Column` or column name - The year to build the date + The year to build the date. + A column that evaluates to an integer. month : :class:`~pyspark.sql.Column` or column name - The month to build the date + The month to build the date. + A column that evaluates to an integer. day : :class:`~pyspark.sql.Column` or column name - The day to build the date + The day to build the date. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date built from given parts. + Returns a column that evaluates to a date. See Also -------- @@ -11112,14 +11367,17 @@ def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11172,14 +11430,17 @@ def dateadd(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days after the given date to calculate. Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11235,14 +11496,17 @@ def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. days : :class:`~pyspark.sql.Column` or column name or int how many days before the given date to calculate. Accepts negative value as well to calculate forward in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` a date before/after given number of days. + Returns a column that evaluates to a date. See Also -------- @@ -11295,13 +11559,16 @@ def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11343,13 +11610,16 @@ def date_diff(end: "ColumnOrName", start: "ColumnOrName") -> Column: ---------- end : :class:`~pyspark.sql.Column` or column name to date column to work on. + A column that evaluates to a date. start : :class:`~pyspark.sql.Column` or column name from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- @@ -11392,11 +11662,13 @@ def date_from_unix_date(days: "ColumnOrName") -> Column: ---------- days : :class:`~pyspark.sql.Column` or column name the target column to work on. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the date from the number of days since 1970-01-01. + Returns a column that evaluates to a date. See Also -------- @@ -11434,14 +11706,17 @@ def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Col ---------- start : :class:`~pyspark.sql.Column` or column name date column to work on. + A column that evaluates to a date. months : :class:`~pyspark.sql.Column` or column name or int how many months after the given date to calculate. Accepts negative value as well to calculate backwards. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` a date after/before given number of months. + Returns a column that evaluates to a date. See Also -------- @@ -11496,15 +11771,19 @@ def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool ---------- date1 : :class:`~pyspark.sql.Column` or column name first date column. + A column that evaluates to a timestamp. date2 : :class:`~pyspark.sql.Column` or column name second date column. + A column that evaluates to a timestamp. roundOff : bool, optional whether to round (to 8 digits) the final value or not (default: True). + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` number of months between two dates. + Returns a column that evaluates to a double. Examples -------- @@ -11556,13 +11835,16 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11613,13 +11895,16 @@ def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a string, date, or timestamp. format: literal string, optional format to use to convert date values. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. See Also -------- @@ -11675,11 +11960,13 @@ def unix_date(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` the number of days since 1970-01-01. + Returns a column that evaluates to an integer. See Also -------- @@ -11718,11 +12005,13 @@ def unix_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of microseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11762,11 +12051,13 @@ def unix_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of milliseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11857,11 +12148,13 @@ def unix_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name input column of values to convert. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` the number of seconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. See Also -------- @@ -11914,13 +12207,16 @@ def to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Col ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a time. See Also -------- @@ -11983,6 +12279,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -11990,6 +12287,7 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column: ------- :class:`~pyspark.sql.Column` timestamp value as :class:`pyspark.sql.types.TimestampType` type. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12056,13 +12354,16 @@ def try_to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> ---------- str : :class:`~pyspark.sql.Column` or column name string to be parsed to time. + A column that evaluates to a string. format: :class:`~pyspark.sql.Column` or column name, optional time format pattern to follow. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` time value as :class:`pyspark.sql.types.TimeType` type. + Returns a column that evaluates to a time. See Also -------- @@ -12123,6 +12424,7 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non ---------- col : :class:`~pyspark.sql.Column` or column name column values to convert. + A column that evaluates to a string, date, timestamp, or numeric. format: literal string, optional format to use to convert timestamp values. @@ -12518,15 +12820,18 @@ def trunc(date: "ColumnOrName", format: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a date. format : literal string 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - Other options are: 'week', 'quarter' + Other options are: 'week', 'quarter'. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` truncated date. + Returns a column that evaluates to a date. See Also -------- @@ -12573,14 +12878,17 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column: 'month', 'mon', 'mm' to truncate by month, 'day', 'dd' to truncate by day, Other options are: - 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter' + 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'. + A column that evaluates to a string. timestamp : :class:`~pyspark.sql.Column` or column name input column of values to truncate. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` truncated timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12625,14 +12933,17 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. dayOfWeek : literal string day of the week, case-insensitive, accepts: - "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column of computed results. + Returns a column that evaluates to a date. Examples -------- @@ -12671,11 +12982,13 @@ def last_day(date: "ColumnOrName") -> Column: ---------- date : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` last day of the month. + Returns a column that evaluates to a date. Examples -------- @@ -12716,13 +13029,16 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss" ---------- timestamp : :class:`~pyspark.sql.Column` or column name column of unix time values. + A column that evaluates to a long. format : literal string, optional - format to use to convert to (default: yyyy-MM-dd HH:mm:ss) + format to use to convert to (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` formatted timestamp as string. + Returns a column that evaluates to a string. See Also -------- @@ -12777,13 +13093,16 @@ def unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name, optional timestamps of string values. + A column that evaluates to a string, date, or timestamp. format : literal string, optional alternative format to use for converting (default: yyyy-MM-dd HH:mm:ss). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` unix time as long integer. + Returns a column that evaluates to a long. Examples -------- @@ -12854,7 +13173,8 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12862,6 +13182,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12870,6 +13191,7 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Col ------- :class:`~pyspark.sql.Column` timestamp value represented in given timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12923,7 +13245,8 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum Parameters ---------- timestamp : :class:`~pyspark.sql.Column` or column name - the column that contains timestamps + the column that contains timestamps. + A column that evaluates to a timestamp. tz : :class:`~pyspark.sql.Column` or literal string A string detailing the time zone ID that the input should be adjusted to. It should be in the format of either region-based zone IDs or zone offsets. Region IDs must @@ -12931,6 +13254,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + A column that evaluates to a string. .. versionchanged:: 2.4.0 `tz` can take a :class:`~pyspark.sql.Column` containing timezone ID strings. @@ -12939,6 +13263,7 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: Union[Column, str]) -> Colum ------- :class:`~pyspark.sql.Column` timestamp value represented in UTC timezone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -12983,11 +13308,13 @@ def timestamp_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13027,15 +13354,19 @@ def time_diff(unit: "ColumnOrName", start: "ColumnOrName", end: "ColumnOrName") unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or column name A starting time. + A column that evaluates to a time. end : :class:`~pyspark.sql.Column` or column name An ending time. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` The difference between two times, in the specified units. + Returns a column that evaluates to a long. See Also -------- @@ -13070,13 +13401,16 @@ def time_trunc(unit: "ColumnOrName", time: "ColumnOrName") -> Column: unit : :class:`~pyspark.sql.Column` or column name The unit to truncate the time to. Supported units are: "HOUR", "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + A column that evaluates to a string. time : :class:`~pyspark.sql.Column` or column name A time to truncate. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` A time truncated to the specified unit. + Returns a column that evaluates to a time. See Also -------- @@ -13110,11 +13444,13 @@ def timestamp_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13152,11 +13488,13 @@ def timestamp_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name unix time values. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` converted timestamp value. + Returns a column that evaluates to a timestamp. See Also -------- @@ -13195,6 +13533,7 @@ def timestamp_nanos(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name a column of ``BIGINT`` or ``DECIMAL`` nanosecond values since the UTC epoch. + A column that evaluates to an integral or decimal. Returns ------- @@ -13639,6 +13978,7 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str]) ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a struct. See Also -------- @@ -13698,8 +14038,10 @@ def to_unix_timestamp( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert UNIX timestamp values. + A column that evaluates to a string. See Also -------- @@ -13771,6 +14113,7 @@ def to_timestamp_ltz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, timestamp, or numeric. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampType` timestamp values. @@ -13842,8 +14185,10 @@ def to_timestamp_ntz( ---------- timestamp : :class:`~pyspark.sql.Column` or column name Input column or strings. + A column that evaluates to a string, date, or timestamp. format : :class:`~pyspark.sql.Column` or column name, optional format to use to convert type `TimestampNTZType` timestamp values. + A column that evaluates to a string. See Also -------- @@ -14133,11 +14478,13 @@ def crc32(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a long. .. versionadded:: 1.5.0 @@ -14168,11 +14515,13 @@ def md5(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. Examples -------- @@ -14201,11 +14550,13 @@ def sha1(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14241,14 +14592,17 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a binary. numBits : int the desired bit length of the result, which must have a value of 224, 256, 384, 512, or 0 (which is equivalent to 256). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a string. See Also -------- @@ -14298,6 +14652,7 @@ def hash(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as int column. + Returns a column that evaluates to an integer. See Also -------- @@ -14350,6 +14705,7 @@ def xxhash64(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hash value as long column. + Returns a column that evaluates to a long. See Also -------- @@ -14397,14 +14753,17 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column name or column that represents the input column to test + column name or column that represents the input column to test. + A column that evaluates to a boolean. errMsg : :class:`~pyspark.sql.Column` or literal string, optional - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` `null` if the input column is `true` otherwise throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14468,12 +14827,14 @@ def raise_error(errMsg: Union[Column, str]) -> Column: Parameters ---------- errMsg : :class:`~pyspark.sql.Column` or literal string - A Python string literal or column containing the error message + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` throws an error with specified message. + Returns a column that always evaluates to NULL. See Also -------- @@ -14517,11 +14878,13 @@ def upper(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` upper case values. + Returns a column that evaluates to a string. See Also -------- @@ -14558,11 +14921,13 @@ def lower(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` lower case values. + Returns a column that evaluates to a string. See Also -------- @@ -14599,11 +14964,13 @@ def ascii(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` numeric value. + Returns a column that evaluates to an integer. Examples -------- @@ -14635,11 +15002,13 @@ def base64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` BASE64 encoding of string value. + Returns a column that evaluates to a string. See Also -------- @@ -14675,11 +15044,13 @@ def unbase64(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - encoded string value. + decoded binary value. + Returns a column that evaluates to a binary. See Also -------- @@ -14715,8 +15086,10 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14724,6 +15097,7 @@ def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` left trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14790,8 +15164,10 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14799,6 +15175,7 @@ def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` right trimmed values. + Returns a column that evaluates to a string. See Also -------- @@ -14865,8 +15242,10 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. .. versionadded:: 4.0.0 @@ -14874,6 +15253,7 @@ def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ------- :class:`~pyspark.sql.Column` trimmed values from both sides. + Returns a column that evaluates to a string. See Also -------- @@ -14941,6 +15321,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ---------- sep : literal string words separator. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name list of columns to work on. @@ -14948,6 +15329,7 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` string of concatenated words. + Returns a column that evaluates to a string. See Also -------- @@ -15028,13 +15410,16 @@ def encode(col: "ColumnOrName", charset: str) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. charset : literal string charset to use to encode. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. + Returns a column that evaluates to a binary. See Also -------- @@ -15067,11 +15452,13 @@ def is_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` whether the input string is a valid UTF-8 string. + Returns a column that evaluates to a boolean. See Also -------- @@ -15104,11 +15491,13 @@ def make_valid_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the valid UTF-8 version of the given input string. + Returns a column that evaluates to a string. See Also -------- @@ -15140,11 +15529,13 @@ def validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, error otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15176,11 +15567,13 @@ def try_validate_utf8(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the input string if it is a valid UTF-8 string, null otherwise. + Returns a column that evaluates to a string. See Also -------- @@ -15215,14 +15608,17 @@ def format_number(col: "ColumnOrName", d: int) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - the column name of the numeric value to be formatted + the column name of the numeric value to be formatted. + A column that evaluates to a numeric. d : int - the N decimal places + the N decimal places. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. Examples -------- @@ -15253,7 +15649,8 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: Parameters ---------- format : literal string - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -15261,6 +15658,7 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` the column of formatted results. + Returns a column that evaluates to a string. See Also -------- @@ -15313,21 +15711,26 @@ def instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. substr : :class:`~pyspark.sql.Column` or literal string substring to look for. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `substr` now accepts column. start : int or :class:`~pyspark.sql.Column`, optional Starting position (1-based, can be negative for backward search). If not specified, defaults to 1. + A column that evaluates to an integer. occurrence : int or :class:`~pyspark.sql.Column`, optional Which occurrence to locate (must be > 0). Defaults to 1. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` location of the substring as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -15416,19 +15819,24 @@ def overlay( Parameters ---------- src : :class:`~pyspark.sql.Column` or column name - the string that will be replaced + the string that will be replaced. + A column that evaluates to a string or binary. replace : :class:`~pyspark.sql.Column` or column name - the substitution string + the substitution string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int - the starting position in src + the starting position in src. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name or int, optional the number of bytes to replace in src - string by 'replace' defaults to -1, which represents the length of the 'replace' string + string by 'replace' defaults to -1, which represents the length of the 'replace' string. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` string with replaced values. + Returns a column of the same type as the input. Examples -------- @@ -15516,16 +15924,20 @@ def sentences( Parameters ---------- string : :class:`~pyspark.sql.Column` or column name - a string to be split + a string to be split. + A column that evaluates to a string. language : :class:`~pyspark.sql.Column` or column name, optional - a language of the locale + a language of the locale. + A column that evaluates to a string. country : :class:`~pyspark.sql.Column` or column name, optional - a country of the locale + a country of the locale. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` arrays of split sentences. + Returns a column that evaluates to an array. See Also -------- @@ -15589,14 +16001,17 @@ def substring( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name or int starting position in str. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pos` now accepts column and column name. len : :class:`~pyspark.sql.Column` or column name or int length of chars. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `len` now accepts column and column name. @@ -15605,6 +16020,7 @@ def substring( ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -15694,15 +16110,19 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. delim : literal string delimiter of values. + A column that evaluates to a string. count : int number of occurrences. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column that evaluates to a string. See Also -------- @@ -15752,11 +16172,14 @@ def levenshtein( ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. threshold : int, optional if set when the levenshtein distance of the two given strings - less than or equal to a given threshold then return result distance, or -1 + less than or equal to a given threshold then return result distance, or -1. + A column that evaluates to an integer. .. versionadded:: 3.5.0 @@ -15764,6 +16187,7 @@ def levenshtein( ------- :class:`~pyspark.sql.Column` Levenshtein distance as integer value. + Returns a column that evaluates to an integer. Examples -------- @@ -15805,13 +16229,16 @@ def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Colu ---------- left : :class:`~pyspark.sql.Column` or column name first column value. + A column that evaluates to a string. right : :class:`~pyspark.sql.Column` or column name second column value. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` Jaro-Winkler similarity as a double value. + Returns a column that evaluates to a double. Examples -------- @@ -15840,16 +16267,20 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: Parameters ---------- substr : literal string - a string + a string. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or column name - a Column of :class:`pyspark.sql.types.StringType` + a Column of :class:`pyspark.sql.types.StringType`. + A column that evaluates to a string. pos : int, optional - start position (zero based) + start position (zero based). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` position of the substring. + Returns a column that evaluates to an integer. Notes ----- @@ -15907,14 +16338,17 @@ def lpad( ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15923,6 +16357,7 @@ def lpad( ------- :class:`~pyspark.sql.Column` left padded result. + Returns a column of the same type as the input. See Also -------- @@ -15977,14 +16412,17 @@ def rpad( ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or int length of the final string. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `pattern` now accepts column. pad : :class:`~pyspark.sql.Column` or literal string chars to prepend. + A column that evaluates to a string or binary. .. versionchanged:: 4.0.0 `pattern` now accepts column. @@ -15993,6 +16431,7 @@ def rpad( ------- :class:`~pyspark.sql.Column` right padded result. + Returns a column of the same type as the input. See Also -------- @@ -16043,8 +16482,10 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. n : :class:`~pyspark.sql.Column` or column name or int number of times to repeat value. + A column that evaluates to an integer. .. versionchanged:: 4.0.0 `n` now accepts column and column name. @@ -16053,6 +16494,7 @@ def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: ------- :class:`~pyspark.sql.Column` string with repeated values. + Returns a column that evaluates to a string. Examples -------- @@ -16108,10 +16550,12 @@ def split( Parameters ---------- str : :class:`~pyspark.sql.Column` or column name - a string expression to split + a string expression to split. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or literal string a string representing a regular expression. The regex string should be a Java regular expression. + A column that evaluates to a string. .. versionchanged:: 4.0.0 `pattern` now accepts column. Does not accept column name since string type remain @@ -16120,6 +16564,7 @@ def split( limit : :class:`~pyspark.sql.Column` or column name or int an integer which controls the number of times `pattern` is applied. + A column that evaluates to an integer. * ``limit > 0``: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last @@ -16134,6 +16579,7 @@ def split( ------- :class:`~pyspark.sql.Column` array of separated strings. + Returns a column that evaluates to an array. See Also -------- @@ -16207,13 +16653,16 @@ def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16260,13 +16709,16 @@ def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16313,13 +16765,16 @@ def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or str regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -16368,13 +16823,16 @@ def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = Non ---------- length : :class:`~pyspark.sql.Column` or int Number of characters in the string to generate. + A column that evaluates to an integer. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random string with the specified length. + Returns a column that evaluates to a string. See Also -------- @@ -16421,13 +16879,16 @@ def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the number of times that a Java regex pattern is matched in the string. + Returns a column that evaluates to an integer. Examples -------- @@ -16478,15 +16939,19 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. pattern : str regex pattern to apply. + A column that evaluates to a string. idx : int matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` matched value specified by `idx` group id. + Returns a column that evaluates to a string. See Also -------- @@ -16539,15 +17004,19 @@ def regexp_extract_all( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` all strings in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an array. See Also -------- @@ -16617,18 +17086,23 @@ def regexp_replace( Parameters ---------- string : :class:`~pyspark.sql.Column` or str - column name or column containing the string value + column name or column containing the string value. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str - column object or str containing the regexp pattern + column object or str containing the regexp pattern. + A column that evaluates to a string. replacement : :class:`~pyspark.sql.Column` or str - column object or str containing the replacement + column object or str containing the replacement. + A column that evaluates to a string. position : :class:`~pyspark.sql.Column` or int, optional position to start replacement. The first position is 1. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` string with all substrings replaced. + Returns a column that evaluates to a string. Examples -------- @@ -16697,13 +17171,16 @@ def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the first substring that matches a Java regex within the string `str`. + Returns a column that evaluates to a string. Examples -------- @@ -16766,16 +17243,20 @@ def regexp_instr( ---------- str : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. + A column that evaluates to a string. idx : :class:`~pyspark.sql.Column` or int, optional matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` the position of the first substring in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an integer. Examples -------- @@ -16841,11 +17322,13 @@ def initcap(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string with all first letters are uppercase in each word. + Returns a column that evaluates to a string. Examples -------- @@ -16875,11 +17358,13 @@ def soundex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` SoundEx encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -16909,11 +17394,13 @@ def bin(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` binary representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16952,6 +17439,7 @@ def hex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a long, binary, or string. See Also -------- @@ -16961,6 +17449,7 @@ def hex(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` hexadecimal representation of given value as string. + Returns a column that evaluates to a string. Examples -------- @@ -16990,6 +17479,7 @@ def unhex(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string. See Also -------- @@ -16998,7 +17488,8 @@ def unhex(col: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - string representation of given hexadecimal value. + byte representation of the given hexadecimal value. + Returns a column that evaluates to a binary. Examples -------- @@ -17032,15 +17523,19 @@ def uniform( ---------- min : :class:`~pyspark.sql.Column`, int, or float Minimum value in the range. + A column that evaluates to a numeric. Must be a constant. max : :class:`~pyspark.sql.Column`, int, or float Maximum value in the range. + A column that evaluates to a numeric. Must be a constant. seed : :class:`~pyspark.sql.Column` or int Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The generated random number within the specified range. + Returns a column of the same type as the input. See Also -------- @@ -17093,11 +17588,13 @@ def length(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to work on. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` length of the value. + Returns a column that evaluates to an integer. See Also -------- @@ -17130,12 +17627,14 @@ def octet_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Byte length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17165,12 +17664,14 @@ def bit_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` Bit length of the col + Returns a column that evaluates to an integer. Examples -------- @@ -17202,17 +17703,21 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: Parameters ---------- srcCol : :class:`~pyspark.sql.Column` or column name - Source column or strings + Source column or strings. + A column that evaluates to a string. matching : str matching characters. + A column that evaluates to a string. replace : str characters for replacement. If this is shorter than `matching` string then those chars that don't have replacement will be dropped. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` replaced value. + Returns a column that evaluates to a string. Examples -------- @@ -17247,8 +17752,10 @@ def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17309,9 +17816,11 @@ def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17355,9 +17864,11 @@ def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert char values. + A column that evaluates to a string. Must be a constant. Examples -------- @@ -17397,8 +17908,10 @@ def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -17426,11 +17939,14 @@ def replace( ---------- src : :class:`~pyspark.sql.Column` or str A column of string to be replaced. + A column that evaluates to a string. search : :class:`~pyspark.sql.Column` or str A column of string, If `search` is not found in `str`, `str` is returned unchanged. + A column that evaluates to a string. replace : :class:`~pyspark.sql.Column` or str, optional A column of string, If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + A column that evaluates to a string. Examples -------- @@ -17462,10 +17978,13 @@ def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnO ---------- src : :class:`~pyspark.sql.Column` or column name A column of string to be split. + A column that evaluates to a string. delimiter : :class:`~pyspark.sql.Column` or column name A column of string, the delimiter used for split. + A column that evaluates to a string. partNum : :class:`~pyspark.sql.Column` or column name - A column of string, requested part of the split (1-based). + The requested part of the split (1-based). + A column that evaluates to an integer. See Also -------- @@ -17507,15 +18026,19 @@ def substr( ---------- str : :class:`~pyspark.sql.Column` or column name A column of string. + A column that evaluates to a string or binary. pos : :class:`~pyspark.sql.Column` or column name - A column of string, the substring of `str` that starts at `pos`. + The starting position of the substring. + A column that evaluates to an integer. len : :class:`~pyspark.sql.Column` or column name, optional - A column of string, the substring of `str` is of length `len`. + The length of the substring. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` substring of given value. + Returns a column of the same type as the input. See Also -------- @@ -17563,15 +18086,19 @@ def try_parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17679,15 +18206,19 @@ def parse_url( ---------- url : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL. + A column that evaluates to a string. partToExtract : :class:`~pyspark.sql.Column` or str A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. key : :class:`~pyspark.sql.Column` or str, optional A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- @@ -17777,7 +18308,8 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: Parameters ---------- format : :class:`~pyspark.sql.Column` or str - string that can contain embedded format tags and used as result column's value + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or str column names or :class:`~pyspark.sql.Column`\\s to be used in formatting @@ -17815,11 +18347,13 @@ def url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17893,11 +18427,13 @@ def try_url_decode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- @@ -17938,11 +18474,13 @@ def url_encode(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str A column of strings, each representing a string to be URL-encoded. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of strings, each representing the URL-encoded string. + Returns a column that evaluates to a string. Examples -------- @@ -18018,10 +18556,13 @@ def position( ---------- substr : :class:`~pyspark.sql.Column` or str A column of string, substring. + A column that evaluates to a string. str : :class:`~pyspark.sql.Column` or str A column of string. + A column that evaluates to a string. start : :class:`~pyspark.sql.Column` or str, optional - A column of string, start position. + The start position. + A column that evaluates to an integer. Examples -------- @@ -18062,9 +18603,11 @@ def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. suffix : :class:`~pyspark.sql.Column` or str - A column of string, the suffix. + The suffix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18100,9 +18643,11 @@ def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: Parameters ---------- str : :class:`~pyspark.sql.Column` or str - A column of string. + The input value to test. + A column that evaluates to a string or binary. prefix : :class:`~pyspark.sql.Column` or str - A column of string, the prefix. + The prefix to test for. + A column that evaluates to a string or binary. Examples -------- @@ -18138,6 +18683,7 @@ def char(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a long. Examples -------- @@ -18163,8 +18709,10 @@ def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. Examples -------- @@ -18195,6 +18743,7 @@ def char_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18227,6 +18776,7 @@ def character_length(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. See Also -------- @@ -18258,6 +18808,7 @@ def chr(n: "ColumnOrName") -> Column: ---------- n : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a long. Examples -------- @@ -18293,8 +18844,10 @@ def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18351,8 +18904,10 @@ def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. format : :class:`~pyspark.sql.Column` or str, optional format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- @@ -18402,9 +18957,11 @@ def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: Parameters ---------- left : :class:`~pyspark.sql.Column` or str - The input column or strings to check, may be NULL. + The input to check; may be NULL. + A column that evaluates to a string or binary. right : :class:`~pyspark.sql.Column` or str - The input column or strings to find, may be NULL. + The value to find; may be NULL. + A column that evaluates to a string or binary. Examples -------- @@ -18468,8 +19025,10 @@ def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str The given string to be found. + A column that evaluates to a string. str_array : :class:`~pyspark.sql.Column` or str The comma-delimited list. + A column that evaluates to a string. Examples -------- @@ -18495,6 +19054,7 @@ def like( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18506,6 +19066,7 @@ def like( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18545,6 +19106,7 @@ def ilike( ---------- str : :class:`~pyspark.sql.Column` or str A string. + A column that evaluates to a string. pattern : :class:`~pyspark.sql.Column` or str A string. The pattern is a string which is matched literally, with exception to the following special symbols: @@ -18556,6 +19118,7 @@ def ilike( When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back to Spark 1.6 behavior regarding string literal parsing. For example, if the config is enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. escapeChar : :class:`~pyspark.sql.Column`, optional An character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the @@ -18591,6 +19154,7 @@ def lcase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18622,6 +19186,7 @@ def ucase(str: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. See Also -------- @@ -18654,8 +19219,10 @@ def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string or binary. len : :class:`~pyspark.sql.Column` or str Input column or strings, the leftmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18678,8 +19245,10 @@ def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: ---------- str : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. len : :class:`~pyspark.sql.Column` or str Input column or strings, the rightmost `len`. + A column that evaluates to an integer. Examples -------- @@ -18708,18 +19277,24 @@ def mask( ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. upperChar: :class:`~pyspark.sql.Column` or str, optional character to replace upper-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. lowerChar: :class:`~pyspark.sql.Column` or str, optional character to replace lower-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. digitChar: :class:`~pyspark.sql.Column` or str, optional character to replace digit characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. otherChar: :class:`~pyspark.sql.Column` or str, optional character to replace all other characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` + Returns a column that evaluates to a string. Examples -------- @@ -18780,11 +19355,13 @@ def collation(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Target string column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` collation name of a given expression. + Returns a column that evaluates to a string. Examples -------- @@ -18810,11 +19387,13 @@ def quote(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to be quoted. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` quoted string + Returns a column that evaluates to a string. Examples -------- @@ -18942,13 +19521,16 @@ def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing a set of keys. All elements should not be null. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing a set of values. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A column of map type. + Returns a column that evaluates to a map. Notes ----- @@ -19028,6 +19610,7 @@ def array( :class:`~pyspark.sql.Column` A new Column of array type, where each value is an array containing the corresponding values from the input columns. + Returns a column that evaluates to an array. Examples -------- @@ -19118,14 +19701,17 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The target column containing the arrays. + A column that evaluates to an array. value : The value or column to check for in the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding array from the input column contains the specified value. + Returns a column that evaluates to a boolean. Examples -------- @@ -19205,6 +19791,7 @@ def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: :class:`~pyspark.sql.Column` A new Column of Boolean type, where each value indicates whether the corresponding arrays from the input columns contain any common elements. + Returns a column that evaluates to a boolean. Examples -------- @@ -19277,17 +19864,21 @@ def slice( ---------- x : :class:`~pyspark.sql.Column` or str Input array column or column name to be sliced. + A column that evaluates to an array. start : :class:`~pyspark.sql.Column`, str, or int The start index for the slice operation. If negative, starts the index from the end of the array. + A column that evaluates to an integer. length : :class:`~pyspark.sql.Column`, str, or int The length of the slice, representing number of elements in the resulting array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new Column object of Array type, where each value is a slice of the corresponding list from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -19354,16 +19945,20 @@ def array_join( ---------- col : :class:`~pyspark.sql.Column` or str The input column containing the arrays to be joined. + A column that evaluates to an array. delimiter : str The string to be used as the delimiter when joining the array elements. + A column that evaluates to a string. null_replacement : str, optional The string to replace null values within the array. If not set, null values are ignored. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of string type, where each value is the result of joining the corresponding array from the input column. + Returns a column that evaluates to a string. Examples -------- @@ -19461,6 +20056,7 @@ def concat(*cols: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` concatenated values. Type of the `Column` depends on input columns' type. + Returns a column of the same type as the input. See Also -------- @@ -19548,8 +20144,10 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to work on. + A column that evaluates to an array. value : Any value or a :class:`~pyspark.sql.Column` expression to look for. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `value` now also accepts a Column type. @@ -19558,6 +20156,7 @@ def array_position(col: "ColumnOrName", value: Any) -> Column: ------- :class:`~pyspark.sql.Column` position of the value in the given array if found and 0 otherwise. + Returns a column that evaluates to a long. Examples -------- @@ -19653,14 +20252,17 @@ def element_at(col: "ColumnOrName", extraction: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` value at given position. + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -19749,9 +20351,16 @@ def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array or map + name of column containing array or map. + A column that evaluates to an array or map. extraction : - index to check for in array or key to check for in map + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. + + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- @@ -19848,13 +20457,16 @@ def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column containing the array. + A column that evaluates to an array. index : :class:`~pyspark.sql.Column` or str or int Index to check for in the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` Value at the given position. + Returns a column of the element type of the input array. Notes ----- @@ -19941,14 +20553,17 @@ def array_prepend(col: "ColumnOrName", value: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` an array with the given value prepended. + Returns a column that evaluates to an array. Examples -------- @@ -20031,9 +20646,11 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array + name of column containing array. + A column that evaluates to an array. element : - element or a :class:`~pyspark.sql.Column` expression to be removed from the array + element or a :class:`~pyspark.sql.Column` expression to be removed from the array. + A column of the same type as the array elements. .. versionchanged:: 4.0.0 `element` now also accepts a Column type. @@ -20042,6 +20659,7 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column: ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the given value from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20132,12 +20750,14 @@ def array_distinct(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array of unique values from the input column. + Returns a column that evaluates to an array. Examples -------- @@ -20217,17 +20837,21 @@ def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: An Parameters ---------- arr : :class:`~pyspark.sql.Column` or str - name of column containing an array + name of column containing an array. + A column that evaluates to an array. pos : :class:`~pyspark.sql.Column` or str or int - name of Numeric type column indicating position of insertion - (starting at index 1, negative position is a start from the back of the array) + name of integral type column indicating position of insertion + (starting at index 1, negative position is a start from the back of the array). + A column that evaluates to an integer. value : a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` an array of values, including the new specified value + Returns a column that evaluates to an array. Notes ----- @@ -20313,13 +20937,16 @@ def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the intersection of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20406,13 +21033,16 @@ def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the union of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- @@ -20499,13 +21129,16 @@ def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: ---------- col1 : :class:`~pyspark.sql.Column` or str Name of column containing the first array. + A column that evaluates to an array. col2 : :class:`~pyspark.sql.Column` or str Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new array containing the elements present in col1 but not in col2. + Returns a column that evaluates to an array. Notes ----- @@ -20587,12 +21220,14 @@ def array_compact(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that is an array excluding the null values from the input column. + Returns a column that evaluates to an array. Notes ----- @@ -20678,13 +21313,16 @@ def array_append(col: "ColumnOrName", value: Any) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column containing the array. + A column that evaluates to an array. value : A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` A new array column with `value` appended to the original array. + Returns a column that evaluates to an array. Notes ----- @@ -20779,6 +21417,8 @@ def explode(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` One row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. See Also -------- @@ -21093,6 +21733,8 @@ def explode_outer(col: "ColumnOrName") -> Column: ------- :class:`~pyspark.sql.Column` one row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. See Also -------- @@ -21275,14 +21917,17 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - string column in json format + string column in json format. + A column that evaluates to a string. path : str - path to the json object to extract + path to the json object to extract. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` string representation of given JSON object value. + Returns a column that evaluates to a string. Examples -------- @@ -21355,6 +22000,7 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column: ------- :class:`~pyspark.sql.Column` a new row for each given field value from json object + Returns a column that evaluates to a string. Examples -------- @@ -21508,12 +22154,14 @@ def try_parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21546,6 +22194,7 @@ def to_variant_object( ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21588,12 +22237,14 @@ def parse_json( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- @@ -21952,6 +22603,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ---------- col : :class:`~pyspark.sql.Column` or str name of column containing a struct, an array, a map, or a variant object. + A column that evaluates to a struct, array, map, or variant. options : dict, optional options to control converting. accepts the same options as the JSON datasource. See `Data Source Option `_ @@ -21965,6 +22617,7 @@ def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> ------- :class:`~pyspark.sql.Column` JSON object as string column. + Returns a column that evaluates to a string. Examples -------- @@ -22125,11 +22778,13 @@ def json_array_length(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` length of json array. + Returns a column that evaluates to an integer. Examples -------- @@ -22153,11 +22808,13 @@ def json_object_keys(col: "ColumnOrName") -> Column: ---------- col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` all the keys of the outermost JSON object. + Returns a column that evaluates to an array. Examples -------- @@ -22620,12 +23277,14 @@ def size(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22650,11 +23309,13 @@ def array_min(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the minimum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22735,11 +23396,13 @@ def array_max(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the maximum value of each array. + Returns a column of the element type of the input array. Examples -------- @@ -22818,11 +23481,13 @@ def array_size(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains the size of each array. + Returns a column that evaluates to an integer. Examples -------- @@ -22900,11 +23565,13 @@ def cardinality(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` length of the array/map. + Returns a column that evaluates to an integer. Examples -------- @@ -22940,14 +23607,17 @@ def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: ---------- col : :class:`~pyspark.sql.Column` or str Name of the column or expression. + A column that evaluates to an array. asc : bool, optional Whether to sort in ascending or descending order. If `asc` is True (default), then the sorting is in ascending order. If False, then in descending order. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` Sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23034,7 +23704,8 @@ def array_sort( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. comparator : callable, optional A binary ``(Column, Column) -> Column: ...``. The comparator will take two @@ -23046,6 +23717,7 @@ def array_sort( ------- :class:`~pyspark.sql.Column` sorted array. + Returns a column that evaluates to an array. Examples -------- @@ -23079,6 +23751,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or expression to be shuffled. + A column that evaluates to an array. seed : :class:`~pyspark.sql.Column` or int, optional Seed value for the random generator. @@ -23088,6 +23761,7 @@ def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> C ------- :class:`~pyspark.sql.Column` A new column that contains an array of elements in random order. + Returns a column that evaluates to an array. Notes ----- @@ -23164,12 +23838,14 @@ def reverse(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be reversed. + A column that evaluates to a string, binary, or array. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a reversed string, a binary value with bytes in reverse order, or an array with elements in reverse order. + Returns a column of the same type as the input. Examples -------- @@ -23665,14 +24341,17 @@ def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Colu ---------- col : :class:`~pyspark.sql.Column` or str The name of the column or an expression that represents the element to be repeated. + A column of any type. count : :class:`~pyspark.sql.Column` or str or int The name of the column, an expression, or an integer that represents the number of times to repeat the element. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an array of repeated elements. + Returns a column that evaluates to an array. Examples -------- @@ -23746,11 +24425,13 @@ def arrays_zip(*cols: "ColumnOrName") -> Column: ---------- cols : :class:`~pyspark.sql.Column` or str Columns of arrays to be merged. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` Merged array of entries. + Returns a column that evaluates to an array. Examples -------- @@ -23931,6 +24612,7 @@ def sequence( ------- :class:`~pyspark.sql.Column` A new column that contains an array of sequence values. + Returns a column that evaluates to an array. Examples -------- @@ -24218,7 +24900,8 @@ def transform( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function a function that is applied to each element of the input array. Can take one of the following forms: @@ -24236,6 +24919,7 @@ def transform( ------- :class:`~pyspark.sql.Column` a new array of transformed elements. + Returns a column that evaluates to an array. Examples -------- @@ -24273,7 +24957,8 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24286,6 +24971,7 @@ def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "any" element of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24314,7 +25000,8 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function ``(x: Column) -> Column: ...`` returning the Boolean expression. Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24327,6 +25014,7 @@ def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: :class:`~pyspark.sql.Column` True if "all" elements of an array evaluates to True when passed as an argument to given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- @@ -24370,7 +25058,8 @@ def filter( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. f : function A function that returns the Boolean expression. Can take one of the following forms: @@ -24389,6 +25078,7 @@ def filter( :class:`~pyspark.sql.Column` filtered array of elements where given function evaluated to True when passed as an argument. + Returns a column that evaluates to an array. Examples -------- @@ -24436,12 +25126,14 @@ def aggregate( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``initialValue`` + of the same type as ``initialValue``. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. @@ -24509,12 +25201,14 @@ def reduce( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression + name of column or expression. + A column that evaluates to an array. initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression + initial value. Name of column or expression. + A column of any type. merge : function a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``zero`` + of the same type as ``zero``. finish : function, optional an optional unary function ``(x: Column) -> Column: ...`` used to convert accumulated value. @@ -24579,9 +25273,11 @@ def zip_with( Parameters ---------- left : :class:`~pyspark.sql.Column` or str - name of the first column or expression + name of the first column or expression. + A column that evaluates to an array. right : :class:`~pyspark.sql.Column` or str - name of the second column or expression + name of the second column or expression. + A column that evaluates to an array. f : function a binary function ``(x1: Column, x2: Column) -> Column...`` Can use methods of :class:`~pyspark.sql.Column`, functions defined in @@ -24593,6 +25289,7 @@ def zip_with( ------- :class:`~pyspark.sql.Column` array of calculated values derived by applying given function to each pair of arguments. + Returns a column that evaluates to an array. Examples -------- @@ -24855,15 +25552,19 @@ def str_to_map( ---------- text : :class:`~pyspark.sql.Column` or str Input column or strings. + A column that evaluates to a string. pairDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split pairs. Default is comma (,). + A column that evaluates to a string. keyValueDelim : :class:`~pyspark.sql.Column` or str, optional Delimiter to use to split key/value. Default is colon (:). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` A new column of map type where each string in the original column is converted into a map. + Returns a column that evaluates to a map. Examples -------- @@ -25129,15 +25830,19 @@ def convert_timezone( sourceTz : :class:`~pyspark.sql.Column`, optional The time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + A column that evaluates to a string. targetTz : :class:`~pyspark.sql.Column` The time zone to which the input timestamp should be converted. + A column that evaluates to a string. sourceTs : :class:`~pyspark.sql.Column` or column name A timestamp without time zone. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp for converted time zone. + Returns a column that evaluates to a timestamp. See Also -------- @@ -25199,17 +25904,22 @@ def make_dt_interval( ---------- days : :class:`~pyspark.sql.Column` or column name, optional The number of days, positive or negative. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The number of hours, positive or negative. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The number of minutes, positive or negative. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a DayTimeIntervalType duration. + Returns a column that evaluates to an interval. See Also -------- @@ -25300,6 +26010,7 @@ def try_make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25317,6 +26028,7 @@ def try_make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25467,6 +26179,7 @@ def make_interval( ---------- years : :class:`~pyspark.sql.Column` or column name, optional The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The number of months, positive or negative. weeks : :class:`~pyspark.sql.Column` or column name, optional @@ -25484,6 +26197,7 @@ def make_interval( ------- :class:`~pyspark.sql.Column` A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- @@ -25616,15 +26330,19 @@ def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrNam ---------- hour : :class:`~pyspark.sql.Column` or column name The hour to represent, from 0 to 23. + A column that evaluates to an integer. minute : :class:`~pyspark.sql.Column` or column name The minute to represent, from 0 to 59. + A column that evaluates to an integer. second : :class:`~pyspark.sql.Column` or column name The second to represent, from 0 to 59.999999. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` A column representing the created time. + Returns a column that evaluates to a time. Examples -------- @@ -25651,6 +26369,7 @@ def time_from_seconds(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Seconds since midnight (0 to 86399.999999). + A column that evaluates to a numeric. Examples -------- @@ -25677,6 +26396,7 @@ def time_from_millis(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Milliseconds since midnight (0 to 86399999). + A column that evaluates to an integral. Examples -------- @@ -25703,6 +26423,7 @@ def time_from_micros(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name Microseconds since midnight (0 to 86399999999). + A column that evaluates to an integral. Examples -------- @@ -25872,22 +26593,27 @@ def make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -25895,6 +26621,7 @@ def make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -25910,6 +26637,7 @@ def make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26097,22 +26825,27 @@ def try_make_timestamp( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26120,6 +26853,7 @@ def try_make_timestamp( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC, and etc. date : :class:`~pyspark.sql.Column` or column name, optional @@ -26135,6 +26869,7 @@ def try_make_timestamp( ------- :class:`~pyspark.sql.Column` A new column that contains a timestamp or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26294,20 +27029,26 @@ def make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or str - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or str - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or str - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or str - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or str - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or str The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or str, optional The time zone identifier. For example, CET, UTC and etc. @@ -26315,6 +27056,7 @@ def make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26391,20 +27133,26 @@ def try_make_timestamp_ltz( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name - The year to represent, from 1 to 9999 + The year to represent, from 1 to 9999. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name - The month-of-year to represent, from 1 (January) to 12 (December) + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name - The day-of-month to represent, from 1 to 31 + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name - The hour-of-day to represent, from 0 to 23 + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name - The minute-of-hour to represent, from 0 to 59 + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. timezone : :class:`~pyspark.sql.Column` or column name, optional The time zone identifier. For example, CET, UTC and etc. @@ -26412,6 +27160,7 @@ def try_make_timestamp_ltz( ------- :class:`~pyspark.sql.Column` A new column that contains a current timestamp, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26528,22 +27277,27 @@ def make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26551,6 +27305,7 @@ def make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26564,6 +27319,7 @@ def make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26683,22 +27439,27 @@ def try_make_timestamp_ntz( The year to represent, from 1 to 9999. Required when creating timestamps from individual components. Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional The month-of-year to represent, from 1 (January) to 12 (December). Required when creating timestamps from individual components. Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. days : :class:`~pyspark.sql.Column` or column name, optional The day-of-month to represent, from 1 to 31. Required when creating timestamps from individual components. Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. hours : :class:`~pyspark.sql.Column` or column name, optional The hour-of-day to represent, from 0 to 23. Required when creating timestamps from individual components. Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. mins : :class:`~pyspark.sql.Column` or column name, optional The minute-of-hour to represent, from 0 to 59. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. secs : :class:`~pyspark.sql.Column` or column name, optional The second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13, or a fraction like 13.123. @@ -26706,6 +27467,7 @@ def try_make_timestamp_ntz( to 0 and 1 minute is added to the final timestamp. Required when creating timestamps from individual components. Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. date : :class:`~pyspark.sql.Column` or column name, optional The date to represent, in valid DATE format. Required when creating timestamps from date and time components. @@ -26719,6 +27481,7 @@ def try_make_timestamp_ntz( ------- :class:`~pyspark.sql.Column` A new column that contains a local date-time, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- @@ -26804,14 +27567,17 @@ def make_ym_interval( Parameters ---------- years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative + The number of years, positive or negative. + A column that evaluates to an integer. months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative + The number of months, positive or negative. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a year-month interval. + Returns a column that evaluates to an interval. See Also -------- @@ -26972,6 +27738,7 @@ def st_geogfromwkb(wkb: "ColumnOrName") -> Column: ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOGRAPHY value. + A column that evaluates to a binary. Examples -------- @@ -26995,8 +27762,10 @@ def st_geomfromwkb( ---------- wkb : :class:`~pyspark.sql.Column` or str A BINARY value in WKB format, representing a GEOMETRY value. + A column that evaluates to a binary. srid : :class:`~pyspark.sql.Column` or int, optional The optional SRID value of the geometry. Default is 0. + A column that evaluates to an integer. Examples -------- @@ -27276,8 +28045,10 @@ def hll_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an integer, long, string, or binary. lgConfigK : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of K, where K is the number of buckets or slots for the HllSketch + The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. + A column that evaluates to an integer. Returns ------- @@ -27482,9 +28253,11 @@ def theta_sketch_agg( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an array, binary, double, float, integer, long, or string. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries, where nominal entries is the size of the sketch - (must be between 4 and 26, defaults to 12) + (must be between 4 and 26, defaults to 12). + A column that evaluates to an integer. Returns ------- @@ -27634,9 +28407,11 @@ def tuple_sketch_agg_double( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing double summary values + The column containing double summary values. + A column that evaluates to a double. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27688,9 +28463,11 @@ def tuple_sketch_agg_integer( Parameters ---------- key : :class:`~pyspark.sql.Column` or column name - The column containing key values + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. summary : :class:`~pyspark.sql.Column` or column name - The column containing integer summary values + The column containing integer summary values. + A column that evaluates to an integer. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27741,7 +28518,8 @@ def tuple_union_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27794,7 +28572,8 @@ def tuple_union_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. lgNomEntries : :class:`~pyspark.sql.Column` or int, optional The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) mode : :class:`~pyspark.sql.Column` or str, optional @@ -27846,7 +28625,8 @@ def tuple_intersection_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27896,7 +28676,8 @@ def tuple_intersection_agg_integer( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations + The column containing binary TupleSketch representations. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional The summary mode: "sum" (default), "min", "max", or "alwaysone" @@ -27947,7 +28728,8 @@ def kll_sketch_agg_bigint( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing bigint values to aggregate + The column containing bigint values to aggregate. + A column that evaluates to an integral. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28025,7 +28807,8 @@ def kll_sketch_agg_double( Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing double values to aggregate + The column containing double values to aggregate. + A column that evaluates to a float or double. k : :class:`~pyspark.sql.Column` or int, optional The k parameter that controls size and accuracy (default 200, range 8-65535) @@ -28191,12 +28974,14 @@ def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28221,12 +29006,14 @@ def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28251,12 +29038,14 @@ def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- @@ -28281,12 +29070,14 @@ def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28314,12 +29105,14 @@ def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28347,12 +29140,14 @@ def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- @@ -28380,14 +29175,17 @@ def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL bigint sketch + The first KLL bigint sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL bigint sketch + The second KLL bigint sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28412,14 +29210,17 @@ def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Colum Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL float sketch + The first KLL float sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL float sketch + The second KLL float sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28444,14 +29245,17 @@ def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Colu Parameters ---------- left : :class:`~pyspark.sql.Column` or column name - The first KLL double sketch + The first KLL double sketch. + A column that evaluates to a binary. right : :class:`~pyspark.sql.Column` or column name - The second KLL double sketch + The second KLL double sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- @@ -28477,14 +29281,17 @@ def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a long. Examples -------- @@ -28513,14 +29320,17 @@ def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a float. Examples -------- @@ -28549,14 +29359,17 @@ def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0) + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The quantile value(s). + Returns a column that evaluates to a double. Examples -------- @@ -28585,14 +29398,17 @@ def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation + The KLL bigint sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a long or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28621,14 +29437,17 @@ def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation + The KLL float sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a float or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -28657,14 +29476,17 @@ def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") Parameters ---------- sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation + The KLL double sketch binary representation. + A column that evaluates to a binary. quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup + The quantile value(s) to lookup. + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double. Examples -------- @@ -29807,7 +30629,9 @@ def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- @@ -29842,7 +30666,9 @@ def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- @@ -29877,6 +30703,7 @@ def nullifzero(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric. Examples -------- @@ -30031,26 +30858,33 @@ def aes_encrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to encrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to encrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. iv : :class:`~pyspark.sql.Column` or column name, optional Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or "". 16-byte array for CBC mode. 12-byte array for GCM mode. + A column that evaluates to a binary. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains an encrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30170,23 +31004,29 @@ def aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value. + Returns a column that evaluates to a binary. See Also -------- @@ -30294,23 +31134,29 @@ def try_aes_decrypt( ---------- input : :class:`~pyspark.sql.Column` or column name The binary value to decrypt. + A column that evaluates to a binary. key : :class:`~pyspark.sql.Column` or column name The passphrase to use to decrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or column name, optional Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + A column that evaluates to a string. padding : :class:`~pyspark.sql.Column` or column name, optional Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + A column that evaluates to a string. aad : :class:`~pyspark.sql.Column` or column name, optional Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` A new column that contains a decrypted value or a NULL value. + Returns a column that evaluates to a binary. See Also -------- @@ -30423,6 +31269,7 @@ def sha(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a binary. See Also -------- @@ -30760,6 +31607,7 @@ def bitmap_bit_position(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30793,6 +31641,7 @@ def bitmap_bucket_number(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column. + A column that evaluates to a long. See Also -------- @@ -30827,6 +31676,7 @@ def bitmap_construct_agg(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name The input column will most likely be bitmap_bit_position(). + A column that evaluates to a long. See Also --------