diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index c8b269a63de5..0816cdbc663c 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -113,6 +113,8 @@ object functions { * * @group normal_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def lit(literal: Any): Column = { literal match { @@ -135,6 +137,8 @@ object functions { * * @group normal_funcs * @since 2.2.0 + * @return + * Returns a column of the same type as the input. */ def typedLit[T: TypeTag](literal: T): Column = { typedlit(literal) @@ -154,6 +158,8 @@ object functions { * * @group normal_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def typedlit[T: TypeTag](literal: T): Column = { literal match { @@ -246,6 +252,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column): Column = approx_count_distinct(e) @@ -253,6 +261,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) @@ -260,6 +270,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) @@ -267,6 +279,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String, rsd: Double): Column = { @@ -276,16 +290,24 @@ object functions { /** * Aggregate function: returns the approximate number of distinct items in a group. * + * @param e + * The column to count distinct values in. A column of any type. * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) /** * Aggregate function: returns the approximate number of distinct items in a group. * + * @param columnName + * The name of the column to count distinct values in. A column of any type. * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String): Column = approx_count_distinct( column(columnName)) @@ -298,6 +320,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column, rsd: Double): Column = { Column.fn("approx_count_distinct", e, lit(rsd)) @@ -311,6 +335,8 @@ object functions { * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String, rsd: Double): Column = { approx_count_distinct(Column(columnName), rsd) @@ -319,64 +345,88 @@ object functions { /** * Aggregate function: returns the average of the values in a group. * + * @param e + * The column to average. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(e: Column): Column = Column.fn("avg", e) /** * Aggregate function: returns the average of the values in a group. * + * @param columnName + * The name of the column to average. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(columnName: String): Column = avg(Column(columnName)) /** * Aggregate function: returns a list of objects with duplicates. * + * @param e + * The column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(e: Column): Column = Column.fn("collect_list", e) /** * Aggregate function: returns a list of objects with duplicates. * + * @param columnName + * The name of the column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(columnName: String): Column = collect_list(Column(columnName)) /** * Aggregate function: returns a set of objects with duplicate elements eliminated. * + * @param e + * The column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(e: Column): Column = Column.fn("collect_set", e) /** * Aggregate function: returns a set of objects with duplicate elements eliminated. * + * @param columnName + * The name of the column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(columnName: String): Column = collect_set(Column(columnName)) @@ -386,8 +436,21 @@ object functions { * sketch is a probabilistic data structure used for cardinality estimation using sub-linear * space. * + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @param seed + * The random seed. A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = Column.fn("count_min_sketch", e, eps, confidence, seed) @@ -398,8 +461,19 @@ object functions { * sketch is a probabilistic data structure used for cardinality estimation using sub-linear * space. * + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) @@ -407,16 +481,28 @@ object functions { /** * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * + * @param column1 + * The first column. A column that evaluates to a numeric. + * @param column2 + * The second column. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) /** * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * + * @param columnName1 + * The name of the first column. A column that evaluates to a numeric. + * @param columnName2 + * The name of the second column. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(columnName1: String, columnName2: String): Column = { corr(Column(columnName1), Column(columnName2)) @@ -425,8 +511,12 @@ object functions { /** * Aggregate function: returns the number of items in a group. * + * @param e + * The column to count. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(e: Column): Column = Column.fn("count", e) @@ -434,8 +524,12 @@ object functions { /** * Aggregate function: returns the number of items in a group. * + * @param columnName + * The name of the column to count. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(columnName: String): TypedColumn[Any, Long] = count(Column(columnName)).as(PrimitiveLongEncoder) @@ -445,8 +539,14 @@ object functions { * * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * + * @param expr + * The first column. A column of any type. + * @param exprs + * Additional columns. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) @@ -456,8 +556,14 @@ object functions { * * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * + * @param columnName + * first column to compute on. A column of any type. + * @param columnNames + * additional columns to compute on. Columns of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(columnName: String, columnNames: String*): Column = @@ -466,8 +572,14 @@ object functions { /** * Aggregate function: returns the number of distinct items in a group. * + * @param expr + * first column to compute on. A column of any type. + * @param exprs + * additional columns to compute on. Columns of any type. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def count_distinct(expr: Column, exprs: Column*): Column = @@ -476,8 +588,14 @@ object functions { /** * Aggregate function: returns the population covariance for two columns. * + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(column1: Column, column2: Column): Column = Column.fn("covar_pop", column1, column2) @@ -485,8 +603,14 @@ object functions { /** * Aggregate function: returns the population covariance for two columns. * + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(columnName1: String, columnName2: String): Column = { covar_pop(Column(columnName1), Column(columnName2)) @@ -495,8 +619,14 @@ object functions { /** * Aggregate function: returns the sample covariance for two columns. * + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(column1: Column, column2: Column): Column = Column.fn("covar_samp", column1, column2) @@ -504,8 +634,14 @@ object functions { /** * Aggregate function: returns the sample covariance for two columns. * + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(columnName1: String, columnName2: String): Column = { covar_samp(Column(columnName1), Column(columnName2)) @@ -517,12 +653,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column, ignoreNulls: Boolean): Column = Column.fn("first", false, e, lit(ignoreNulls)) @@ -533,12 +676,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String, ignoreNulls: Boolean): Column = { first(Column(columnName), ignoreNulls) @@ -550,12 +700,16 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column): Column = first(e, ignoreNulls = false) @@ -565,24 +719,32 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String): Column = first(Column(columnName)) /** * Aggregate function: returns the first value in a group. * + * @param e + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column): Column = Column.fn("first_value", e) @@ -592,12 +754,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column, ignoreNulls: Column): Column = Column.fn("first_value", e, ignoreNulls) @@ -606,8 +775,12 @@ object functions { * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or * not, returns 1 for aggregated or 0 for not aggregated in the result set. * + * @param e + * column to check if it is aggregated. A column of any type. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(e: Column): Column = Column.fn("grouping", e) @@ -615,8 +788,12 @@ object functions { * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or * not, returns 1 for aggregated or 0 for not aggregated in the result set. * + * @param columnName + * column to check if it is aggregated. A column of any type. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(columnName: String): Column = grouping(Column(columnName)) @@ -627,12 +804,16 @@ object functions { * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) * }}} * + * @param cols + * columns to check for. Columns of any type. * @note * The list of columns should match with grouping columns exactly, or empty (means all the * grouping columns). * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) @@ -644,11 +825,17 @@ object functions { * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) * }}} * + * @param colName + * the name of the first grouping column. A column of any type. + * @param colNames + * the names of the remaining grouping columns. Columns of any type. * @note * The list of columns should match with grouping columns exactly. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(colName: String, colNames: String*): Column = { @@ -659,8 +846,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Column): Column = Column.fn("hll_sketch_agg", e, lgConfigK) @@ -669,8 +864,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Int): Column = Column.fn("hll_sketch_agg", e, lit(lgConfigK)) @@ -679,8 +882,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { hll_sketch_agg(Column(columnName), lgConfigK) @@ -690,8 +901,13 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with default lgConfigK value. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column): Column = Column.fn("hll_sketch_agg", e) @@ -700,8 +916,13 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with default lgConfigK value. * + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String): Column = { hll_sketch_agg(Column(columnName)) @@ -713,8 +934,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = Column.fn("hll_union_agg", e, allowDifferentLgConfigK) @@ -725,8 +954,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) @@ -737,8 +974,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { hll_union_agg(Column(columnName), allowDifferentLgConfigK) @@ -749,8 +994,13 @@ object functions { * HllSketch, generated by merging previously created Datasketches HllSketch instances via a * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column): Column = Column.fn("hll_union_agg", e) @@ -760,8 +1010,13 @@ object functions { * HllSketch, generated by merging previously created Datasketches HllSketch instances via a * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String): Column = { hll_union_agg(Column(columnName)) @@ -770,16 +1025,24 @@ object functions { /** * Aggregate function: returns the kurtosis of the values in a group. * + * @param e + * the column to compute the kurtosis on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(e: Column): Column = Column.fn("kurtosis", e) /** * Aggregate function: returns the kurtosis of the values in a group. * + * @param columnName + * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) @@ -789,12 +1052,18 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column, ignoreNulls: Boolean): Column = Column.fn("last", false, e, lit(ignoreNulls)) @@ -805,12 +1074,18 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * the name of the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String, ignoreNulls: Boolean): Column = { last(Column(columnName), ignoreNulls) @@ -822,12 +1097,16 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the last value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column): Column = last(e, ignoreNulls = false) @@ -843,18 +1122,24 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) /** * Aggregate function: returns the last value in a group. * + * @param e + * column to fetch the last value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column): Column = Column.fn("last_value", e) @@ -864,12 +1149,18 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the last value for. A column of any type. + * @param ignoreNulls + * whether to skip null values. A column that evaluates to a boolean. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column, ignoreNulls: Column): Column = Column.fn("last_value", e, ignoreNulls) @@ -878,13 +1169,15 @@ object functions { * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * * @param hour - * the hour to represent, from 0 to 23 + * the hour to represent, from 0 to 23. A column that evaluates to an integer. * @param minute - * the minute to represent, from 0 to 59 + * the minute to represent, from 0 to 59. A column that evaluates to an integer. * @param second - * the second to represent, from 0 to 59.999999 + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a time. */ def make_time(hour: Column, minute: Column, second: Column): Column = { Column.fn("make_time", hour, minute, second) @@ -893,8 +1186,12 @@ object functions { /** * Aggregate function: returns the most frequent value in a group. * + * @param e + * target column to compute on. A column of any type. * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column): Column = Column.fn("mode", e) @@ -905,16 +1202,27 @@ object functions { * if deterministic is false or is not defined, or the lowest value is returned if deterministic * is true. * + * @param e + * target column to compute on. A column of any type. + * @param deterministic + * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a + * constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) /** * Aggregate function: returns the maximum value of the expression in a group. * + * @param e + * the target column on which the maximum value is computed. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(e: Column): Column = Column.fn("max", e) @@ -923,18 +1231,26 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(columnName: String): Column = max(Column(columnName)) /** * Aggregate function: returns the value associated with the maximum value of ord. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. * @note * The function is non-deterministic so the output order can be different for those associated * the same values of `e`. * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) @@ -944,6 +1260,12 @@ object functions { * The result array contains values in descending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. An integer. Must be a constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -953,6 +1275,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) @@ -962,6 +1286,12 @@ object functions { * The result array contains values in descending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. A column that evaluates to an integer. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -971,14 +1301,20 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) /** * Aggregate function: returns the average of the values in a group. Alias for avg. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(e: Column): Column = avg(e) @@ -987,42 +1323,63 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(columnName: String): Column = avg(columnName) /** * Aggregate function: returns the median of the values in a group. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a double. */ def median(e: Column): Column = Column.fn("median", e) /** * Aggregate function: returns the minimum value of the expression in a group. * + * @param e + * the target column on which the minimum value is computed. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(e: Column): Column = Column.fn("min", e) /** * Aggregate function: returns the minimum value of the column in a group. * + * @param columnName + * the name of the column on which the minimum value is computed. A column of an orderable + * type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(columnName: String): Column = min(Column(columnName)) /** * Aggregate function: returns the value associated with the minimum value of ord. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. * @note * The function is non-deterministic so the output order can be different for those associated * the same values of `e`. * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) @@ -1033,6 +1390,12 @@ object functions { * The result array contains values in ascending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. Must be a constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -1042,6 +1405,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) @@ -1052,6 +1417,13 @@ object functions { * The result array contains values in ascending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. A column that evaluates to an integral. Must be a + * constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -1061,6 +1433,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) @@ -1068,8 +1442,15 @@ object functions { * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given * percentage(s) with value range in [0.0, 1.0]. * + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) @@ -1077,8 +1458,18 @@ object functions { * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given * percentage(s) with value range in [0.0, 1.0]. * + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param frequency + * the positive frequency with which to weight each value. A column that evaluates to an + * integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column, frequency: Column): Column = Column.fn("percentile", e, percentage, frequency) @@ -1095,8 +1486,19 @@ object functions { * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the * relative error of the approximation. * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = Column.fn("percentile_approx", e, percentage, accuracy) @@ -1113,8 +1515,19 @@ object functions { * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the * relative error of the approximation. * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { Column.fn("approx_percentile", e, percentage, accuracy) @@ -1123,96 +1536,149 @@ object functions { /** * Aggregate function: returns the product of all numerical elements in a group. * + * @param e + * the column to compute the product on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a double. */ def product(e: Column): Column = Column.internalFn("product", e) /** * Aggregate function: returns the skewness of the values in a group. * + * @param e + * the column to compute the skewness on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(e: Column): Column = Column.fn("skewness", e) /** * Aggregate function: returns the skewness of the values in a group. * + * @param columnName + * the name of the column to compute the skewness on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(columnName: String): Column = skewness(Column(columnName)) /** * Aggregate function: alias for `stddev_samp`. * + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def std(e: Column): Column = Column.fn("std", e) /** * Aggregate function: alias for `stddev_samp`. * + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(e: Column): Column = Column.fn("stddev", e) /** * Aggregate function: alias for `stddev_samp`. * + * @param columnName + * the name of the column to compute the standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(columnName: String): Column = stddev(Column(columnName)) /** * Aggregate function: returns the sample standard deviation of the expression in a group. * + * @param e + * the column to compute the sample standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) /** * Aggregate function: returns the sample standard deviation of the expression in a group. * + * @param columnName + * Name of the column to compute the sample standard deviation on. A column that evaluates to + * a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) /** * Aggregate function: returns the population standard deviation of the expression in a group. * + * @param e + * The column to compute the population standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) /** * Aggregate function: returns the population standard deviation of the expression in a group. * + * @param columnName + * Name of the column to compute the population standard deviation on. A column that evaluates + * to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) /** * Aggregate function: returns the sum of all values in the expression. * + * @param e + * The column to sum. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(e: Column): Column = Column.fn("sum", e) /** * Aggregate function: returns the sum of all values in the given column. * + * @param columnName + * Name of the column to sum. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(columnName: String): Column = sum(Column(columnName)) @@ -1221,6 +1687,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(e: Column): Column = sum_distinct(e) @@ -1230,6 +1698,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) @@ -1237,8 +1707,12 @@ object functions { /** * Aggregate function: returns the sum of distinct values in the expression. * + * @param e + * The column to sum distinct values of. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) @@ -1247,8 +1721,13 @@ object functions { * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input * column via a Datasketches Intersection instance. * + * @param e + * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(e: Column): Column = Column.fn("theta_intersection_agg", e) @@ -1258,8 +1737,13 @@ object functions { * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input * volumn via a Datasketches Intersection instance. * + * @param columnName + * Name of the column of Datasketches ThetaSketch instances to intersect. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(columnName: String): Column = theta_intersection_agg(Column(columnName)) @@ -1269,8 +1753,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_sketch_agg", e, lgNomEntries) @@ -1280,8 +1772,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) @@ -1291,8 +1791,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = theta_sketch_agg(Column(columnName), lgNomEntries) @@ -1302,8 +1810,13 @@ object functions { * built with the values in the input column and configured with the default value of 12 for * `lgNomEntries`. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column): Column = Column.fn("theta_sketch_agg", e) @@ -1313,8 +1826,13 @@ object functions { * built with the values in the input column and configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String): Column = theta_sketch_agg(Column(columnName)) @@ -1325,8 +1843,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_union_agg", e, lgNomEntries) @@ -1337,8 +1863,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_union_agg", e, lit(lgNomEntries)) @@ -1349,8 +1883,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String, lgNomEntries: Int): Column = theta_union_agg(Column(columnName), lgNomEntries) @@ -1361,8 +1903,13 @@ object functions { * via a Datasketches Union instance. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column): Column = Column.fn("theta_union_agg", e) @@ -1373,8 +1920,13 @@ object functions { * via a Datasketches Union instance. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String): Column = theta_union_agg(Column(columnName)) @@ -1385,8 +1937,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_double", e, mode) @@ -1397,8 +1957,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_double", e, lit(mode)) @@ -1409,8 +1977,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String, mode: String): Column = tuple_intersection_agg_double(Column(columnName), mode) @@ -1421,8 +1997,13 @@ object functions { * in the input column via a Datasketches Intersection instance. It is configured with the * default mode of 'sum'. * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column): Column = Column.fn("tuple_intersection_agg_double", e) @@ -1433,8 +2014,13 @@ object functions { * in the input column via a Datasketches Intersection instance. It is configured with the * default mode of 'sum'. * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String): Column = tuple_intersection_agg_double(Column(columnName)) @@ -1446,8 +2032,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_integer", e, mode) @@ -1459,8 +2053,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_integer", e, lit(mode)) @@ -1472,8 +2074,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String, mode: String): Column = tuple_intersection_agg_integer(Column(columnName), mode) @@ -1484,8 +2094,13 @@ object functions { * instances in the input column via a Datasketches Intersection instance. It is configured with * the default mode of 'sum'. * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column): Column = Column.fn("tuple_intersection_agg_integer", e) @@ -1496,8 +2111,13 @@ object functions { * instances in the input column via a Datasketches Intersection instance. It is configured with * the default mode of 'sum'. * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String): Column = tuple_intersection_agg_integer(Column(columnName)) @@ -1508,8 +2128,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1524,8 +2158,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1540,8 +2188,20 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1555,8 +2215,19 @@ object functions { * with a double type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) @@ -1566,8 +2237,17 @@ object functions { * with a double type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1580,8 +2260,16 @@ object functions { * with a double type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_double", key, summary) @@ -1591,8 +2279,14 @@ object functions { * with a double type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) @@ -1603,8 +2297,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1619,8 +2327,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1635,8 +2357,20 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1650,8 +2384,19 @@ object functions { * with an integer type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) @@ -1661,8 +2406,17 @@ object functions { * with an integer type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1675,8 +2429,16 @@ object functions { * with an integer type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_integer", key, summary) @@ -1686,8 +2448,14 @@ object functions { * with an integer type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) @@ -1699,8 +2467,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * the column containing binary TupleSketch representations to union. A column that evaluates + * to a binary. + * @param lgNomEntries + * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column + * that evaluates to an integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) @@ -1712,8 +2491,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) @@ -1725,8 +2515,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_double(Column(columnName), lgNomEntries, mode) @@ -1737,8 +2538,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) @@ -1749,8 +2558,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_double(Column(columnName), lgNomEntries) @@ -1761,8 +2578,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column): Column = Column.fn("tuple_union_agg_double", e) @@ -1773,8 +2595,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String): Column = tuple_union_agg_double(Column(columnName)) @@ -1786,8 +2613,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) @@ -1799,8 +2637,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) @@ -1812,8 +2661,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) @@ -1824,8 +2684,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) @@ -1836,8 +2704,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries) @@ -1848,8 +2724,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column): Column = Column.fn("tuple_union_agg_integer", e) @@ -1860,8 +2741,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String): Column = tuple_union_agg_integer(Column(columnName)) @@ -1871,8 +2757,16 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_bigint", e, k) @@ -1882,8 +2776,16 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_bigint", e, lit(k)) @@ -1893,8 +2795,15 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String, k: Int): Column = kll_sketch_agg_bigint(Column(columnName), k) @@ -1903,8 +2812,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllLongsSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing bigint values to aggregate. A column that evaluates to an integral. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column): Column = Column.fn("kll_sketch_agg_bigint", e) @@ -1913,8 +2826,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllLongsSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String): Column = kll_sketch_agg_bigint(Column(columnName)) @@ -1924,8 +2841,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing float values to aggregate. A column that evaluates to a float. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_float", e, k) @@ -1935,8 +2859,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_float", e, lit(k)) @@ -1946,8 +2877,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String, k: Int): Column = kll_sketch_agg_float(Column(columnName), k) @@ -1956,8 +2894,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllFloatsSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column): Column = Column.fn("kll_sketch_agg_float", e) @@ -1966,8 +2908,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllFloatsSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String): Column = kll_sketch_agg_float(Column(columnName)) @@ -1977,8 +2923,16 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing double values to aggregate. A column that evaluates to a float or + * double. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_double", e, k) @@ -1988,8 +2942,15 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_double", e, lit(k)) @@ -1999,8 +2960,15 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String, k: Int): Column = kll_sketch_agg_double(Column(columnName), k) @@ -2009,8 +2977,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllDoublesSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column): Column = Column.fn("kll_sketch_agg_double", e) @@ -2019,8 +2991,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllDoublesSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String): Column = kll_sketch_agg_double(Column(columnName)) @@ -2031,8 +3007,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_merge_agg_bigint", e, k) @@ -2043,8 +3027,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_merge_agg_bigint", e, lit(k)) @@ -2055,8 +3047,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String, k: Int): Column = kll_merge_agg_bigint(Column(columnName), k) @@ -2066,8 +3066,13 @@ object functions { * sketch. If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column): Column = Column.fn("kll_merge_agg_bigint", e) @@ -2077,8 +3082,13 @@ object functions { * sketch. If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String): Column = kll_merge_agg_bigint(Column(columnName)) @@ -2088,8 +3098,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Column): Column = Column.fn("kll_merge_agg_float", e, k) @@ -2099,8 +3117,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Int): Column = Column.fn("kll_merge_agg_float", e, lit(k)) @@ -2110,8 +3136,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String, k: Int): Column = kll_merge_agg_float(Column(columnName), k) @@ -2120,8 +3154,13 @@ object functions { * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column): Column = Column.fn("kll_merge_agg_float", e) @@ -2130,8 +3169,13 @@ object functions { * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String): Column = kll_merge_agg_float(Column(columnName)) @@ -2141,8 +3185,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Column): Column = Column.fn("kll_merge_agg_double", e, k) @@ -2152,8 +3204,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Int): Column = Column.fn("kll_merge_agg_double", e, lit(k)) @@ -2163,8 +3223,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String, k: Int): Column = kll_merge_agg_double(Column(columnName), k) @@ -2173,8 +3241,13 @@ object functions { * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column): Column = Column.fn("kll_merge_agg_double", e) @@ -2183,8 +3256,13 @@ object functions { * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String): Column = kll_merge_agg_double(Column(columnName)) @@ -2192,8 +3270,12 @@ object functions { /** * Aggregate function: returns the concatenation of non-null input values. * + * @param e + * The target column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column): Column = Column.fn("listagg", e) @@ -2201,16 +3283,27 @@ object functions { * Aggregate function: returns the concatenation of non-null input values, separated by the * delimiter. * + * @param e + * The target column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * The delimiter used to separate the values. A column that evaluates to a string or binary. + * Must be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) /** * Aggregate function: returns the concatenation of distinct non-null input values. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) @@ -2218,8 +3311,15 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values, separated by * the delimiter. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column, delimiter: Column): Column = Column.fn("listagg", isDistinct = true, e, delimiter) @@ -2227,8 +3327,12 @@ object functions { /** * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column): Column = Column.fn("string_agg", e) @@ -2236,8 +3340,15 @@ object functions { * Aggregate function: returns the concatenation of non-null input values, separated by the * delimiter. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) @@ -2245,8 +3356,12 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values. Alias for * `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) @@ -2254,8 +3369,15 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values, separated by * the delimiter. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column, delimiter: Column): Column = Column.fn("string_agg", isDistinct = true, e, delimiter) @@ -2263,8 +3385,12 @@ object functions { /** * Aggregate function: alias for `var_samp`. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(e: Column): Column = Column.fn("variance", e) @@ -2273,14 +3399,20 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(columnName: String): Column = variance(Column(columnName)) /** * Aggregate function: returns the unbiased variance of the values in a group. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(e: Column): Column = Column.fn("var_samp", e) @@ -2289,14 +3421,20 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(columnName: String): Column = var_samp(Column(columnName)) /** * Aggregate function: returns the population variance of the values in a group. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(e: Column): Column = Column.fn("var_pop", e) @@ -2305,6 +3443,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(columnName: String): Column = var_pop(Column(columnName)) @@ -2312,8 +3452,14 @@ object functions { * Aggregate function: returns the average of the independent variable for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) @@ -2321,8 +3467,14 @@ object functions { * Aggregate function: returns the average of the independent variable for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) @@ -2330,8 +3482,14 @@ object functions { * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the * dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) @@ -2340,8 +3498,14 @@ object functions { * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent * variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) @@ -2349,8 +3513,14 @@ object functions { * Aggregate function: returns the coefficient of determination for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) @@ -2358,8 +3528,14 @@ object functions { * Aggregate function: returns the slope of the linear regression line for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) @@ -2367,8 +3543,14 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) @@ -2376,8 +3558,14 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) @@ -2385,16 +3573,26 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) /** * Aggregate function: returns some value of `e` for a group of rows. * + * @param e + * The column to return some value from. A column of any type. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column): Column = Column.fn("any_value", e) @@ -2402,8 +3600,15 @@ object functions { * Aggregate function: returns some value of `e` for a group of rows. If `isIgnoreNull` is true, * returns only non-null values. * + * @param e + * The column to return some value from. A column of any type. + * @param ignoreNulls + * If true, returns only non-null values. A column that evaluates to a boolean. Must be a + * constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column, ignoreNulls: Column): Column = Column.fn("any_value", e, ignoreNulls) @@ -2411,8 +3616,12 @@ object functions { /** * Aggregate function: returns the number of `TRUE` values for the expression. * + * @param e + * The expression to count TRUE values of. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def count_if(e: Column): Column = Column.fn("count_if", e) @@ -2421,8 +3630,7 @@ object functions { * 6 fractional digits of seconds. * * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2437,8 +3645,7 @@ object functions { * An integer literal in the range [0..6], indicating how many fractional digits of seconds to * include in the result. * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2457,8 +3664,14 @@ object functions { * statistical computing packages. Note: the output type of the 'x' field in the return value is * propagated from the input value consumed in the aggregate function. * + * @param e + * The column to compute the histogram on. A column that evaluates to a numeric. + * @param nBins + * The number of histogram bins. A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def histogram_numeric(e: Column, nBins: Column): Column = Column.fn("histogram_numeric", e, nBins) @@ -2466,64 +3679,96 @@ object functions { /** * Aggregate function: returns true if all values of `e` are true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def every(e: Column): Column = Column.fn("every", e) /** * Aggregate function: returns true if all values of `e` are true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_and(e: Column): Column = Column.fn("bool_and", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def some(e: Column): Column = Column.fn("some", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def any(e: Column): Column = Column.fn("any", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * column to check if at least one value is true. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_or(e: Column): Column = Column.fn("bool_or", e) /** * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_and(e: Column): Column = Column.fn("bit_and", e) /** * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_or(e: Column): Column = Column.fn("bit_or", e) /** * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_xor(e: Column): Column = Column.fn("bit_xor", e) @@ -2550,8 +3795,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2580,8 +3825,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2599,6 +3844,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def cume_dist(): Column = Column.fn("cume_dist") @@ -2615,6 +3862,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def dense_rank(): Column = Column.fn("dense_rank") @@ -2625,8 +3874,14 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int): Column = lag(e, offset, null) @@ -2637,8 +3892,14 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) @@ -2649,8 +3910,16 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int, defaultValue: Any): Column = { lag(Column(columnName), offset, defaultValue) @@ -2663,8 +3932,16 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any): Column = { lag(e, offset, defaultValue, false) @@ -2679,8 +3956,18 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @param ignoreNulls + * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2692,8 +3979,14 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } @@ -2704,8 +3997,14 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } @@ -2716,8 +4015,16 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int, defaultValue: Any): Column = { lead(Column(columnName), offset, defaultValue) @@ -2730,8 +4037,16 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any): Column = { lead(e, offset, defaultValue, false) @@ -2746,8 +4061,21 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * The column to compute the lead value for. A column of any type. + * @param offset + * Number of rows after the current row to look ahead. A column that evaluates to an integral. + * Must be a constant. + * @param defaultValue + * Value to return when there are fewer than `offset` rows after the current row. Must be a + * constant. + * @param ignoreNulls + * Whether to skip null values when computing the result. A column that evaluates to a + * boolean. Must be a constant. * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2761,8 +4089,18 @@ object functions { * * This is equivalent to the nth_value function in SQL. * + * @param e + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @param ignoreNulls + * Whether the nth value should skip nulls when determining which row to use. A column that + * evaluates to a boolean. Must be a constant. * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) @@ -2773,8 +4111,15 @@ object functions { * * This is equivalent to the nth_value function in SQL. * + * @param e + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) @@ -2785,8 +4130,13 @@ object functions { * * This is equivalent to the NTILE function in SQL. * + * @param n + * The number of groups to divide the window partition into. A column that evaluates to an + * integral. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def ntile(n: Int): Column = Column.fn("ntile", lit(n)) @@ -2803,6 +4153,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def percent_rank(): Column = Column.fn("percent_rank") @@ -2819,6 +4171,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def rank(): Column = Column.fn("rank") @@ -2827,6 +4181,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def row_number(): Column = Column.fn("row_number") @@ -2837,8 +4193,13 @@ object functions { /** * Creates a new array column. The input columns must all have the same data type. * + * @param cols + * The columns to combine into an array. Each is a column of any type, and all must share the + * same data type. * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(cols: Column*): Column = Column.fn("array", cols: _*) @@ -2848,6 +4209,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(colName: String, colNames: String*): Column = { @@ -2859,8 +4222,13 @@ object functions { * value1, key2, value2, ...). The key columns must all have the same data type, and can't be * null. The value columns must all have the same data type. * + * @param cols + * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column + * of any type; key columns must share a type and value columns must share a type. * @group map_funcs * @since 2.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map(cols: Column*): Column = Column.fn("map", cols: _*) @@ -2868,8 +4236,13 @@ object functions { /** * Creates a struct with the given field names and values. * + * @param cols + * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are + * columns that evaluate to a string; values are columns of any type. * @group struct_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) @@ -2878,8 +4251,15 @@ object functions { * Creates a new map column. The array in the first column is used for keys. The array in the * second column is used for values. All elements in the array for key should not be null. * + * @param keys + * The array of keys for the map; elements must not be null. A column that evaluates to an + * array. + * @param values + * The array of values for the map. A column that evaluates to an array. * @group map_funcs * @since 2.4 + * @return + * Returns a column that evaluates to a map. */ def map_from_arrays(keys: Column, values: Column): Column = Column.fn("map_from_arrays", keys, values) @@ -2888,8 +4268,18 @@ object functions { * Creates a map after splitting the text into key/value pairs using delimiters. Both * `pairDelim` and `keyValueDelim` are treated as regular expressions. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @param keyValueDelim + * Delimiter used to split key and value, treated as a regular expression. A column that + * evaluates to a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = Column.fn("str_to_map", text, pairDelim, keyValueDelim) @@ -2898,8 +4288,15 @@ object functions { * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` * is treated as regular expressions. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column): Column = Column.fn("str_to_map", text, pairDelim) @@ -2907,8 +4304,12 @@ object functions { /** * Creates a map after splitting the text into key/value pairs using delimiters. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column): Column = Column.fn("str_to_map", text) @@ -2934,8 +4335,12 @@ object functions { * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is * not null, or c if both a and b are null but c is not null. * + * @param e + * the columns to work on. A column that evaluates to any type. * @group conditional_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) @@ -2945,22 +4350,32 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def input_file_name(): Column = Column.fn("input_file_name") /** * Return true iff the column is NaN. * + * @param e + * the column to check. A column that evaluates to a numeric. * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnan(e: Column): Column = e.isNaN /** * Return true iff the column is null. * + * @param e + * the column to check. A column that evaluates to any type. * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnull(e: Column): Column = e.isNull @@ -2981,6 +4396,8 @@ object functions { * * @group misc_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use monotonically_increasing_id()", "2.0.0") def monotonicallyIncreasingId(): Column = monotonically_increasing_id() @@ -3002,6 +4419,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a long. */ def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") @@ -3010,8 +4429,14 @@ object functions { * * Both inputs should be floating point columns (DoubleType or FloatType). * + * @param col1 + * the first column to check. A column that evaluates to a numeric. + * @param col2 + * the column to return if the first is NaN. A column that evaluates to a numeric. * @group conditional_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) @@ -3026,8 +4451,12 @@ object functions { * df.select( negate(df.col("amount")) ); * }}} * + * @param e + * the column to negate. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def negate(e: Column): Column = -e @@ -3041,8 +4470,12 @@ object functions { * df.filter( not(df.col("isActive")) ); * }}} * + * @param e + * the column to invert. A column that evaluates to a boolean. * @group predicate_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def not(e: Column): Column = !e @@ -3050,11 +4483,15 @@ object functions { * Generate a random column with independent and identically distributed (i.i.d.) samples * uniformly distributed in [0.0, 1.0). * + * @param seed + * the seed for the random generator. * @note * The function is non-deterministic in general case. * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(seed: Long): Column = Column.fn("rand", lit(seed)) @@ -3067,6 +4504,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(): Column = rand(SparkClassUtils.random.nextLong) @@ -3074,11 +4513,15 @@ object functions { * Generate a column with independent and identically distributed (i.i.d.) samples from the * standard normal distribution. * + * @param seed + * the seed for the random generator. * @note * The function is non-deterministic in general case. * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(seed: Long): Column = Column.fn("randn", lit(seed)) @@ -3091,6 +4534,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(): Column = randn(SparkClassUtils.random.nextLong) @@ -3099,8 +4544,13 @@ object functions { * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant * two-byte or four-byte integer (SMALLINT or INT, respectively). * + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column): Column = randstr(length, lit(SparkClassUtils.random.nextLong)) @@ -3110,8 +4560,15 @@ object functions { * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). * + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. + * @param seed + * the random seed to use. A column that evaluates to an integral. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) @@ -3123,22 +4580,32 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def spark_partition_id(): Column = Column.fn("spark_partition_id") /** * Computes the square root of the specified float value. * + * @param e + * the value to compute the square root of. A column that evaluates to a numeric. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(e: Column): Column = Column.fn("sqrt", e) /** * Computes the square root of the specified float value. * + * @param colName + * the name of a numeric column to compute the square root of. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(colName: String): Column = sqrt(Column(colName)) @@ -3146,16 +4613,26 @@ object functions { * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable * input types are the same with the `+` operator. * + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) /** * Returns the mean calculated from values of a group and the result is null on overflow. * + * @param e + * the value to compute the mean of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def try_avg(e: Column): Column = Column.fn("try_avg", e) @@ -3163,16 +4640,28 @@ object functions { * Returns `dividend``/``divisor`. It always performs floating point division. Its result is * always null if `divisor` is 0. * + * @param left + * the dividend. A column that evaluates to a numeric or interval. + * @param right + * the divisor. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) /** * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. * + * @param left + * the dividend. A column that evaluates to a numeric. + * @param right + * the divisor. A column that evaluates to a numeric. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) @@ -3180,8 +4669,14 @@ object functions { * Returns `left``*``right` and the result is null on overflow. The acceptable input types are * the same with the `*` operator. * + * @param left + * the multiplicand. A column that evaluates to a numeric or interval. + * @param right + * the multiplier. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) @@ -3189,16 +4684,26 @@ object functions { * Returns `left``-``right` and the result is null on overflow. The acceptable input types are * the same with the `-` operator. * + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) /** * Returns the sum calculated from values of a group and the result is null on overflow. * + * @param e + * the value to compute the sum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a numeric. */ def try_sum(e: Column): Column = Column.fn("try_sum", e) @@ -3208,8 +4713,12 @@ object functions { * StructField's name, otherwise, the newly generated StructField's name would be auto generated * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... * + * @param cols + * the columns to contain in the output struct. A column of any type. * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(cols: Column*): Column = Column.fn("struct", cols: _*) @@ -3217,8 +4726,14 @@ object functions { /** * Creates a new struct column that composes multiple input columns. * + * @param colName + * the name of the first column to contain in the output struct. + * @param colNames + * the names of the remaining columns to contain in the output struct. * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(colName: String, colNames: String*): Column = { @@ -3243,8 +4758,14 @@ object functions { * .otherwise(2)) * }}} * + * @param condition + * the condition to evaluate. A column that evaluates to a boolean. + * @param value + * the value to return when the condition is true. A literal value, or a column expression. * @group conditional_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def when(condition: Column, value: Any): Column = Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) @@ -3254,6 +4775,8 @@ object functions { * * @group bitwise_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use bitwise_not", "3.2.0") def bitwiseNOT(e: Column): Column = bitwise_not(e) @@ -3261,8 +4784,12 @@ object functions { /** * Computes bitwise NOT (~) of a number. * + * @param e + * the target column to compute on. A column that evaluates to an integral. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def bitwise_not(e: Column): Column = Column.fn("~", e) @@ -3270,8 +4797,12 @@ object functions { * Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, * or NULL if the argument is NULL. * + * @param e + * the target column to compute on. A column that evaluates to an integral or boolean. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_count(e: Column): Column = Column.fn("bit_count", e) @@ -3279,8 +4810,15 @@ object functions { * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered * from right to left, starting at zero. The position argument cannot be negative. * + * @param e + * the target column to compute on. A column that evaluates to an integral. + * @param pos + * the bit position, numbered from right to left starting at zero. A column that evaluates to + * an integer. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos) @@ -3288,8 +4826,15 @@ object functions { * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered * from right to left, starting at zero. The position argument cannot be negative. * + * @param e + * the target column to compute on. A column that evaluates to an integral. + * @param pos + * the bit position, numbered from right to left starting at zero. A column that evaluates to + * an integer. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) @@ -3312,14 +4857,22 @@ object functions { /** * Computes the absolute value of a numeric value. * + * @param e + * the value to compute the absolute value of. A column that evaluates to a numeric or + * interval. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def abs(e: Column): Column = Column.fn("abs", e) /** + * @param e + * the value to compute the inverse cosine of. A column that evaluates to a double. * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3327,17 +4880,21 @@ object functions { def acos(e: Column): Column = Column.fn("acos", e) /** + * @param columnName + * the value to compute the inverse cosine of. * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos` - * + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def acos(columnName: String): Column = acos(Column(columnName)) /** + * @param e + * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. * @return - * inverse hyperbolic cosine of `e` + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3345,17 +4902,21 @@ object functions { def acosh(e: Column): Column = Column.fn("acosh", e) /** + * @param columnName + * the value to compute the inverse hyperbolic cosine of. * @return - * inverse hyperbolic cosine of `columnName` - * + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ def acosh(columnName: String): Column = acosh(Column(columnName)) /** + * @param e + * the value to compute the inverse sine of. A column that evaluates to a double. * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3363,17 +4924,21 @@ object functions { def asin(e: Column): Column = Column.fn("asin", e) /** + * @param columnName + * the value to compute the inverse sine of. * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin` - * + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def asin(columnName: String): Column = asin(Column(columnName)) /** + * @param e + * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. * @return - * inverse hyperbolic sine of `e` + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3381,17 +4946,21 @@ object functions { def asinh(e: Column): Column = Column.fn("asinh", e) /** + * @param columnName + * the value to compute the inverse hyperbolic sine of. * @return - * inverse hyperbolic sine of `columnName` - * + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ def asinh(columnName: String): Column = asinh(Column(columnName)) /** + * @param e + * the value to compute the inverse tangent of. A column that evaluates to a double. * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan` + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3399,9 +4968,11 @@ object functions { def atan(e: Column): Column = Column.fn("atan", e) /** + * @param columnName + * the value to compute the inverse tangent of. * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan` - * + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3409,13 +4980,13 @@ object functions { /** * @param y - * coordinate on y-axis + * coordinate on y-axis. A column that evaluates to a double. * @param x - * coordinate on x-axis + * coordinate on x-axis. A column that evaluates to a double. * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3430,8 +5001,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3445,8 +5015,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3460,8 +5029,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3476,8 +5044,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3491,8 +5058,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3506,8 +5072,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3521,16 +5086,17 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) /** + * @param e + * target column to compute on. A column that evaluates to a numeric. * @return - * inverse hyperbolic tangent of `e` + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3538,9 +5104,10 @@ object functions { def atanh(e: Column): Column = Column.fn("atanh", e) /** + * @param columnName + * target column to compute on. * @return - * inverse hyperbolic tangent of `columnName` - * + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3550,8 +5117,12 @@ object functions { * An expression that returns the string representation of the binary value of the given long * column. For example, bin("12") returns "1100". * + * @param e + * target column to work on. A column that evaluates to an integral. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(e: Column): Column = Column.fn("bin", e) @@ -3559,81 +5130,128 @@ object functions { * An expression that returns the string representation of the binary value of the given long * column. For example, bin("12") returns "1100". * + * @param columnName + * target column to work on. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(columnName: String): Column = bin(Column(columnName)) /** * Computes the cube-root of the given value. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(e: Column): Column = Column.fn("cbrt", e) /** * Computes the cube-root of the given column. * + * @param columnName + * target column to compute on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(columnName: String): Column = cbrt(Column(columnName)) /** * Computes the ceiling of the given value of `e` to `scale` decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integral. Must be + * a constant. * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) /** * Computes the ceiling of the given value of `e` to 0 decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column): Column = Column.fn("ceil", e) /** * Computes the ceiling of the given value of `e` to 0 decimal places. * + * @param columnName + * the value to compute the ceiling on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(columnName: String): Column = ceil(Column(columnName)) /** * Computes the ceiling of the given value of `e` to `scale` decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integer. Must be + * a constant. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) /** * Computes the ceiling of the given value of `e` to 0 decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column): Column = Column.fn("ceiling", e) /** * Convert a number in a string column from one base to another. * + * @param num + * a column to convert base for. A column that evaluates to a string. + * @param fromBase + * from base number. A column that evaluates to an integer. + * @param toBase + * to base number. A column that evaluates to an integer. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def conv(num: Column, fromBase: Int, toBase: Int): Column = Column.fn("conv", num, lit(fromBase), lit(toBase)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3644,8 +5262,8 @@ object functions { * @param columnName * angle in radians * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` - * + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3653,9 +5271,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3666,8 +5285,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` - * + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3675,9 +5294,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cotangent of the angle + * cotangent of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3686,9 +5305,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosecant of the angle + * cosecant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3700,70 +5319,107 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def e(): Column = Column.fn("e") /** * Computes the exponential of the given value. * + * @param e + * target column to compute on. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(e: Column): Column = Column.fn("exp", e) /** * Computes the exponential of the given column. * + * @param columnName + * target column to compute on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(columnName: String): Column = exp(Column(columnName)) /** * Computes the exponential of the given value minus one. * + * @param e + * column to calculate exponential for. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(e: Column): Column = Column.fn("expm1", e) /** * Computes the exponential of the given column minus one. * + * @param columnName + * column name to calculate exponential for. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(columnName: String): Column = expm1(Column(columnName)) /** * Computes the factorial of the given value. * + * @param e + * a column to calculate factorial for. A column that evaluates to an integral. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def factorial(e: Column): Column = Column.fn("factorial", e) /** * Computes the floor of the given value of `e` to `scale` decimal places. * + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to control the rounding behavior. A column that evaluates to + * an integral. * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) /** * Computes the floor of the given value of `e` to 0 decimal places. * + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column): Column = Column.fn("floor", e) /** * Computes the floor of the given column value to 0 decimal places. * + * @param columnName + * the target column name to compute the floor on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(columnName: String): Column = floor(Column(columnName)) @@ -3771,8 +5427,12 @@ object functions { * Returns the greatest value of the list of values, skipping null values. This function takes * at least 2 parameters. It will return null iff all parameters are null. * + * @param exprs + * columns to check for greatest value. A column that evaluates to any type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) @@ -3781,8 +5441,14 @@ object functions { * Returns the greatest value of the list of column names, skipping null values. This function * takes at least 2 parameters. It will return null iff all parameters are null. * + * @param columnName + * the first column name to check for greatest value. A column of a comparable type. + * @param columnNames + * the remaining column names to check for greatest value. Columns of a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(columnName: String, columnNames: String*): Column = { @@ -3792,8 +5458,12 @@ object functions { /** * Computes hex value of the given column. * + * @param column + * target column to work on. A column that evaluates to an integral, string or binary. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def hex(column: Column): Column = Column.fn("hex", column) @@ -3801,40 +5471,68 @@ object functions { * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to * the byte representation of number. * + * @param column + * target column to work on. A column that evaluates to a string. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unhex(column: Column): Column = Column.fn("unhex", column) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, rightName: String): Column = hypot(Column(leftName), Column(rightName)) @@ -3842,32 +5540,56 @@ object functions { /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * The a leg of the triangle. A column that evaluates to a numeric. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, r: Column): Column = hypot(lit(l), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param rightName + * The b leg of the triangle. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) @@ -3875,8 +5597,12 @@ object functions { * Returns the least value of the list of values, skipping null values. This function takes at * least 2 parameters. It will return null iff all parameters are null. * + * @param exprs + * The values to be compared. Columns that evaluate to a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(exprs: Column*): Column = Column.fn("least", exprs: _*) @@ -3885,8 +5611,14 @@ object functions { * Returns the least value of the list of column names, skipping null values. This function * takes at least 2 parameters. It will return null iff all parameters are null. * + * @param columnName + * The name of the first column to be compared. A column of a comparable type. + * @param columnNames + * The names of the remaining columns to be compared. Columns of a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(columnName: String, columnNames: String*): Column = { @@ -3896,96 +5628,152 @@ object functions { /** * Computes the natural logarithm of the given value. * + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def ln(e: Column): Column = Column.fn("ln", e) /** * Computes the natural logarithm of the given value. * + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(e: Column): Column = ln(e) /** * Computes the natural logarithm of the given column. * + * @param columnName + * The name of the column to compute the natural logarithm of. A column that evaluates to a + * numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(columnName: String): Column = log(Column(columnName)) /** * Returns the first argument-base logarithm of the second argument. * + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param a + * The value to compute the logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) /** * Returns the first argument-base logarithm of the second argument. * + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param columnName + * The name of the column to compute the logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, columnName: String): Column = log(base, Column(columnName)) /** * Computes the logarithm of the given value in base 10. * + * @param e + * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(e: Column): Column = Column.fn("log10", e) /** * Computes the logarithm of the given value in base 10. * + * @param columnName + * The name of the column to compute the base-10 logarithm of. A column that evaluates to a + * numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(columnName: String): Column = log10(Column(columnName)) /** * Computes the natural logarithm of the given value plus one. * + * @param e + * The value to compute the natural logarithm of the value plus one. A column that evaluates + * to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(e: Column): Column = Column.fn("log1p", e) /** * Computes the natural logarithm of the given column plus one. * + * @param columnName + * The name of the column to compute the natural logarithm of the value plus one. A column + * that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(columnName: String): Column = log1p(Column(columnName)) /** * Computes the logarithm of the given column in base 2. * + * @param expr + * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(expr: Column): Column = Column.fn("log2", expr) /** * Computes the logarithm of the given value in base 2. * + * @param columnName + * a column to calculate logarithm for. A column that evaluates to a double. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(columnName: String): Column = log2(Column(columnName)) /** * Returns the negated value. * + * @param e + * column to calculate negative value for. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def negative(e: Column): Column = Column.fn("negative", e) @@ -3994,94 +5782,162 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def pi(): Column = Column.fn("pi") /** * Returns the value. * + * @param e + * input value column. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def positive(e: Column): Column = Column.fn("positive", e) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Column): Column = Column.fn("power", l, r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param rightName + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. + * @param rightName + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Double): Column = pow(l, lit(r)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. + * @param r + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, r: Column): Column = pow(lit(l), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. + * @param rightName + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def power(l: Column, r: Column): Column = Column.fn("power", l, r) /** * Returns the positive value of dividend mod divisor. * + * @param dividend + * the column that contains dividend, or the specified dividend value. A column that evaluates + * to a numeric. + * @param divisor + * the column that contains divisor, or the specified divisor value. A column that evaluates + * to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) @@ -4089,8 +5945,12 @@ object functions { * Returns the double value that is closest in value to the argument and is equal to a * mathematical integer. * + * @param e + * target column to compute on. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(e: Column): Column = Column.fn("rint", e) @@ -4098,16 +5958,24 @@ object functions { * Returns the double value that is closest in value to the argument and is equal to a * mathematical integer. * + * @param columnName + * the numeric column name to round to the closest integer. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(columnName: String): Column = rint(Column(columnName)) /** * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. * + * @param e + * the value to round. A column that evaluates to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column): Column = round(e, 0) @@ -4115,8 +5983,14 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) @@ -4124,16 +5998,27 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) /** * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. * + * @param e + * the value to round. A column that evaluates to a numeric. * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column): Column = bround(e, 0) @@ -4141,8 +6026,15 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) @@ -4150,16 +6042,23 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * secant of the angle + * secant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4172,6 +6071,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftleft", "3.2.0") def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) @@ -4180,8 +6081,15 @@ object functions { * Shift the given value numBits left. If the given value is a long value, this function will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift left. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) @@ -4191,6 +6099,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftright", "3.2.0") def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) @@ -4199,8 +6109,15 @@ object functions { * (Signed) shift the given value numBits right. If the given value is a long value, it will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) @@ -4210,6 +6127,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftrightunsigned", "3.2.0") def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) @@ -4218,8 +6137,15 @@ object functions { * Unsigned shift the given value numBits right. If the given value is a long value, it will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftrightunsigned(e: Column, numBits: Int): Column = Column.fn("shiftrightunsigned", e, lit(numBits)) @@ -4227,32 +6153,45 @@ object functions { /** * Computes the signum of the given value. * + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def sign(e: Column): Column = Column.fn("sign", e) /** * Computes the signum of the given value. * + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(e: Column): Column = Column.fn("signum", e) /** * Computes the signum of the given column. * + * @param columnName + * column to compute the signum on. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(columnName: String): Column = signum(Column(columnName)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -4263,8 +6202,8 @@ object functions { * @param columnName * angle in radians * @return - * sine of the angle, as if computed by `java.lang.Math.sin` - * + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * @group math_funcs * @since 1.4.0 */ @@ -4272,10 +6211,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4285,8 +6224,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4294,9 +6233,10 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4307,8 +6247,8 @@ object functions { * @param columnName * angle in radians * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` - * + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4316,10 +6256,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4329,8 +6269,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4339,6 +6279,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(e: Column): Column = degrees(e) @@ -4346,6 +6288,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(columnName: String): Column = degrees(Column(columnName)) @@ -4355,9 +6299,10 @@ object functions { * degrees. * * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4371,8 +6316,8 @@ object functions { * @param columnName * angle in radians * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` - * + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4381,6 +6326,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(e: Column): Column = radians(e) @@ -4388,6 +6335,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(columnName: String): Column = radians(Column(columnName)) @@ -4397,9 +6346,10 @@ object functions { * radians. * * @param e - * angle in degrees + * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4413,8 +6363,8 @@ object functions { * @param columnName * angle in degrees * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` - * + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4426,15 +6376,17 @@ object functions { * method will return null. * * @param v - * value to compute a bucket number in the histogram + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. * @param min - * minimum value of the histogram + * minimum value of the histogram. A column that evaluates to a double or interval. * @param max - * maximum value of the histogram + * maximum value of the histogram. A column that evaluates to a double or interval. * @param numBucket - * the number of buckets + * the number of buckets. A column that evaluates to a long. * @return - * the bucket number into which the value would fall after being evaluated + * the bucket number into which the value would fall after being evaluated. Returns a column + * that evaluates to a long. * @group math_funcs * @since 3.5.0 */ @@ -4450,6 +6402,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_catalog(): Column = Column.fn("current_catalog") @@ -4458,6 +6412,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_database(): Column = Column.fn("current_database") @@ -4466,6 +6422,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_schema(): Column = Column.fn("current_schema") @@ -4474,6 +6432,8 @@ object functions { * * @group misc_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a string. */ def current_path(): Column = Column.fn("current_path") @@ -4482,6 +6442,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_user(): Column = Column.fn("current_user") @@ -4489,8 +6451,12 @@ object functions { * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex * string. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def md5(e: Column): Column = Column.fn("md5", e) @@ -4498,8 +6464,12 @@ object functions { * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex * string. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha1(e: Column): Column = Column.fn("sha1", e) @@ -4508,12 +6478,14 @@ object functions { * hex string. * * @param e - * column to compute SHA-2 on. + * column to compute SHA-2 on. A column that evaluates to a binary. * @param numBits - * one of 224, 256, 384, or 512. + * one of 224, 256, 384, or 512. A column that evaluates to an integer. * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha2(e: Column, numBits: Int): Column = { require( @@ -4526,16 +6498,24 @@ object functions { * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value * as a bigint. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def crc32(e: Column): Column = Column.fn("crc32", e) /** * Calculates the hash code of given columns, and returns the result as an int column. * + * @param cols + * one or more columns to compute on. A column of any type. * @group hash_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to an integer. */ @scala.annotation.varargs def hash(cols: Column*): Column = Column.fn("hash", cols: _*) @@ -4544,8 +6524,12 @@ object functions { * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, * and returns the result as a long column. The hash computation uses an initial seed of 42. * + * @param cols + * one or more columns to compute on. A column of any type. * @group hash_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) @@ -4553,24 +6537,38 @@ object functions { /** * Returns null if the condition is true, and throws an exception otherwise. * + * @param c + * The condition to check. A column that evaluates to a boolean. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column): Column = Column.fn("assert_true", c) /** * Returns null if the condition is true; throws an exception with the error message otherwise. * + * @param c + * The condition to check. A column that evaluates to a boolean. + * @param e + * The error message to throw. A column that evaluates to a string. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) /** * Throws an exception with the provided error message. * + * @param c + * The error message to throw. A column that evaluates to a string. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def raise_error(c: Column): Column = Column.fn("raise_error", c) @@ -4579,6 +6577,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def user(): Column = Column.fn("user") @@ -4587,6 +6587,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def session_user(): Column = Column.fn("session_user") @@ -4596,6 +6598,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) @@ -4603,8 +6607,12 @@ object functions { * Returns an universally unique identifier (UUID) string. The value is returned as a canonical * UUID 36-character string. * + * @param seed + * The random number seed to use. A column that evaluates to an integral. Must be a constant. * @group misc_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(seed: Column): Column = Column.fn("uuid", seed) @@ -4619,25 +6627,29 @@ object functions { * GCM. * * @param input - * The binary value to encrypt. + * The binary value to encrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to encrypt the data. + * The passphrase to use to encrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param iv * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt( input: Column, @@ -4650,12 +6662,29 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv) @@ -4663,12 +6692,25 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding) @@ -4676,12 +6718,21 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_encrypt", input, key, mode) @@ -4689,12 +6740,18 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column): Column = Column.fn("aes_encrypt", input, key) @@ -4707,22 +6764,25 @@ object functions { * for decryption. The default mode is GCM. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt( input: Column, @@ -4735,11 +6795,24 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_decrypt", input, key, mode, padding) @@ -4747,11 +6820,20 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_decrypt", input, key, mode) @@ -4759,11 +6841,17 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column): Column = Column.fn("aes_decrypt", input, key) @@ -4773,22 +6861,25 @@ object functions { * NULL value instead of raising an error if the decryption cannot be performed. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt( input: Column, @@ -4801,11 +6892,24 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("try_aes_decrypt", input, key, mode, padding) @@ -4813,11 +6917,20 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("try_aes_decrypt", input, key, mode) @@ -4825,11 +6938,17 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column): Column = Column.fn("try_aes_decrypt", input, key) @@ -4837,8 +6956,12 @@ object functions { /** * Returns a sha1 hash value as a hex string of the `col`. * + * @param col + * The value to hash. A column that evaluates to a string or binary. * @group hash_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha(col: Column): Column = Column.fn("sha", col) @@ -4847,6 +6970,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_length(): Column = Column.fn("input_file_block_length") @@ -4855,6 +6980,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_start(): Column = Column.fn("input_file_block_start") @@ -4863,6 +6990,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) @@ -4872,6 +7001,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) @@ -4882,6 +7013,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) @@ -4892,14 +7025,20 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def version(): Column = Column.fn("version") /** * Return DDL-formatted type string for the data type of the input. * + * @param col + * The value whose data type is returned. A column of any type. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def typeof(col: Column): Column = Column.fn("typeof", col) @@ -4907,8 +7046,13 @@ object functions { * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default * unless specified otherwise. * + * @param cols + * The first column must be a constant integer for the number of rows, and the remaining + * columns are the input elements to be separated into rows. * @group generator_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def stack(cols: Column*): Column = Column.fn("stack", cols: _*) @@ -4920,8 +7064,14 @@ object functions { * be an integer. Otherwise if one or both of these are floating-point numbers, then the result * will also be a floating-point number. * + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column): Column = uniform(min, max, lit(SparkClassUtils.random.nextLong)) @@ -4933,8 +7083,16 @@ object functions { * integers, then the result will also be an integer. Otherwise if one or both of these are * floating-point numbers, then the result will also be a floating-point number. * + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column, seed: Column): Column = Column.fn("uniform", min, max, seed) @@ -4943,8 +7101,12 @@ object functions { * Returns a random value with independent and identically distributed (i.i.d.) uniformly * distributed values in [0, 1). * + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(seed: Column): Column = Column.fn("random", seed) @@ -4954,14 +7116,20 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(): Column = random(lit(SparkClassUtils.random.nextLong)) /** * Returns the bucket number for the given input column. * + * @param col + * The input column. A column that evaluates to an integral. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bit_position(col: Column): Column = Column.fn("bitmap_bit_position", col) @@ -4969,8 +7137,12 @@ object functions { /** * Returns the bit position for the given input column. * + * @param col + * The input column. A column that evaluates to an integral. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bucket_number(col: Column): Column = Column.fn("bitmap_bucket_number", col) @@ -4979,8 +7151,13 @@ object functions { * Returns a bitmap with the positions of the bits set from all the values from the input * column. The input column will most likely be bitmap_bit_position(). * + * @param col + * The input column will most likely be bitmap_bit_position(). A column that evaluates to an + * integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_construct_agg(col: Column): Column = Column.fn("bitmap_construct_agg", col) @@ -4988,8 +7165,12 @@ object functions { /** * Returns the number of set bits in the input bitmap. * + * @param col + * The input bitmap. A column that evaluates to a binary. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) @@ -4997,8 +7178,13 @@ object functions { * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The * input column should be bitmaps created from bitmap_construct_agg(). * + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) @@ -5006,8 +7192,13 @@ object functions { * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The * input column should be bitmaps created from bitmap_construct_agg(). * + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) @@ -5019,8 +7210,12 @@ object functions { * Computes the numeric value of the first character of the string column, and returns the * result as an int column. * + * @param e + * The target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def ascii(e: Column): Column = Column.fn("ascii", e) @@ -5028,16 +7223,24 @@ object functions { * Computes the BASE64 encoding of a binary column and returns it as a string column. This is * the reverse of unbase64. * + * @param e + * The target column to work on. A column that evaluates to a binary. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def base64(e: Column): Column = Column.fn("base64", e) /** * Calculates the bit length for the specified string column. * + * @param e + * The source column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_length(e: Column): Column = Column.fn("bit_length", e) @@ -5045,11 +7248,17 @@ object functions { * Concatenates multiple input string columns together into a single string column, using the * given separator. * + * @param sep + * The words separator. A column that evaluates to a string. Must be a constant. + * @param exprs + * The list of columns to work on. Each a column that evaluates to a string. * @note * Input strings which are null are skipped. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def concat_ws(sep: String, exprs: Column*): Column = @@ -5060,8 +7269,14 @@ object functions { * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either * argument is null, the result will also be null. * + * @param value + * The target column to work on. A column that evaluates to a binary. + * @param charset + * The charset to use to decode to. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def decode(value: Column, charset: String): Column = Column.fn("decode", value, lit(charset)) @@ -5071,8 +7286,14 @@ object functions { * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either * argument is null, the result will also be null. * + * @param value + * The target column to work on. A column that evaluates to a string. + * @param charset + * The charset to use to encode. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def encode(value: Column, charset: String): Column = Column.fn("encode", value, lit(charset)) @@ -5080,8 +7301,13 @@ object functions { /** * Returns true if the input is a valid UTF-8 string, otherwise returns false. * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_utf8(str: Column): Column = Column.fn("is_valid_utf8", str) @@ -5090,8 +7316,13 @@ object functions { * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the * Unicode replacement character (U+FFFD). * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def make_valid_utf8(str: Column): Column = Column.fn("make_valid_utf8", str) @@ -5100,8 +7331,13 @@ object functions { * Returns the input value if it corresponds to a valid UTF-8 string, or emits a * SparkIllegalArgumentException exception otherwise. * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def validate_utf8(str: Column): Column = Column.fn("validate_utf8", str) @@ -5109,8 +7345,12 @@ object functions { /** * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. * + * @param str + * the input value. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_validate_utf8(str: Column): Column = Column.fn("try_validate_utf8", str) @@ -5122,16 +7362,29 @@ object functions { * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the * result will be null. * + * @param x + * the numeric value to be formatted. A column that evaluates to a numeric. + * @param d + * the number of decimal places. A column that evaluates to an integral. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) /** * Formats the arguments in printf-style and returns the result as a string column. * + * @param format + * the format string that can contain embedded format tags. A column that evaluates to a + * string. Must be a constant. + * @param arguments + * the values to be used in formatting. Columns that evaluate to any type. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def format_string(format: String, arguments: Column*): Column = @@ -5143,8 +7396,12 @@ object functions { * * For example, "hello world" will become "Hello World". * + * @param e + * the target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def initcap(e: Column): Column = Column.fn("initcap", e) @@ -5152,12 +7409,18 @@ object functions { * Locate the position of the first occurrence of substr column in the given string. Returns * null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. Must be a constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: String): Column = instr(str, lit(substring)) @@ -5165,12 +7428,18 @@ object functions { * Locate the position of the first occurrence of substr column in the given string. Returns * null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) @@ -5178,6 +7447,13 @@ object functions { * Locate the position of the first occurrence of `substring` in `str`, starting the search from * position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5187,6 +7463,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int): Column = Column.fn("instr", str, substring, lit(start)) @@ -5195,6 +7473,12 @@ object functions { * Locate the position of the first occurrence of `substring` in `str`, starting the search from * position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5204,6 +7488,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column): Column = Column.fn("instr", str, substring, start) @@ -5212,6 +7498,16 @@ object functions { * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the * search from position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. Must + * be a constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5223,6 +7519,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = Column.fn("instr", str, substring, lit(start), lit(occurrence)) @@ -5231,6 +7529,14 @@ object functions { * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the * search from position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5242,6 +7548,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = Column.fn("instr", str, substring, start, occurrence) @@ -5251,8 +7559,12 @@ object functions { * length of character strings include the trailing spaces. The length of binary strings * includes binary zeros. * + * @param e + * the target column to work on. A column that evaluates to a string or binary. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def length(e: Column): Column = Column.fn("length", e) @@ -5261,24 +7573,39 @@ object functions { * length of character strings include the trailing spaces. The length of binary strings * includes binary zeros. * + * @param e + * the target column to work on. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def len(e: Column): Column = Column.fn("len", e) /** * Converts a string column to lower case. * + * @param e + * the target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def lower(e: Column): Column = Column.fn("lower", e) /** * Computes the Levenshtein distance of the two given string columns if it's less than or equal * to a given threshold. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @param threshold + * the maximum distance to compute. A column that evaluates to an integral. Must be a + * constant. * @return - * result distance, or -1 + * result distance, or -1. Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -5287,16 +7614,28 @@ object functions { /** * Computes the Levenshtein distance of the two given string columns. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) /** * Computes the Jaro-Winkler similarity between the two given string columns. The result is a * double between 0.0 (no similarity) and 1.0 (identical). + * @param l + * A column that evaluates to a string. + * @param r + * A column that evaluates to a string. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a double. */ def jaro_winkler_similarity(l: Column, r: Column): Column = Column.fn("jaro_winkler_similarity", l, r) @@ -5304,24 +7643,38 @@ object functions { /** * Locate the position of the first occurrence of substr. * + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) /** * Locate the position of the first occurrence of substr in a string column, after position pos. * + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. + * @param pos + * The starting position. A column that evaluates to an integer. * @note * The position is not zero based, but 1 based index. returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column, pos: Int): Column = Column.fn("locate", lit(substr), str, lit(pos)) @@ -5330,8 +7683,16 @@ object functions { * Left-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) @@ -5339,8 +7700,16 @@ object functions { * Left-pad the binary column with pad to a byte length of len. If the binary column is longer * than len, the return value is shortened to len bytes. * + * @param str + * A column that evaluates to a binary. + * @param len + * The byte length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding bytes. A column that evaluates to a binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) @@ -5348,78 +7717,134 @@ object functions { * Left-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) /** * Trim the spaces from left end for the specified string value. * + * @param e + * A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column): Column = Column.fn("ltrim", e) /** * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trimString + * The trim string. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) /** * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trim + * The trim string. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) /** * Calculates the byte length for the specified string column. * + * @param e + * A column that evaluates to a string or binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def octet_length(e: Column): Column = Column.fn("octet_length", e) /** * Marks a given column with specified collation. * + * @param e + * A column that evaluates to a string. + * @param collation + * The collation name. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) /** * Returns the collation name of a given column. * + * @param e + * A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collation(e: Column): Column = Column.fn("collation", e) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) @@ -5427,8 +7852,14 @@ object functions { * Returns a count of the number of times that the regular expression pattern `regexp` is * matched in the string `str`. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) @@ -5438,8 +7869,16 @@ object functions { * the specified group index exceeds the group count of regex, an IllegalArgumentException will * be thrown. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param exp + * regex pattern to apply. A string. Must be a constant. + * @param groupIdx + * matched group id. An integer. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) @@ -5448,8 +7887,14 @@ object functions { * Extract all strings in the `str` that match the `regexp` expression and corresponding to the * first regex group index. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column): Column = Column.fn("regexp_extract_all", str, regexp) @@ -5458,8 +7903,16 @@ object functions { * Extract all strings in the `str` that match the `regexp` expression and corresponding to the * regex group index. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_extract_all", str, regexp, idx) @@ -5467,8 +7920,16 @@ object functions { /** * Replace all substrings of the specified string value that match regexp with rep. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String): Column = regexp_replace(e, lit(pattern), lit(replacement)) @@ -5477,8 +7938,18 @@ object functions { * Replace all substrings of the specified string value that match regexp with rep, starting at * the specified position `pos`. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @param pos + * position to start replacement. The first position is 1. An integer. Must be a constant. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) @@ -5486,8 +7957,16 @@ object functions { /** * Replace all substrings of the specified string value that match regexp with rep. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. * @group string_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = Column.fn("regexp_replace", e, pattern, replacement) @@ -5496,8 +7975,19 @@ object functions { * Replace all substrings of the specified string value that match regexp with rep, starting at * the specified position `pos`. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @param pos + * position to start replacement. The first position is 1. A column that evaluates to an + * integer. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = Column.fn("regexp_replace", e, pattern, replacement, pos) @@ -5506,8 +7996,14 @@ object functions { * Returns the substring that matches the regular expression `regexp` within the string `str`. * If the regular expression is not found, the result is null. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) @@ -5516,8 +8012,14 @@ object functions { * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match * is found, returns 0. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) @@ -5526,8 +8028,16 @@ object functions { * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match * is found, returns 0. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_instr", str, regexp, idx) @@ -5536,8 +8046,12 @@ object functions { * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse * of base64. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unbase64(e: Column): Column = Column.fn("unbase64", e) @@ -5545,8 +8059,16 @@ object functions { * Right-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param len + * length of the final string. An integer. Must be a constant. + * @param pad + * chars to append. A string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) @@ -5554,8 +8076,16 @@ object functions { * Right-pad the binary column with pad to a byte length of len. If the binary column is longer * than len, the return value is shortened to len bytes. * + * @param str + * target column to work on. A column that evaluates to a binary. + * @param len + * byte length of the final binary. An integer. Must be a constant. + * @param pad + * bytes to append. A binary. Must be a constant. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) @@ -5563,54 +8093,95 @@ object functions { * Right-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param len + * length of the final result. A column that evaluates to an integer. + * @param pad + * chars or bytes to append. A column that evaluates to a string or binary. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) /** * Repeats a string column n times, and returns it as a new string column. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. Must be a + * constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) /** * Repeats a string column n times, and returns it as a new string column. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) /** * Trim the spaces from right end for the specified string value. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column): Column = Column.fn("rtrim", e) /** * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trimString + * the trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) /** * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trim + * the trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) /** * Returns the soundex code for the specified expression. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def soundex(e: Column): Column = Column.fn("soundex", e) @@ -5625,6 +8196,8 @@ object functions { * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) @@ -5639,6 +8212,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) @@ -5659,6 +8234,8 @@ object functions { * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String, limit: Int): Column = Column.fn("split", str, lit(pattern), lit(limit)) @@ -5680,6 +8257,8 @@ object functions { * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column, limit: Column): Column = Column.fn("split", str, pattern, limit) @@ -5688,11 +8267,19 @@ object functions { * Substring starts at `pos` and is of length `len` when str is String type or returns the slice * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. Must be a constant. + * @param len + * length of chars. A column that evaluates to an integral. Must be a constant. * @note * The position is not zero based, but 1 based index. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Int, len: Int): Column = Column.fn("substring", str, lit(pos), lit(len)) @@ -5701,11 +8288,19 @@ object functions { * Substring starts at `pos` and is of length `len` when str is String type or returns the slice * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. + * @param len + * length of chars. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Column, len: Column): Column = Column.fn("substring", str, pos, len) @@ -5716,7 +8311,15 @@ object functions { * returned. If count is negative, every to the right of the final delimiter (counting from the * right) is returned. substring_index performs a case-sensitive match when searching for delim. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param delim + * delimiter of values. A column that evaluates to a string. Must be a constant. + * @param count + * number of occurrences. A column that evaluates to an integral. Must be a constant. * @group string_funcs + * @return + * Returns a column that evaluates to a string. */ def substring_index(str: Column, delim: String, count: Int): Column = Column.fn("substring_index", str, lit(delim), lit(count)) @@ -5725,8 +8328,18 @@ object functions { * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of * `src` and proceeding for `len` bytes. * + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @param len + * the number of bytes to replace in src. A column that evaluates to an integral. * @group string_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the input. */ def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = Column.fn("overlay", src, replace, pos, len) @@ -5735,16 +8348,32 @@ object functions { * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of * `src`. * + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def overlay(src: Column, replace: Column, pos: Column): Column = Column.fn("overlay", src, replace, pos) /** * Splits a string into arrays of sentences, where each sentence is an array of words. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @param country + * a country of the locale. A column that evaluates to a string. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column, country: Column): Column = Column.fn("sentences", string, language, country) @@ -5752,8 +8381,14 @@ object functions { /** * Splits a string into arrays of sentences, where each sentence is an array of words. The * default `country`('') is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column): Column = Column.fn("sentences", string, language) @@ -5761,8 +8396,12 @@ object functions { /** * Splits a string into arrays of sentences, where each sentence is an array of words. The * default locale is used. + * @param string + * a string to be split. A column that evaluates to a string. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column): Column = Column.fn("sentences", string) @@ -5771,8 +8410,16 @@ object functions { * replaceString correspond to the characters in matchingString. The translate will happen when * any character in the string matches the character in the `matchingString`. * + * @param src + * source column to work on. A column that evaluates to a string. + * @param matchingString + * matching characters. A column that evaluates to a string. Must be a constant. + * @param replaceString + * characters for replacement. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def translate(src: Column, matchingString: String, replaceString: String): Column = Column.fn("translate", src, lit(matchingString), lit(replaceString)) @@ -5780,30 +8427,50 @@ object functions { /** * Trim the spaces from both ends for the specified string column. * + * @param e + * The string column to trim. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column): Column = Column.fn("trim", e) /** * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trimString + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) /** * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) /** * Converts a string column to upper case. * + * @param e + * The input column to convert to upper case. A column that evaluates to a string. * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def upper(e: Column): Column = Column.fn("upper", e) @@ -5813,8 +8480,15 @@ object functions { * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at * least one of the input parameters is NULL. * + * @param e + * The input value to convert. A column that evaluates to a string. + * @param f + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) @@ -5822,8 +8496,12 @@ object functions { * Converts the input `e` to a binary value based on the default format "hex". The function * returns NULL if at least one of the input parameters is NULL. * + * @param e + * The input value to convert. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column): Column = Column.fn("to_binary", e) @@ -5851,8 +8529,16 @@ object functions { * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input * binary is decoded to UTF-8 string. * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) @@ -5881,8 +8567,16 @@ object functions { * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input * binary is decoded to UTF-8 string. * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) @@ -5904,8 +8598,15 @@ object functions { * Only allowed at the end of the format string; specifies that 'expr' indicates a negative * number with wrapping angled brackets. * + * @param e + * The input string to convert to a number. A column that evaluates to a string. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) @@ -5913,15 +8614,18 @@ object functions { * Replaces all occurrences of `search` with `replace`. * * @param src - * A column of string to be replaced + * A column of string to be replaced. A column that evaluates to a string. * @param search - * A column of string, If `search` is not found in `str`, `str` is returned unchanged. + * A column of string, If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. * @param replace * A column of string, If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. + * the string that is removed from `str`. A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column, replace: Column): Column = Column.fn("replace", src, search, replace) @@ -5936,6 +8640,8 @@ object functions { * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) @@ -5945,8 +8651,16 @@ object functions { * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. * + * @param str + * A column of string to be split. A column that evaluates to a string. + * @param delimiter + * The delimiter used for split. A column that evaluates to a string. + * @param partNum + * The requested part of the split (1-based). A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def split_part(str: Column, delimiter: Column, partNum: Column): Column = Column.fn("split_part", str, delimiter, partNum) @@ -5955,8 +8669,16 @@ object functions { * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of * byte array that starts at `pos` and is of length `len`. * + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @param len + * The length of the substring. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column, len: Column): Column = Column.fn("substr", str, pos, len) @@ -5965,16 +8687,30 @@ object functions { * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts * at `pos`. * + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("try_parse_url", url, partToExtract, key) @@ -5982,8 +8718,14 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column): Column = Column.fn("try_parse_url", url, partToExtract) @@ -5991,8 +8733,16 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("parse_url", url, partToExtract, key) @@ -6000,8 +8750,14 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column): Column = Column.fn("parse_url", url, partToExtract) @@ -6009,8 +8765,14 @@ object functions { /** * Formats the arguments in printf-style and returns the result as a string column. * + * @param format + * A format string that can contain embedded format tags. A column that evaluates to a string. + * @param arguments + * The values to be used in formatting. Columns that evaluate to any type. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def printf(format: Column, arguments: Column*): Column = @@ -6020,8 +8782,12 @@ object functions { * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding * scheme. * + * @param str + * A URL-encoded string. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_decode(str: Column): Column = Column.fn("url_decode", str) @@ -6029,8 +8795,12 @@ object functions { * This is a special version of `url_decode` that performs the same operation, but returns a * NULL value instead of raising an error if the decoding cannot be performed. * + * @param str + * A URL-encoded string. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) @@ -6038,8 +8808,12 @@ object functions { * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding * scheme. * + * @param str + * A string to encode. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_encode(str: Column): Column = Column.fn("url_encode", str) @@ -6047,8 +8821,16 @@ object functions { * Returns the position of the first occurrence of `substr` in `str` after position `start`. The * given `start` and return value are 1-based. * + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @param start + * The 1-based position to start the search from. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column, start: Column): Column = Column.fn("position", substr, str, start) @@ -6057,8 +8839,14 @@ object functions { * Returns the position of the first occurrence of `substr` in `str` after position `1`. The * return value are 1-based. * + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column): Column = Column.fn("position", substr, str) @@ -6068,8 +8856,14 @@ object functions { * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY * type. * + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param suffix + * The suffix to test for. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def endswith(str: Column, suffix: Column): Column = Column.fn("endswith", str, suffix) @@ -6079,8 +8873,14 @@ object functions { * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY * type. * + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param prefix + * The prefix to test for. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def startswith(str: Column, prefix: Column): Column = Column.fn("startswith", str, prefix) @@ -6089,24 +8889,38 @@ object functions { * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the * result is equivalent to char(n % 256) * + * @param n + * The code point value. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def char(n: Column): Column = Column.fn("char", n) /** * Removes the leading and trailing space characters from `str`. * + * @param str + * The string to trim. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column): Column = Column.fn("btrim", str) /** * Remove the leading and trailing `trim` characters from `str`. * + * @param str + * The string to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) @@ -6114,8 +8928,15 @@ object functions { * This is a special version of `to_binary` that performs the same operation, but returns a NULL * value instead of raising an error if the conversion cannot be performed. * + * @param e + * The string to convert. A column that evaluates to a string. + * @param f + * The format to use for the conversion. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) @@ -6123,8 +8944,12 @@ object functions { * This is a special version of `to_binary` that performs the same operation, but returns a NULL * value instead of raising an error if the conversion cannot be performed. * + * @param e + * The string to convert. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) @@ -6133,8 +8958,15 @@ object functions { * string `e` does not match the expected format. The format follows the same semantics as the * to_number function. * + * @param e + * The string to convert. A column that evaluates to a string. + * @param format + * The format used to convert the string to a number. A column that evaluates to a string. + * Must be a constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) @@ -6142,8 +8974,12 @@ object functions { * Returns the character length of string data or number of bytes of binary data. The length of * string data includes the trailing spaces. The length of binary data includes binary zeros. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def char_length(str: Column): Column = Column.fn("char_length", str) @@ -6151,8 +8987,12 @@ object functions { * Returns the character length of string data or number of bytes of binary data. The length of * string data includes the trailing spaces. The length of binary data includes binary zeros. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def character_length(str: Column): Column = Column.fn("character_length", str) @@ -6160,8 +9000,12 @@ object functions { * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the * result is equivalent to chr(n % 256) * + * @param n + * The code point. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def chr(n: Column): Column = Column.fn("chr", n) @@ -6170,8 +9014,14 @@ object functions { * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or * BINARY type. * + * @param left + * The input to check, may be NULL. A column that evaluates to a string or binary. + * @param right + * The input to find, may be NULL. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) @@ -6181,8 +9031,13 @@ object functions { * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid * indices. * + * @param inputs + * The index followed by the inputs to select from. Columns where the first evaluates to an + * integral and the rest evaluate to strings or binaries. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) @@ -6192,8 +9047,14 @@ object functions { * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains * a comma. * + * @param str + * The given string to be found. A column that evaluates to a string. + * @param strArray + * The comma-delimited list. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) @@ -6201,8 +9062,16 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, * false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("like", str, pattern, escapeChar) @@ -6211,8 +9080,14 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, * false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) @@ -6220,8 +9095,16 @@ object functions { * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any * arguments are null, false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("ilike", str, pattern, escapeChar) @@ -6230,24 +9113,38 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any * arguments are null, false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) /** * Returns `str` with all characters changed to lowercase. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def lcase(str: Column): Column = Column.fn("lcase", str) /** * Returns `str` with all characters changed to uppercase. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def ucase(str: Column): Column = Column.fn("ucase", str) @@ -6255,8 +9152,14 @@ object functions { * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if * `len` is less or equal than 0 the result is an empty string. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @param len + * The number of leftmost characters. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def left(str: Column, len: Column): Column = Column.fn("left", str, len) @@ -6264,8 +9167,14 @@ object functions { * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if * `len` is less or equal than 0 the result is an empty string. * + * @param str + * Input column or strings. A column that evaluates to a string. + * @param len + * The number of rightmost characters. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def right(str: Column, len: Column): Column = Column.fn("right", str, len) @@ -6273,8 +9182,12 @@ object functions { * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded * by a backslash. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def quote(str: Column): Column = Column.fn("quote", str) @@ -6286,8 +9199,12 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches HllSketch. * + * @param c + * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) @@ -6295,8 +9212,12 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches HllSketch. * + * @param columnName + * Name of the column containing the binary representation of a Datasketches HllSketch. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(columnName: String): Column = { hll_sketch_estimate(Column(columnName)) @@ -6306,8 +9227,16 @@ object functions { * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches * Union object. Throws an exception if sketches have different lgConfigK values. * + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column): Column = Column.fn("hll_union", c1, c2) @@ -6316,8 +9245,14 @@ object functions { * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches * Union object. Throws an exception if sketches have different lgConfigK values. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(columnName1: String, columnName2: String): Column = { hll_union(Column(columnName1), Column(columnName2)) @@ -6328,8 +9263,19 @@ object functions { * Union object. Throws an exception if sketches have different lgConfigK values and * allowDifferentLgConfigK is set to false. * + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) @@ -6339,8 +9285,17 @@ object functions { * Union object. Throws an exception if sketches have different lgConfigK values and * allowDifferentLgConfigK is set to false. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union( columnName1: String, @@ -6353,8 +9308,16 @@ object functions { * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns * using a Datasketches AnotB object * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(c1: Column, c2: Column): Column = Column.fn("theta_difference", c1, c2) @@ -6363,8 +9326,16 @@ object functions { * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns * using a Datasketches AnotB object * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(columnName1: String, columnName2: String): Column = { theta_difference(Column(columnName1), Column(columnName2)) @@ -6374,8 +9345,16 @@ object functions { * Intersects two binary representations of Datasketches ThetaSketch objects in the input * columns using a Datasketches Intersection object * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(c1: Column, c2: Column): Column = Column.fn("theta_intersection", c1, c2) @@ -6384,8 +9363,16 @@ object functions { * Intersects two binary representations of Datasketches ThetaSketch objects in the input * columns using a Datasketches Intersection object * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(columnName1: String, columnName2: String): Column = { theta_intersection(Column(columnName1), Column(columnName2)) @@ -6395,8 +9382,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches ThetaSketch. * + * @param c + * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) @@ -6404,8 +9396,12 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches ThetaSketch. * + * @param columnName + * Name of the column containing the binary representation of a Datasketches ThetaSketch. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(columnName: String): Column = { theta_sketch_estimate(Column(columnName)) @@ -6416,8 +9412,16 @@ object functions { * using a Datasketches Union object. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column): Column = Column.fn("theta_union", c1, c2) @@ -6427,8 +9431,16 @@ object functions { * using a Datasketches Union object. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String): Column = { theta_union(Column(columnName1), Column(columnName2)) @@ -6439,8 +9451,19 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("theta_union", c1, c2, lit(lgNomEntries)) @@ -6450,8 +9473,17 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param columnName1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param columnName2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { theta_union(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6462,8 +9494,17 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param c1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param c2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = Column.fn("theta_union", c1, c2, lgNomEntries) @@ -6473,8 +9514,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_double", c1, c2) @@ -6484,8 +9531,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(columnName1: String, columnName2: String): Column = tuple_difference_double(Column(columnName1), Column(columnName2)) @@ -6495,8 +9548,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_integer", c1, c2) @@ -6506,8 +9565,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(columnName1: String, columnName2: String): Column = tuple_difference_integer(Column(columnName1), Column(columnName2)) @@ -6518,8 +9583,14 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_double", c1, c2) @@ -6530,8 +9601,14 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2)) @@ -6542,8 +9619,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_double", c1, c2, lit(mode)) @@ -6554,8 +9640,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2), mode) @@ -6566,8 +9661,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_double", c1, c2, mode) @@ -6578,8 +9682,14 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_integer", c1, c2) @@ -6590,8 +9700,14 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2)) @@ -6602,8 +9718,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) @@ -6614,8 +9739,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) @@ -6626,8 +9760,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_integer", c1, c2, mode) @@ -6636,8 +9779,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with double summary data type. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(c: Column): Column = Column.fn("tuple_sketch_estimate_double", c) @@ -6646,8 +9794,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with double summary data type. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(columnName: String): Column = tuple_sketch_estimate_double(Column(columnName)) @@ -6656,8 +9809,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with integer summary data type. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(c: Column): Column = Column.fn("tuple_sketch_estimate_integer", c) @@ -6666,8 +9824,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with integer summary data type. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(columnName: String): Column = tuple_sketch_estimate_integer(Column(columnName)) @@ -6677,8 +9840,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column): Column = Column.fn("tuple_sketch_summary_double", c) @@ -6688,8 +9856,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String): Column = tuple_sketch_summary_double(Column(columnName)) @@ -6698,8 +9871,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_double", c, lit(mode)) @@ -6708,8 +9889,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String, mode: String): Column = tuple_sketch_summary_double(Column(columnName), mode) @@ -6718,8 +9907,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_double", c, mode) @@ -6729,8 +9926,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column): Column = Column.fn("tuple_sketch_summary_integer", c) @@ -6740,8 +9942,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String): Column = tuple_sketch_summary_integer(Column(columnName)) @@ -6750,8 +9957,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_integer", c, lit(mode)) @@ -6760,8 +9975,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String, mode: String): Column = tuple_sketch_summary_integer(Column(columnName), mode) @@ -6770,8 +9993,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_integer", c, mode) @@ -6781,8 +10012,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(c: Column): Column = Column.fn("tuple_sketch_theta_double", c) @@ -6792,8 +10028,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(columnName: String): Column = tuple_sketch_theta_double(Column(columnName)) @@ -6803,8 +10044,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(c: Column): Column = Column.fn("tuple_sketch_theta_integer", c) @@ -6814,8 +10060,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(columnName: String): Column = tuple_sketch_theta_integer(Column(columnName)) @@ -6825,8 +10076,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_double", c1, c2) @@ -6836,8 +10093,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String): Column = tuple_union_double(Column(columnName1), Column(columnName2)) @@ -6848,8 +10111,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) @@ -6860,8 +10132,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6872,8 +10153,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6884,8 +10177,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double( columnName1: String, @@ -6900,8 +10205,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) @@ -6911,8 +10228,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_integer", c1, c2) @@ -6922,8 +10245,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String): Column = tuple_union_integer(Column(columnName1), Column(columnName2)) @@ -6934,8 +10263,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) @@ -6946,8 +10284,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6958,8 +10305,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6970,8 +10329,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer( columnName1: String, @@ -6986,8 +10357,18 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) @@ -6997,8 +10378,14 @@ object functions { * double summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_double", c1, c2) @@ -7008,8 +10395,14 @@ object functions { * double summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = tuple_difference_theta_double(Column(columnName1), Column(columnName2)) @@ -7019,8 +10412,14 @@ object functions { * integer summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_integer", c1, c2) @@ -7030,8 +10429,14 @@ object functions { * integer summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) @@ -7042,8 +10447,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2) @@ -7054,8 +10465,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) @@ -7066,8 +10483,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) @@ -7078,8 +10504,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double( columnName1: String, @@ -7093,8 +10528,16 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2, mode) @@ -7105,8 +10548,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2) @@ -7117,8 +10566,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) @@ -7129,8 +10584,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) @@ -7141,8 +10605,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer( columnName1: String, @@ -7156,8 +10629,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, mode) @@ -7167,8 +10649,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_double", c1, c2) @@ -7178,8 +10666,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(columnName1: String, columnName2: String): Column = tuple_union_theta_double(Column(columnName1), Column(columnName2)) @@ -7190,8 +10684,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) @@ -7202,8 +10705,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7217,8 +10729,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7229,8 +10753,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7245,8 +10781,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( c1: Column, @@ -7260,8 +10808,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_integer", c1, c2) @@ -7271,8 +10825,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = tuple_union_theta_integer(Column(columnName1), Column(columnName2)) @@ -7283,8 +10843,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) @@ -7295,8 +10864,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7310,8 +10888,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7322,8 +10912,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7338,8 +10940,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( c1: Column, @@ -7351,8 +10965,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL bigint sketch. * + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_bigint(e: Column): Column = Column.fn("kll_sketch_to_string_bigint", e) @@ -7360,8 +10978,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL float sketch. * + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_float(e: Column): Column = Column.fn("kll_sketch_to_string_float", e) @@ -7369,8 +10991,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL double sketch. * + * @param e + * The KLL double sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_double(e: Column): Column = Column.fn("kll_sketch_to_string_double", e) @@ -7378,8 +11004,12 @@ object functions { /** * Returns the number of items collected in the KLL bigint sketch. * + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_bigint(e: Column): Column = Column.fn("kll_sketch_get_n_bigint", e) @@ -7387,8 +11017,12 @@ object functions { /** * Returns the number of items collected in the KLL float sketch. * + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_float(e: Column): Column = Column.fn("kll_sketch_get_n_float", e) @@ -7396,8 +11030,12 @@ object functions { /** * Returns the number of items collected in the KLL double sketch. * + * @param e + * The KLL double sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_double(e: Column): Column = Column.fn("kll_sketch_get_n_double", e) @@ -7405,8 +11043,14 @@ object functions { /** * Merges two KLL bigint sketch buffers together into one. * + * @param left + * The first KLL bigint sketch. A column that evaluates to a binary. + * @param right + * The second KLL bigint sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_bigint(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_bigint", left, right) @@ -7414,8 +11058,14 @@ object functions { /** * Merges two KLL float sketch buffers together into one. * + * @param left + * The first KLL float sketch. A column that evaluates to a binary. + * @param right + * The second KLL float sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_float(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_float", left, right) @@ -7423,8 +11073,14 @@ object functions { /** * Merges two KLL double sketch buffers together into one. * + * @param left + * The first KLL double sketch. A column that evaluates to a binary. + * @param right + * The second KLL double sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_double(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_double", left, right) @@ -7433,8 +11089,15 @@ object functions { * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) @@ -7443,8 +11106,15 @@ object functions { * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a float. */ def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_float", sketch, rank) @@ -7453,8 +11123,15 @@ object functions { * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_double", sketch, rank) @@ -7463,8 +11140,14 @@ object functions { * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile * can be a single value or an array. * + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to an integral or an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) @@ -7473,8 +11156,14 @@ object functions { * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can * be a single value or an array. * + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_float", sketch, quantile) @@ -7483,8 +11172,15 @@ object functions { * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile * can be a single value or an array. * + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be + * a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double. */ def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_double", sketch, quantile) @@ -7498,11 +11194,14 @@ object functions { * * @param startDate * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7518,7 +11217,8 @@ object functions { * @param numMonths * A column of the number of months to add to `startDate`, can be negative to subtract months * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7531,6 +11231,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def curdate(): Column = Column.fn("curdate") @@ -7540,6 +11242,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def current_date(): Column = Column.fn("current_date") @@ -7548,6 +11252,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_timezone(): Column = Column.fn("current_timezone") @@ -7557,6 +11263,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def current_timestamp(): Column = Column.fn("current_timestamp") @@ -7565,6 +11273,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def now(): Column = Column.fn("now") @@ -7575,6 +11285,8 @@ object functions { * * @group datetime_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def localtimestamp(): Column = Column.fn("localtimestamp") @@ -7587,11 +11299,14 @@ object functions { * * @param dateExpr * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993` + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns + * a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a * specialized implementation. @@ -7608,11 +11323,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to add to `start`, can be negative to subtract days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7627,7 +11345,8 @@ object functions { * @param days * A column of the number of days to add to `start`, can be negative to subtract days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7638,11 +11357,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * A column of the number of days to add to `start`, can be negative to subtract days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.5.0 */ @@ -7653,11 +11375,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to subtract from `start`, can be negative to add days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7672,7 +11397,8 @@ object functions { * @param days * A column of the number of days to subtract from `start`, can be negative to add days * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7690,13 +11416,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7713,13 +11441,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7728,15 +11458,23 @@ object functions { /** * Create date from the number of `days` since 1970-01-01. * + * @param days + * The number of days since 1970-01-01. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) /** * Extracts the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the year from. A column that evaluates to a date, + * timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7744,8 +11482,12 @@ object functions { /** * Extracts the quarter as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the quarter from. A column that evaluates to a + * date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7753,8 +11495,12 @@ object functions { /** * Extracts the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the month from. A column that evaluates to a date, + * timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7763,8 +11509,12 @@ object functions { /** * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 * for a Sunday through to 7 for a Saturday + * @param e + * The date, timestamp or string to extract the day of the week from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 */ @@ -7772,8 +11522,12 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7781,8 +11535,12 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7790,8 +11548,12 @@ object functions { /** * Extracts the day of the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the year from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7801,8 +11563,12 @@ object functions { * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the hours from. A column that evaluates to a date, time, timestamp or + * string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7814,9 +11580,10 @@ object functions { * @param field * selects which part of the source should be extracted. * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * a date, time, timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7831,9 +11598,10 @@ object functions { * selects which part of the source should be extracted, and supported string values are as * same as the fields of the equivalent function `extract`. * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * a date/timestamp or time or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7850,7 +11618,8 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7864,9 +11633,11 @@ object functions { * * @param e * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * A date, or null if the input was a string that could not be cast to a date + * A date, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7876,8 +11647,12 @@ object functions { * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the minutes from. A column that evaluates to a date, time, timestamp + * or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7886,14 +11661,25 @@ object functions { /** * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). * + * @param e + * The column to extract the day of the week from. A column that evaluates to a date, + * timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def weekday(e: Column): Column = Column.fn("weekday", e) /** + * @param year + * The year to build the date. A column that evaluates to an integral. + * @param month + * The month to build the date. A column that evaluates to an integral. + * @param day + * The day to build the date. A column that evaluates to an integral. * @return - * A date created from year, month and day fields. + * A date created from year, month and day fields. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.3.0 */ @@ -7922,7 +11708,8 @@ object functions { * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` * @return * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start` + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 1.5.0 */ @@ -7932,8 +11719,21 @@ object functions { /** * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the * result is rounded off to 8 digits; it is not rounded otherwise. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. + * @param roundOff + * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be + * a constant. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a double. */ def months_between(end: Column, start: Column, roundOff: Boolean): Column = Column.fn("months_between", end, start, lit(roundOff)) @@ -7947,12 +11747,14 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7973,7 +11775,7 @@ object functions { * "Fri", "Sat", "Sun" * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.2.0 */ @@ -7984,8 +11786,12 @@ object functions { * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the seconds from. A column that evaluates to a date, time, timestamp + * or string. * @return - * An integer, or null if the input was a string that could not be cast to a timestamp + * An integer, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7997,8 +11803,12 @@ object functions { * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, * as defined by ISO 8601 * + * @param e + * The column to extract the week number from. A column that evaluates to a date, timestamp or + * string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8013,7 +11823,8 @@ object functions { * A number of a type that is castable to a long, such as string or integer. Can be negative * for timestamps before the unix epoch * @return - * A string, or null if the input was a string that could not be cast to a long + * A string, or null if the input was a string that could not be cast to a long. Returns a + * column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8034,7 +11845,7 @@ object functions { * A date time pattern that the input will be formatted to * @return * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern + * invalid date time pattern. Returns a column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8050,6 +11861,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_timestamp(): Column = unix_timestamp(current_timestamp()) @@ -8061,7 +11874,8 @@ object functions { * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` * format * @return - * A long, or null if the input was a string not of the correct format + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8080,7 +11894,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format + * format. Returns a column that evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8093,7 +11907,8 @@ object functions { * @param str * A string to be parsed to time. * @return - * A time, or raises an error if the input is malformed. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * * @group datetime_funcs * @since 4.1.0 @@ -8113,8 +11928,8 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or raises an error if the input is malformed. - * + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * @group datetime_funcs * @since 4.1.0 */ @@ -8127,9 +11942,11 @@ object functions { * * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp + * A timestamp, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8148,7 +11965,7 @@ object functions { * A date time pattern detailing the format of `s` when `s` is a string * @return * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format + * an invalid format. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8160,7 +11977,7 @@ object functions { * @param str * A string to be parsed to time. * @return - * A time, or null if the input is malformed. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * * @group datetime_funcs * @since 4.1.0 @@ -8180,8 +11997,7 @@ object functions { * @param format * A time format pattern to follow. * @return - * A time, or null if the input is malformed. - * + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -8194,8 +12010,14 @@ object functions { * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with * the value of configuration `spark.sql.timestampType`. * + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @param format + * Format to use to convert timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column, format: Column): Column = Column.fn("try_to_timestamp", s, format) @@ -8205,16 +12027,24 @@ object functions { * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result * data type is consistent with the value of configuration `spark.sql.timestampType`. * + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) /** * Converts the column into `DateType` by casting rules to `DateType`. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def to_date(e: Column): Column = Column.fn("to_date", e) @@ -8231,7 +12061,7 @@ object functions { * A date time pattern detailing the format of `e` when `e`is a string * @return * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format + * invalid format. Returns a column that evaluates to a date. * @group datetime_funcs * @since 2.2.0 */ @@ -8241,8 +12071,12 @@ object functions { * This is a special version of `to_date` that performs the same operation, but returns a NULL * value instead of raising an error if date cannot be created. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column): Column = Column.fn("try_to_date", e) @@ -8250,24 +12084,39 @@ object functions { * This is a special version of `to_date` that performs the same operation, but returns a NULL * value instead of raising an error if date cannot be created. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @param fmt + * Format to use to convert date values. A column that evaluates to a string. Must be a + * constant. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) /** * Returns the number of days since 1970-01-01. * + * @param e + * Input column of values to convert. A column that evaluates to a date. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def unix_date(e: Column): Column = Column.fn("unix_date", e) /** * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. * + * @param e + * Input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_micros(e: Column): Column = Column.fn("unix_micros", e) @@ -8276,8 +12125,13 @@ object functions { * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a * lossless `DECIMAL(21, 0)`. * + * @param e + * input column of nanosecond-precision timestamp values to convert. A column that evaluates + * to a timestamp. * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a decimal. */ def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) @@ -8285,8 +12139,12 @@ object functions { * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of * precision. * + * @param e + * input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_millis(e: Column): Column = Column.fn("unix_millis", e) @@ -8294,8 +12152,12 @@ object functions { * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of * precision. * + * @param e + * input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) @@ -8306,14 +12168,15 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param format: * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter' + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value + * invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8327,13 +12190,14 @@ object functions { * @param format: * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter' + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. * @param timestamp * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value + * `format` was an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8347,16 +12211,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8366,8 +12232,17 @@ object functions { * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 * 03:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def from_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("from_utc_timestamp", ts, tz) @@ -8379,16 +12254,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8398,8 +12275,17 @@ object functions { * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield * '2017-07-14 01:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) @@ -8447,6 +12333,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window( timeColumn: Column, @@ -8496,6 +12384,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { window(timeColumn, windowDuration, slideDuration, "0 second") @@ -8534,6 +12424,8 @@ object functions { * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String): Column = { window(timeColumn, windowDuration, windowDuration, "0 second") @@ -8552,6 +12444,8 @@ object functions { * * @group datetime_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) @@ -8578,6 +12472,8 @@ object functions { * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: String): Column = session_window(timeColumn, lit(gapDuration)) @@ -8614,30 +12510,44 @@ object functions { * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: Column): Column = Column.fn("session_window", timeColumn, gapDuration) /** * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. + * @param e + * unix time values. A column that evaluates to a numeric. * @group datetime_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) /** * Creates timestamp from the number of milliseconds since UTC epoch. * + * @param e + * unix time values. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) /** * Creates timestamp from the number of microseconds since UTC epoch. * + * @param e + * unix time values. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) @@ -8645,8 +12555,12 @@ object functions { * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from * the number of nanoseconds since UTC epoch. * + * @param e + * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) @@ -8654,8 +12568,19 @@ object functions { * Gets the difference between the timestamps in the specified units by truncating the fraction * part. * + * @param unit + * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', + * 'HOUR'. A column that evaluates to a string. Must be a constant. + * @param start + * A timestamp which the expression subtracts from `end`. A column that evaluates to a + * timestamp. + * @param end + * A timestamp from which the expression subtracts `start`. A column that evaluates to a + * timestamp. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a long. */ def timestamp_diff(unit: String, start: Column, end: Column): Column = Column.internalFn("timestampdiff", lit(unit), start, end) @@ -8663,8 +12588,17 @@ object functions { /** * Adds the specified number of units to the given timestamp. * + * @param unit + * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates + * to a string. Must be a constant. + * @param quantity + * the number of units of time to add. A column that evaluates to an integral. + * @param ts + * A timestamp to which to add. A column that evaluates to a timestamp. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def timestamp_add(unit: String, quantity: Column, ts: Column): Column = Column.internalFn("timestampadd", lit(unit), quantity, ts) @@ -8681,6 +12615,8 @@ object functions { * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column): Column = Column.fn("time_bucket", bucketSize, ts) @@ -8699,6 +12635,8 @@ object functions { * Alignment anchor. Must be the same type as `ts` and must be foldable. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = Column.fn("time_bucket", bucketSize, ts, origin) @@ -8709,13 +12647,15 @@ object functions { * * @param unit * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param start - * A starting TIME. + * A starting TIME. A column that evaluates to a time. * @param end - * An ending TIME. + * An ending TIME. A column that evaluates to a time. * @return - * The difference between `end` and `start` times, measured in specified units. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @group datetime_funcs @@ -8730,11 +12670,12 @@ object functions { * * @param unit * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param time - * A TIME to truncate. + * A TIME to truncate. A column that evaluates to a time. * @return - * A TIME truncated to the specified unit. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @throws IllegalArgumentException @@ -8749,24 +12690,36 @@ object functions { /** * Creates a TIME from the number of seconds since midnight. * + * @param e + * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) /** * Creates a TIME from the number of milliseconds since midnight. * + * @param e + * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) /** * Creates a TIME from the number of microseconds since midnight. * + * @param e + * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) @@ -8774,24 +12727,36 @@ object functions { * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a * DECIMAL(14,6) to preserve microsecond precision. * + * @param e + * TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a decimal. */ def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) /** * Extracts the number of milliseconds since midnight from a TIME value. * + * @param e + * the TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) /** * Extracts the number of microseconds since midnight from a TIME value. * + * @param e + * the TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) @@ -8799,8 +12764,14 @@ object functions { * Parses the `timestamp` expression with the `format` expression to a timestamp without time * zone. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ltz", timestamp, format) @@ -8809,8 +12780,12 @@ object functions { * Parses the `timestamp` expression with the default format to a timestamp without time zone. * The default format follows casting rules to a timestamp. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column): Column = Column.fn("to_timestamp_ltz", timestamp) @@ -8819,8 +12794,14 @@ object functions { * Parses the `timestamp_str` expression with the `format` expression to a timestamp without * time zone. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ntz", timestamp, format) @@ -8829,8 +12810,12 @@ object functions { * Parses the `timestamp` expression with the default format to a timestamp without time zone. * The default format follows casting rules to a timestamp. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column): Column = Column.fn("to_timestamp_ntz", timestamp) @@ -8838,8 +12823,14 @@ object functions { /** * Returns the UNIX timestamp of the given time. * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to convert the time values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column, format: Column): Column = Column.fn("to_unix_timestamp", timeExp, format) @@ -8847,8 +12838,12 @@ object functions { /** * Returns the UNIX timestamp of the given time. * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column): Column = Column.fn("to_unix_timestamp", timeExp) @@ -8856,8 +12851,13 @@ object functions { /** * Extracts the three-letter abbreviated month name from a given date/timestamp/string. * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def monthname(timeExp: Column): Column = Column.fn("monthname", timeExp) @@ -8865,8 +12865,13 @@ object functions { /** * Extracts the three-letter abbreviated day name from a given date/timestamp/string. * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def dayname(timeExp: Column): Column = Column.fn("dayname", timeExp) @@ -8877,8 +12882,15 @@ object functions { /** * Returns null if the array is null, true if the array contains `value`, and false otherwise. + * @param column + * the target column containing the arrays. A column that evaluates to an array. + * @param value + * the value to check for in the array. A column that evaluates to a value matching the + * array's element type. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def array_contains(column: Column, value: Any): Column = Column.fn("array_contains", column, lit(value)) @@ -8887,8 +12899,15 @@ object functions { * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. * The new element/column is located at end of the ARRAY. * + * @param column + * the source column containing the array. A column that evaluates to an array. + * @param element + * the value to append to the array. A column that evaluates to a value matching the array's + * element type. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_append(column: Column, element: Any): Column = Column.fn("array_append", column, lit(element)) @@ -8897,8 +12916,14 @@ object functions { * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns * `false` otherwise. + * @param a1 + * the first input array. A column that evaluates to an array. + * @param a2 + * the second input array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a boolean. */ def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) @@ -8907,14 +12932,16 @@ object functions { * end if `start` is negative) with the specified `length`. * * @param x - * the array column to be sliced + * the array column to be sliced. A column that evaluates to an array. * @param start - * the starting index + * the starting index. A column that evaluates to an integer. * @param length - * the length of the slice + * the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Int, length: Int): Column = slice(x, lit(start), lit(length)) @@ -8932,6 +12959,8 @@ object functions { * * @group array_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Column, length: Column): Column = Column.fn("slice", x, start, length) @@ -8939,16 +12968,30 @@ object functions { /** * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with * `nullReplacement`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @param nullReplacement + * the string used to replace null values. A column that evaluates to a string. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String, nullReplacement: String): Column = Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) /** * Concatenates the elements of `column` using the `delimiter`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String): Column = Column.fn("array_join", column, lit(delimiter)) @@ -8957,11 +13000,15 @@ object functions { * Concatenates multiple input columns together into a single column. The function works with * strings, binary and compatible array columns. * + * @param exprs + * Input columns to concatenate. A column that evaluates to a string, binary or an array. * @note * Returns null if any of the input columns are null. * * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) @@ -8970,12 +13017,18 @@ object functions { * Locates the position of the first occurrence of the value in the given array as long. Returns * null if either of the arguments are null. * + * @param column + * The array to search. A column that evaluates to an array. + * @param value + * The value to locate. A column. * @note * The position is not zero based, but 1 based index. Returns 0 if value could not be found in * array. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a long. */ def array_position(column: Column, value: Any): Column = Column.fn("array_position", column, lit(value)) @@ -8984,8 +13037,15 @@ object functions { * Returns element of array at given index in value if column is array. Returns value for the * given key in value if column is map. * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input + * map. */ def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) @@ -8997,8 +13057,15 @@ object functions { * (map, key) - Returns value for given key. The function always returns NULL if the key is not * contained in the map. * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input + * map. */ def try_element_at(column: Column, value: Column): Column = Column.fn("try_element_at", column, value) @@ -9007,8 +13074,14 @@ object functions { * Returns element of array at given (0-based) index. If the index points outside of the array * boundaries, then this function returns NULL. * + * @param column + * The array to extract from. A column that evaluates to an array. + * @param index + * The 0-based index. A column that evaluates to an integral. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column of the element type of the input array. */ def get(column: Column, index: Column): Column = Column.fn("get", column, index) @@ -9021,8 +13094,12 @@ object functions { * fields in the struct must be orderable. If the default comparator does not support the input * type, you can specify a custom comparator. * + * @param e + * The array to sort. A column that evaluates to an array. * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column): Column = Column.fn("array_sort", e) @@ -9032,8 +13109,15 @@ object functions { * positive integer as the first element is less than, equal to, or greater than the second * element. If the comparator function returns null, the function will fail and raise an error. * + * @param e + * The array to sort. A column that evaluates to an array. + * @param comparator + * A binary comparator function that returns a negative integer, 0, or a positive integer as + * the first element is less than, equal to, or greater than the second element. * @group collection_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column, comparator: (Column, Column) => Column): Column = Column.fn("array_sort", e, createLambda(comparator)) @@ -9041,8 +13125,14 @@ object functions { /** * Remove all elements that equal to element from the given array. * + * @param column + * The array to remove from. A column that evaluates to an array. + * @param element + * The element to remove. A column. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_remove(column: Column, element: Any): Column = Column.fn("array_remove", column, lit(element)) @@ -9050,8 +13140,12 @@ object functions { /** * Remove all null elements from the given array. * + * @param column + * The array to compact. A column that evaluates to an array. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_compact(column: Column): Column = Column.fn("array_compact", column) @@ -9059,16 +13153,26 @@ object functions { * Returns an array containing value as well as all elements from array. The new element is * positioned at the beginning of the array. * + * @param column + * The array to prepend to. A column that evaluates to an array. + * @param element + * The element to prepend. A column. * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_prepend(column: Column, element: Any): Column = Column.fn("array_prepend", column, lit(element)) /** * Removes duplicate values from the array. + * @param e + * The array to deduplicate. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_distinct(e: Column): Column = Column.fn("array_distinct", e) @@ -9076,8 +13180,14 @@ object functions { * Returns an array of the elements in the intersection of the given two arrays, without * duplicates. * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_intersect(col1: Column, col2: Column): Column = Column.fn("array_intersect", col1, col2) @@ -9085,8 +13195,17 @@ object functions { /** * Adds an item into a given array at a specified position * + * @param arr + * The array to insert into. A column that evaluates to an array. + * @param pos + * The 1-based position at which to insert (negative counts from the end). A column that + * evaluates to an integral. + * @param value + * The value to insert. A column. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_insert(arr: Column, pos: Column, value: Column): Column = Column.fn("array_insert", arr, pos, value) @@ -9094,8 +13213,14 @@ object functions { /** * Returns an array of the elements in the union of the given two arrays, without duplicates. * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_union(col1: Column, col2: Column): Column = Column.fn("array_union", col1, col2) @@ -9104,8 +13229,14 @@ object functions { * Returns an array of the elements in the first array but not in the second array, without * duplicates. The order of elements in the result is not determined * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_except(col1: Column, col2: Column): Column = Column.fn("array_except", col1, col2) @@ -9139,12 +13270,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => transformed_col, the lambda function to transform the input column + * col => transformed_col, the lambda function to transform the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: Column => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9164,6 +13297,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: (Column, Column) => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9175,12 +13310,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def exists(column: Column, f: Column => Column): Column = Column.fn("exists", column, createLambda(f)) @@ -9192,12 +13329,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def forall(column: Column, f: Column => Column): Column = Column.fn("forall", column, createLambda(f)) @@ -9209,12 +13348,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to filter the input column + * col => predicate, the Boolean predicate to filter the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: Column => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9233,6 +13374,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: (Column, Column) => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9246,18 +13389,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate( expr: Column, @@ -9282,6 +13427,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = aggregate(expr, initialValue, merge, c => c) @@ -9295,18 +13442,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce( expr: Column, @@ -9331,6 +13480,8 @@ object functions { * to the combined_value * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = reduce(expr, initialValue, merge, c => c) @@ -9344,14 +13495,16 @@ object functions { * }}} * * @param left - * the left input array column + * the left input array column. A column that evaluates to an array. * @param right - * the right input array column + * the right input array column. A column that evaluates to an array. * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = Column.fn("zip_with", left, right, createLambda(f)) @@ -9370,6 +13523,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_keys(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_keys", expr, createLambda(f)) @@ -9388,6 +13543,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_values(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_values", expr, createLambda(f)) @@ -9405,6 +13562,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_filter(expr: Column, f: (Column, Column) => Column): Column = Column.fn("map_filter", expr, createLambda(f)) @@ -9424,6 +13583,8 @@ object functions { * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = Column.fn("map_zip_with", left, right, createLambda(f)) @@ -9433,8 +13594,13 @@ object functions { * name `col` for elements in the array and `key` and `value` for elements in the map unless * specified otherwise. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 1.3.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ def explode(e: Column): Column = Column.fn("explode", e) @@ -9443,8 +13609,13 @@ object functions { * name `col` for elements in the array and `key` and `value` for elements in the map unless * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.2.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ def explode_outer(e: Column): Column = Column.fn("explode_outer", e) @@ -9453,6 +13624,8 @@ object functions { * default column name `pos` for position, and `col` for elements in the array and `key` and * `value` for elements in the map unless specified otherwise. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.1.0 */ @@ -9464,6 +13637,8 @@ object functions { * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the * array/map is null or empty then the row (null, null) is produced. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.2.0 */ @@ -9472,8 +13647,12 @@ object functions { /** * Creates a new row for each element in the given array of structs. * + * @param e + * the target column to explode. A column that evaluates to an array of structs. * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline(e: Column): Column = Column.fn("inline", e) @@ -9481,8 +13660,12 @@ object functions { * Creates a new row for each element in the given array of structs. Unlike inline, if the array * is null or empty then null is produced for each nested column. * + * @param e + * the target column to explode. A column that evaluates to an array of structs. * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline_outer(e: Column): Column = Column.fn("inline_outer", e) @@ -9490,8 +13673,14 @@ object functions { * Extracts json object from a json string based on json path specified, and returns json string * of the extracted json object. It will return null if the input json string is invalid. * + * @param e + * the JSON string column. A column that evaluates to a string. + * @param path + * the JSON path to extract. A column that evaluates to a string. Must be a constant. * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def get_json_object(e: Column, path: String): Column = Column.fn("get_json_object", e, lit(path)) @@ -9499,8 +13688,14 @@ object functions { /** * Creates a new row for a json column according to the given field names. * + * @param json + * the JSON string column. A column that evaluates to a string. + * @param fields + * the field names to extract. A column that evaluates to a string. Must be a constant. * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def json_tuple(json: Column, fields: String*): Column = { @@ -9525,6 +13720,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -9548,6 +13745,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { @@ -9571,6 +13770,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -9594,6 +13795,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { @@ -9611,6 +13814,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: StructType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9627,6 +13832,8 @@ object functions { * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: DataType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9649,6 +13856,8 @@ object functions { * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -9673,6 +13882,8 @@ object functions { * * @group json_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: Map[String, String]): Column = { @@ -9691,6 +13902,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: Column): Column = { from_json(e, schema, Map.empty[String, String].asJava) @@ -9714,6 +13927,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { @@ -9736,6 +13951,8 @@ object functions { * * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) @@ -9743,9 +13960,11 @@ object functions { * Parses a JSON string and constructs a Variant value. * * @param json - * a string column that contains JSON data. + * a string column that contains JSON data. A column that evaluates to a string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def parse_json(json: Column): Column = Column.fn("parse_json", json) @@ -9758,6 +13977,8 @@ object functions { * a column with a nested schema or column name. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) @@ -9769,6 +13990,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) @@ -9780,6 +14003,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) @@ -9797,6 +14022,8 @@ object functions { * additional JSONPath arguments, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: Column, paths: Column*): Column = @@ -9816,6 +14043,8 @@ object functions { * additional JSONPath strings, applied after `path` in order. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: String, paths: String*): Column = @@ -9836,6 +14065,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: Column, value: Column): Column = Column.fn("variant_insert", v, path, value) @@ -9855,6 +14086,8 @@ object functions { * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: String, value: Column): Column = Column.fn("variant_insert", v, lit(path), value) @@ -9872,6 +14105,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: String, targetType: String): Column = Column.fn("variant_get", v, lit(path), lit(targetType)) @@ -9890,6 +14125,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("variant_get", v, path, lit(targetType)) @@ -9907,6 +14144,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: String, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -9925,6 +14164,8 @@ object functions { * the target data type to cast into, in a DDL-formatted string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -9936,6 +14177,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) @@ -9946,6 +14189,8 @@ object functions { * a variant column. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) @@ -9957,6 +14202,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: String): Column = schema_of_json(lit(json)) @@ -9968,6 +14215,8 @@ object functions { * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) @@ -9983,7 +14232,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-json.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * * @group json_funcs * @since 3.0.0 @@ -9996,8 +14246,12 @@ object functions { * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any * other valid JSON string, `NULL` or an invalid JSON. * + * @param e + * the JSON array string column. A column that evaluates to a string. * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def json_array_length(e: Column): Column = Column.fn("json_array_length", e) @@ -10006,8 +14260,12 @@ object functions { * given, all the keys of the outermost object will be returned as an array. If it is any other * valid JSON string, an invalid JSON string or an empty string, the function returns null. * + * @param e + * the JSON object string column. A column that evaluates to a string. * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) @@ -10018,7 +14276,7 @@ object functions { * type. * * @param e - * a column containing a struct, an array or a map. + * a column containing a struct, an array, a map, or a variant. * @param options * options to control how the struct column is converted into a json string. accepts the same * options and the json data source. See Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. - * + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * @group csv_funcs * @since 3.0.0 */ @@ -10474,6 +14873,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_csv(e: Column, options: java.util.Map[String, String]): Column = @@ -10488,6 +14889,8 @@ object functions { * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) @@ -10507,6 +14910,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -10528,6 +14933,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -10545,6 +14952,8 @@ object functions { * the schema to use when parsing the XML string * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column): Column = { @@ -10567,6 +14976,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -10583,6 +14994,8 @@ object functions { * * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ def from_xml(e: Column, schema: StructType): Column = from_xml(e, schema, Map.empty[String, String].asJava) @@ -10598,6 +15011,8 @@ object functions { * a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) @@ -10608,6 +15023,8 @@ object functions { * a foldable string column containing a XML string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) @@ -10624,7 +15041,8 @@ object functions { * "https://spark.apache.org/docs/latest/sql-data-sources-xml.html#data-source-option"> Data * Source Option in the version you use. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * @group xml_funcs * @since 4.0.0 */ @@ -10647,6 +15065,8 @@ object functions { * Source Option in the version you use. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_xml(e: Column, options: java.util.Map[String, String]): Column = @@ -10660,12 +15080,16 @@ object functions { * a column containing a struct. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) /** * (Java-specific) A transform for timestamps and dates to partition data into years. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10674,6 +15098,8 @@ object functions { /** * (Java-specific) A transform for timestamps and dates to partition data into months. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10682,6 +15108,8 @@ object functions { /** * (Java-specific) A transform for timestamps and dates to partition data into days. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10690,8 +15118,14 @@ object functions { /** * Returns a string array of values within the nodes of xml that match the XPath expression. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def xpath(xml: Column, path: Column): Column = Column.fn("xpath", xml, path) @@ -10699,8 +15133,14 @@ object functions { /** * Returns true if the XPath expression evaluates to true, or if a matching node is found. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def xpath_boolean(xml: Column, path: Column): Column = Column.fn("xpath_boolean", xml, path) @@ -10709,8 +15149,14 @@ object functions { * Returns a double value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_double(xml: Column, path: Column): Column = Column.fn("xpath_double", xml, path) @@ -10719,8 +15165,14 @@ object functions { * Returns a double value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_number(xml: Column, path: Column): Column = Column.fn("xpath_number", xml, path) @@ -10729,8 +15181,14 @@ object functions { * Returns a float value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a float. */ def xpath_float(xml: Column, path: Column): Column = Column.fn("xpath_float", xml, path) @@ -10739,8 +15197,14 @@ object functions { * Returns an integer value, or the value zero if no match is found, or a match is found but the * value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def xpath_int(xml: Column, path: Column): Column = Column.fn("xpath_int", xml, path) @@ -10749,8 +15213,14 @@ object functions { * Returns a long integer value, or the value zero if no match is found, or a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def xpath_long(xml: Column, path: Column): Column = Column.fn("xpath_long", xml, path) @@ -10759,8 +15229,14 @@ object functions { * Returns a short integer value, or the value zero if no match is found, or a match is found * but the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a short. */ def xpath_short(xml: Column, path: Column): Column = Column.fn("xpath_short", xml, path) @@ -10768,8 +15244,14 @@ object functions { /** * Returns the text contents of the first xml node that matches the XPath expression. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def xpath_string(xml: Column, path: Column): Column = Column.fn("xpath_string", xml, path) @@ -10777,6 +15259,8 @@ object functions { /** * (Java-specific) A transform for timestamps to partition data into hours. * + * @param e + * target date or timestamp column to work on. A column that evaluates to a date or timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10788,13 +15272,16 @@ object functions { * * @param sourceTz * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. + * used as the source time zone. A column that evaluates to a string. * @param targetTz - * the time zone to which the input timestamp should be converted. + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. * @param sourceTs - * a timestamp without time zone. + * a timestamp without time zone. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) @@ -10808,6 +15295,8 @@ object functions { * a timestamp without time zone. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", targetTz, sourceTs) @@ -10815,8 +15304,19 @@ object functions { /** * Make DayTimeIntervalType duration from days, hours, mins and secs. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = Column.fn("make_dt_interval", days, hours, mins, secs) @@ -10824,8 +15324,16 @@ object functions { /** * Make DayTimeIntervalType duration from days, hours and mins. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column): Column = Column.fn("make_dt_interval", days, hours, mins) @@ -10833,8 +15341,14 @@ object functions { /** * Make DayTimeIntervalType duration from days and hours. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column): Column = Column.fn("make_dt_interval", days, hours) @@ -10842,8 +15356,12 @@ object functions { /** * Make DayTimeIntervalType duration from days. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column): Column = Column.fn("make_dt_interval", days) @@ -10853,6 +15371,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(): Column = Column.fn("make_dt_interval") @@ -10861,8 +15381,25 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10877,8 +15414,25 @@ object functions { /** * Make interval from years, months, weeks, days, hours, mins and secs. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10894,8 +15448,22 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10909,8 +15477,22 @@ object functions { /** * Make interval from years, months, weeks, days, hours and mins. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10925,8 +15507,20 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -10939,8 +15533,20 @@ object functions { /** * Make interval from years, months, weeks, days and hours. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -10954,8 +15560,18 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("try_make_interval", years, months, weeks, days) @@ -10963,8 +15579,18 @@ object functions { /** * Make interval from years, months, weeks and days. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("make_interval", years, months, weeks, days) @@ -10973,8 +15599,16 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("try_make_interval", years, months, weeks) @@ -10982,8 +15616,16 @@ object functions { /** * Make interval from years, months and weeks. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * The number of weeks, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("make_interval", years, months, weeks) @@ -10992,8 +15634,14 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column): Column = Column.fn("try_make_interval", years, months) @@ -11001,8 +15649,14 @@ object functions { /** * Make interval from years and months. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column): Column = Column.fn("make_interval", years, months) @@ -11011,8 +15665,12 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column): Column = Column.fn("try_make_interval", years) @@ -11020,8 +15678,12 @@ object functions { /** * Make interval from years. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column): Column = Column.fn("make_interval", years) @@ -11031,6 +15693,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(): Column = Column.fn("make_interval") @@ -11041,8 +15705,25 @@ object functions { * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. * Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11060,8 +15741,23 @@ object functions { * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it * will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11075,8 +15771,16 @@ object functions { /** * Create a local date-time from date, time, and timezone fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("make_timestamp", date, time, timezone) @@ -11084,8 +15788,14 @@ object functions { /** * Create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column): Column = Column.fn("make_timestamp", date, time) @@ -11095,8 +15805,25 @@ object functions { * The result data type is consistent with the value of configuration `spark.sql.timestampType`. * The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11113,8 +15840,23 @@ object functions { * data type is consistent with the value of configuration `spark.sql.timestampType`. The * function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11128,8 +15870,16 @@ object functions { /** * Try to create a local date-time from date, time, and timezone fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("try_make_timestamp", date, time, timezone) @@ -11137,8 +15887,14 @@ object functions { /** * Try to create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column): Column = Column.fn("try_make_timestamp", date, time) @@ -11148,8 +15904,25 @@ object functions { * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function * returns NULL on invalid inputs. Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11166,8 +15939,24 @@ object functions { * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns * NULL on invalid inputs. Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11182,8 +15971,26 @@ object functions { * Try to create the current timestamp with local time zone from years, months, days, hours, * mins, secs and timezone fields. The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11199,8 +16006,24 @@ object functions { * Try to create the current timestamp with local time zone from years, months, days, hours, * mins and secs fields. The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11216,8 +16039,24 @@ object functions { * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. * Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz( years: Column, @@ -11231,8 +16070,14 @@ object functions { /** * Create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("make_timestamp_ntz", date, time) @@ -11241,8 +16086,24 @@ object functions { * Try to create a local date-time from years, months, days, hours, mins, secs fields. The * function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz( years: Column, @@ -11256,8 +16117,14 @@ object functions { /** * Try to create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("try_make_timestamp_ntz", date, time) @@ -11265,8 +16132,14 @@ object functions { /** * Make year-month interval from years, months. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column, months: Column): Column = Column.fn("make_ym_interval", years, months) @@ -11274,8 +16147,12 @@ object functions { /** * Make year-month interval from years. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) @@ -11284,12 +16161,18 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(): Column = Column.fn("make_ym_interval") /** * (Java-specific) A transform for any type that partitions by a hash of the input column. * + * @param numBuckets + * The number of buckets. A column that evaluates to an integral. Must be a constant. + * @param e + * The input column to partition. A column of any type. * @group partition_transforms * @since 3.0.0 */ @@ -11298,6 +16181,10 @@ object functions { /** * (Java-specific) A transform for any type that partitions by a hash of the input column. * + * @param numBuckets + * The number of buckets. Must be a constant. + * @param e + * The input column to partition. A column of any type. * @group partition_transforms * @since 3.0.0 */ @@ -11310,16 +16197,26 @@ object functions { /** * Returns `col2` if `col1` is null, or `col1` otherwise. * + * @param col1 + * The column to test for null. A column of any type. + * @param col2 + * The column to return when col1 is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) /** * Returns true if `col` is not null, or false otherwise. * + * @param col + * The column to check. A column of any type. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnotnull(col: Column): Column = Column.fn("isnotnull", col) @@ -11327,48 +16224,82 @@ object functions { * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both * are null, false if one of the them is null. * + * @param col1 + * The first column to compare. A column of any type. + * @param col2 + * The second column to compare. A column of any type. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) /** * Returns null if `col1` equals to `col2`, or `col1` otherwise. * + * @param col1 + * The value to return if it is not equal to `col2`. A column of any type. + * @param col2 + * The value compared with `col1`. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) /** * Returns null if `col` is equal to zero, or `col` otherwise. * + * @param col + * The input value. A column that evaluates to a numeric. * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def nullifzero(col: Column): Column = Column.fn("nullifzero", col) /** * Returns `col2` if `col1` is null, or `col1` otherwise. * + * @param col1 + * The value to return if it is not null. A column of any type. + * @param col2 + * The value to return if `col1` is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) /** * Returns `col2` if `col1` is not null, or `col3` otherwise. * + * @param col1 + * The value that determines which branch to return. A column of any type. + * @param col2 + * The value to return if `col1` is not null. A column of any type. + * @param col3 + * The value to return if `col1` is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) /** * Returns zero if `col` is null, or `col` otherwise. * + * @param col + * The input value. A column that evaluates to a numeric. * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) @@ -11423,8 +16354,13 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column): Column = Column.fn("st_asbinary", geo) @@ -11432,8 +16368,16 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format using the specified endianness. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param endianness + * The endianness of the output WKB, 'NDR' for little-endian or 'XDR' for big-endian. A column + * that evaluates to a string. * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: Column): Column = Column.fn("st_asbinary", geo, endianness) @@ -11441,8 +16385,16 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format using the specified endianness. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param endianness + * The endianness of the output WKB, 'NDR' for little-endian or 'XDR' for big-endian. A column + * that evaluates to a string. * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: String): Column = Column.fn("st_asbinary", geo, lit(endianness)) @@ -11450,6 +16402,8 @@ object functions { /** * Parses the WKB description of a geography and returns the corresponding GEOGRAPHY value. * + * @param wkb + * A value in WKB format, representing a GEOGRAPHY value. A column that evaluates to a binary. * @group st_funcs * @since 4.1.0 */ @@ -11459,6 +16413,8 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. * @group st_funcs * @since 4.1.0 */ @@ -11468,6 +16424,10 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. + * @param srid + * The SRID value of the geometry. A column that evaluates to an integer. * @group st_funcs * @since 4.2.0 */ @@ -11477,6 +16437,10 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. + * @param srid + * The SRID value of the geometry. A column that evaluates to an integer. * @group st_funcs * @since 4.2.0 */ @@ -11486,6 +16450,11 @@ object functions { /** * Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param srid + * The new SRID of the geospatial value. A column that evaluates to an integer. * @group st_funcs * @since 4.1.0 */ @@ -11495,6 +16464,11 @@ object functions { /** * Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param srid + * The new SRID of the geospatial value. A column that evaluates to an integer. * @group st_funcs * @since 4.1.0 */ @@ -11504,8 +16478,13 @@ object functions { /** * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to an integer. */ def st_srid(geo: Column): Column = Column.fn("st_srid", geo) @@ -12082,6 +17061,8 @@ object functions { /** * Unwrap UDT data type column into its underlying type. + * @param column + * the UDT column to unwrap. A column that evaluates to a user-defined type. * @group udf_funcs * @since 3.4.0 */ @@ -12091,71 +17072,117 @@ object functions { /** * Returns the cosine similarity between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_cosine_similarity(left: Column, right: Column): Column = Column.fn("vector_cosine_similarity", left, right) /** * Returns the inner product (dot product) between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_inner_product(left: Column, right: Column): Column = Column.fn("vector_inner_product", left, right) /** * Returns the Euclidean (L2) distance between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_l2_distance(left: Column, right: Column): Column = Column.fn("vector_l2_distance", left, right) /** * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column, degree: Column): Column = Column.fn("vector_norm", vector, degree) /** * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column): Column = Column.fn("vector_norm", vector) /** * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column, degree: Column): Column = Column.fn("vector_normalize", vector, degree) /** * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column): Column = Column.fn("vector_normalize", vector) /** * Aggregate function: returns the element-wise mean of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_avg(col: Column): Column = Column.fn("vector_avg", col) /** * Aggregate function: returns the element-wise sum of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_sum(col: Column): Column = Column.fn("vector_sum", col)