From 99fbac9f5f81c697006774887c311450d1304b65 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 10 Jul 2026 16:46:28 +0900 Subject: [PATCH] [SPARK-58085][SQL][DOCS] Document accepted input types for built-in functions in the SQL function reference ### What changes were proposed in this pull request? This documents, for every built-in function argument, which input types it accepts, in the SQL `@ExpressionDescription` `arguments` blocks (rendered on the SQL functions reference page), using a numeric umbrella (`a numeric` / `a numeric or interval`) where numeric subtypes cast in, and `timestamp` for TIMESTAMP_NTZ in type phrases. Subtask of SPARK-57999, split out from apache/spark#57079 for reviewability. This PR covers only the SQL surface. ### Why are the changes needed? The SQL function reference previously had no per-argument type information for most built-in functions. This fills that gap. ### Does this PR introduce _any_ user-facing change? Yes, documentation only. No behavioral or API change. ### How was this patch tested? Existing `ExpressionInfoSuite` and the SPARK-32870 argument-block-format tests pass. Types were derived from and verified against the analyzer's accept/reject behavior and each expression's `inputTypes` / `checkInputDataTypes`. Co-authored-by: Isaac --- .../sql/catalyst/expressions/Between.scala | 4 + .../sql/catalyst/expressions/TryEval.scala | 42 +++ .../expressions/aggregate/AnyValue.scala | 7 + .../aggregate/ApproxTopKAggregates.scala | 16 ++ .../expressions/aggregate/Average.scala | 10 + .../aggregate/CentralMomentAgg.scala | 30 ++ .../catalyst/expressions/aggregate/Corr.scala | 7 + .../expressions/aggregate/CountIf.scala | 5 + .../expressions/aggregate/Covariance.scala | 14 + .../expressions/aggregate/First.scala | 7 + .../catalyst/expressions/aggregate/Last.scala | 7 + .../expressions/aggregate/Measure.scala | 5 + .../catalyst/expressions/aggregate/Mode.scala | 8 + .../catalyst/expressions/aggregate/Sum.scala | 10 + .../aggregate/bitwiseAggregates.scala | 15 + .../aggregate/boolAggregates.scala | 10 + .../expressions/aggregate/collect.scala | 2 + .../aggregate/datasketchesAggregates.scala | 9 +- .../expressions/aggregate/kllAggregates.scala | 12 + .../aggregate/linearRegression.scala | 63 +++++ .../expressions/aggregate/percentiles.scala | 5 + .../aggregate/thetasketchesAggregates.scala | 8 + .../aggregate/tupleIntersectionAgg.scala | 16 ++ .../aggregate/tupleSketchAgg.scala | 20 ++ .../expressions/aggregate/tupleUnionAgg.scala | 15 + .../sql/catalyst/expressions/arithmetic.scala | 57 ++++ .../expressions/bitmapExpressions.scala | 15 + .../expressions/bitwiseExpressions.scala | 38 +++ .../expressions/collationExpressions.scala | 1 + .../expressions/collectionOperations.scala | 171 +++++++++++ .../expressions/complexTypeCreator.scala | 16 ++ .../expressions/datetimeExpressions.scala | 221 +++++++++++++++ .../spark/sql/catalyst/expressions/hash.scala | 22 ++ .../expressions/higherOrderFunctions.scala | 55 ++++ .../expressions/intervalExpressions.scala | 8 + .../expressions/jsonExpressions.scala | 15 + .../catalyst/expressions/kllExpressions.scala | 93 ++++++ .../expressions/maskExpressions.scala | 5 + .../expressions/mathExpressions.scala | 203 +++++++++++++ .../spark/sql/catalyst/expressions/misc.scala | 36 +++ .../expressions/nullExpressions.scala | 24 ++ .../expressions/numberFormatExpressions.scala | 21 ++ .../sql/catalyst/expressions/predicates.scala | 26 ++ .../expressions/randomExpressions.scala | 26 ++ .../expressions/regexpExpressions.scala | 28 ++ .../expressions/st/stExpressions.scala | 3 + .../expressions/stringExpressions.scala | 267 ++++++++++++++++++ .../expressions/timeExpressions.scala | 30 ++ .../catalyst/expressions/urlExpressions.scala | 23 +- .../variant/variantExpressions.scala | 10 + .../sql/catalyst/expressions/xml/xpath.scala | 56 ++++ .../sql/execution/command/DDLSuite.scala | 6 + 52 files changed, 1821 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala index c226e48c6be5e..adaa340ffd402 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Between.scala @@ -30,7 +30,11 @@ import org.apache.spark.sql.internal.SQLConf arguments = """ Arguments: * input - An expression that is being compared with lower and upper bound. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * lower - Lower bound of the between check. + An expression that evaluates to a boolean, numeric, string, binary, date, timestamp, time, + interval, array, or struct. * upper - Upper bound of the between check. """, since = "1.0.0", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala index b167ff71449d0..289b102c1f6f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TryEval.scala @@ -58,6 +58,13 @@ case class TryEval(child: Expression) extends UnaryExpression { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sum of `expr1`and `expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `+` operator.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(1, 2); @@ -102,6 +109,13 @@ case class TryAdd(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns `dividend`/`divisor`. It always performs floating point division. Its result is always null if `expr2` is 0. " + "`dividend` must be a numeric or an interval. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a numeric or interval. + * divisor - The expression to divide by. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -141,6 +155,13 @@ case class TryDivide(left: Expression, right: Expression, replacement: Expressio @ExpressionDescription( usage = "_FUNC_(dividend, divisor) - Returns the remainder after `expr1`/`expr2`. " + "`dividend` must be a numeric. `divisor` must be a numeric.", + arguments = """ + Arguments: + * dividend - The expression to be divided. + An expression that evaluates to a numeric. + * divisor - The expression to divide by. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(3, 2); @@ -177,6 +198,13 @@ case class TryMod(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`-`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `-` operator.", + arguments = """ + Arguments: + * expr1 - The minuend to subtract from. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The subtrahend to subtract. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT _FUNC_(2, 1); @@ -219,6 +247,13 @@ case class TrySubtract(left: Expression, right: Expression, replacement: Express @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1`*`expr2` and the result is null on overflow. " + "The acceptable input types are the same with the `*` operator.", + arguments = """ + Arguments: + * expr1 - The first factor. + An expression that evaluates to a numeric or interval. + * expr2 - The second factor. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -251,6 +286,13 @@ case class TryMultiply(left: Expression, right: Expression, replacement: Express // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str[, fmt]) - This is a special version of `to_binary` that performs the same operation, but returns a NULL value instead of raising an error if the conversion cannot be performed.", + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format of the input string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala index 9fbca1629c95f..86192a955b889 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/AnyValue.scala @@ -34,6 +34,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns some value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return some value of. + An expression of any type. + * isIgnoreNull - Whether to return only non-null values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala index 7ae542f190d56..0fcd8aec16186 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala @@ -59,6 +59,15 @@ import org.apache.spark.unsafe.types.UTF8String `k` An optional INTEGER literal greater than 0. If k is not specified, it defaults to 5. `maxItemsTracked` An optional INTEGER literal greater than or equal to k and has upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression to compute the top k most frequent items of. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. + * k - The number of top items to return. + An expression that evaluates to an integer. Must be a constant. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(expr) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); @@ -488,6 +497,13 @@ object ApproxTopKAggregateBuffer { _FUNC_(expr, maxItemsTracked) - Accumulates items into a sketch. `maxItemsTracked` An optional positive INTEGER literal with upper limit of 1000000. If maxItemsTracked is not specified, it defaults to 10000. """, + arguments = """ + Arguments: + * expr - The expression whose values are accumulated into the sketch. + An expression that evaluates to a boolean, numeric, date, timestamp, or string. + * maxItemsTracked - The maximum number of items to track in the sketch. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT approx_top_k_estimate(_FUNC_(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index fe30e2ea6f3ff..85325068297a8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -145,6 +150,11 @@ case class Average( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to average over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 99fb512ea047d..eb4f2c1191523 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -147,6 +147,11 @@ abstract class CentralMomentAgg(child: Expression, nullOnDivideByZero: Boolean) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population standard deviation of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -178,6 +183,11 @@ case class StddevPop( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample standard deviation calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample standard deviation of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -210,6 +220,11 @@ case class StddevSamp( // Compute the population variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the population variance of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -239,6 +254,11 @@ case class VariancePop( // Compute the sample variance of a column @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the sample variance of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col); @@ -281,6 +301,11 @@ case class RegrReplacement(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the skewness of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); @@ -312,6 +337,11 @@ case class Skewness( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to compute the kurtosis of. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index ea154b9bce88c..72f682c00b4ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -105,6 +105,13 @@ abstract class PearsonCorrelation(x: Expression, y: Expression, nullOnDivideByZe // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala index 248ade05ab1d3..f389405ec495f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala @@ -25,6 +25,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType} usage = """ _FUNC_(expr) - Returns the number of `TRUE` values for the expression. """, + arguments = """ + Arguments: + * expr - The boolean expression whose TRUE values are counted. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 8e03daee9b659..7d047aca06068 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -90,6 +90,13 @@ abstract class Covariance(val left: Expression, val right: Expression, nullOnDiv @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); @@ -119,6 +126,13 @@ case class CovPopulation( @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", + arguments = """ + Arguments: + * expr1 - The first variable of the number pairs. + An expression that evaluates to a numeric. + * expr2 - The second variable of the number pairs. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 9a39a6fe98796..eb39f96ed2ee9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -36,6 +36,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values.""", + arguments = """ + Arguments: + * expr - The expression to return the first value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the first non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 5840c783cb8be..a0f11ffcb7142 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -35,6 +35,13 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows. If `isIgnoreNull` is true, returns only non-null values""", + arguments = """ + Arguments: + * expr - The expression to return the last value of. + An expression of any type. + * isIgnoreNull - Whether to ignore null values and return only the last non-null value. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (10), (5), (20) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala index e2d85cafa52f2..21959a66b2b34 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Measure.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType} // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - this function is used and can only be used to calculate a measure defined in a metric view.", + arguments = """ + Arguments: + * expr - The measure defined in a metric view to calculate. + An expression of any type. + """, examples = """ Examples: > SELECT dimension_col, _FUNC_(measure_col) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala index 30e00ac68004b..b87c0ede1ab81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Mode.scala @@ -233,6 +233,14 @@ case class Mode( _FUNC_() WITHIN GROUP (ORDER BY col) - Returns the most frequent value for the values within `col` (specified in ORDER BY clause). NULL values are ignored. If all the values are NULL, or there are 0 rows, returns NULL. When multiple values have the same greatest frequency only one value will be returned. The value will be chosen based on sort direction. Return the smallest value if sort direction is asc or the largest value if sort direction is desc from multiple values with the same frequency.""", + arguments = """ + Arguments: + * col - The column to compute the most frequent value of. + An expression of any type. + * deterministic - Whether to return a deterministic result when there are multiple most + frequent values. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index d066a87fc7919..949a107928df7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); @@ -201,6 +206,11 @@ case class Sum( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group and the result is null on overflow.", + arguments = """ + Arguments: + * expr - The expression to sum over the group. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (5), (10), (15) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala index 86a16ad389b5d..b1d9a46a237ec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala @@ -55,6 +55,11 @@ abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise AND of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise AND of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -76,6 +81,11 @@ case class BitAndAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise OR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise OR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); @@ -97,6 +107,11 @@ case class BitOrAgg(child: Expression) extends BitAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bitwise XOR of all non-null input values, or null if none.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise XOR of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala index ae759abf8a4f5..3eba7352e3477 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/boolAggregates.scala @@ -23,6 +23,11 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if all values of `expr` are true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (true), (true) AS tab(col); @@ -45,6 +50,11 @@ case class BoolAnd(child: Expression) extends AggregateFunction with RuntimeRepl @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if at least one value of `expr` is true.", + arguments = """ + Arguments: + * expr - The boolean expression to evaluate over the group. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (true), (false), (false) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index 8a370e9a9dc78..17e3bf3ffc3d4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -387,8 +387,10 @@ private[aggregate] object CollectTopK { arguments = """ Arguments: * expr - a string or binary expression to be concatenated. + An expression that evaluates to a string or binary. * delimiter - an optional string or binary foldable expression used to separate the input values. If NULL, the concatenation will be performed without a delimiter. Default is NULL. + An expression that evaluates to a string, binary, or null. Must be a constant. * key - an optional expression for ordering the input values. Multiple keys can be specified. If none are specified, the order of the rows in the result is non-deterministic. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala index 8fb1bf51319cc..952b331b7227d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/datasketchesAggregates.scala @@ -48,6 +48,13 @@ import org.apache.spark.unsafe.types.UTF8String _FUNC_(expr, lgConfigK) - Returns the HllSketch's updatable binary representation. `lgConfigK` (optional) the log-base-2 of K, with K is the number of buckets or slots for the HllSketch. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the HLL sketch. + An expression that evaluates to an integer, long, string, or binary. + * lgConfigK - The log-base-2 of K, where K is the number of buckets for the sketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT hll_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); @@ -233,7 +240,7 @@ object HllSketchAgg { // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(expr, allowDifferentLgConfigK) - Returns the estimated number of unique values. + _FUNC_(expr, allowDifferentLgConfigK) - Returns the merged HllSketch's updatable binary representation. `allowDifferentLgConfigK` (optional) Allow sketches with different lgConfigK values to be unioned (defaults to false).""", examples = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala index 80ac45d273afa..d9a517be86d06 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/kllAggregates.scala @@ -54,6 +54,12 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, ByteType, DataT The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to an integral. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -337,6 +343,12 @@ case class KllSketchAggFloat( The optional k parameter controls the size and accuracy of the sketch (default 200, range 8-65535). Larger k values provide more accurate quantile estimates but result in larger, slower sketches. """, + arguments = """ + Arguments: + * expr - The expression to aggregate into the KLL sketch. + An expression that evaluates to a float or double. + * k - The parameter controlling the size and accuracy of the sketch. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala index 08cc7827b4779..b397b7fdb0d2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/linearRegression.scala @@ -25,6 +25,13 @@ import org.apache.spark.sql.types.{AbstractDataType, DataType, DoubleType, Numer // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the number of non-null number pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -57,6 +64,13 @@ case class RegrCount(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the independent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable to average. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -92,6 +106,13 @@ case class RegrAvgX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the average of the dependent variable for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable to average. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -127,6 +148,13 @@ case class RegrAvgY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the coefficient of determination for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -160,6 +188,13 @@ case class RegrR2(y: Expression, x: Expression) extends PearsonCorrelation(y, x, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -195,6 +230,13 @@ case class RegrSXX( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -224,6 +266,13 @@ case class RegrSXY(y: Expression, x: Expression) extends Covariance(y, x, true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); @@ -259,6 +308,13 @@ case class RegrSYY( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the slope of the linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); @@ -322,6 +378,13 @@ case class RegrSlope(left: Expression, right: Expression) extends DeclarativeAgg // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(y, x) - Returns the intercept of the univariate linear regression line for non-null pairs in a group, where `y` is the dependent variable and `x` is the independent variable.", + arguments = """ + Arguments: + * y - The dependent variable. + An expression that evaluates to a numeric. + * x - The independent variable. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(y, x) FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala index ac351db9e4710..d382182221b72 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala @@ -331,6 +331,11 @@ case class Percentile( @ExpressionDescription( usage = "_FUNC_(col) - Returns the median of numeric, ANSI interval or TIME column `col`.", + arguments = """ + Arguments: + * col - The column to compute the median of. + An expression that evaluates to a numeric, interval, or time. + """, examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (0), (10) AS tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala index 0f148d03cd70b..d8cd93129ec81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/thetasketchesAggregates.scala @@ -74,6 +74,14 @@ case class FinalizedSketch(sketch: CompactSketch) extends ThetaSketchState { _FUNC_(expr, lgNomEntries) - Returns the ThetaSketch compact binary representation. `lgNomEntries` (optional) is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the ThetaSketch. """, + arguments = """ + Arguments: + * expr - The expression whose distinct values are aggregated into the sketch. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + ThetaSketch. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT theta_sketch_estimate(_FUNC_(col, 12)) FROM VALUES (1), (1), (2), (2), (3) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala index 8cd864c914041..4439358ac8e11 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleIntersectionAgg.scala @@ -54,6 +54,14 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType} _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with a double type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary)); @@ -143,6 +151,14 @@ case class TupleIntersectionAggDouble( _FUNC_(child, mode) - Returns the intersected TupleSketch compact binary representation. `child` should be a binary TupleSketch representation created with an integer type summary. `mode` is the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + intersect. + An expression that evaluates to a binary. + * mode - The aggregation mode for numeric summaries during intersection (sum, min, max, + alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala index 7bf9240456857..bc6cb9de57759 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleSketchAgg.scala @@ -478,6 +478,16 @@ abstract class TupleSketchAggBase[U, S <: UpdatableSummary[U]] `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The double value to be aggregated as the summary. + An expression that evaluates to a double. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary); @@ -510,6 +520,16 @@ object TupleSketchAggDoubleExpressionBuilder extends ExpressionBuilder { `lgNomEntries` is the log-base-2 of nominal entries, with nominal entries deciding the number buckets or slots for the TupleSketch. Default is 12. `mode` is the aggregation mode for numeric summaries (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * key - The expression used for counting unique values. + An expression that evaluates to an array, binary, double, float, integer, long, or string. + * summary - The integer value to be aggregated as the summary. + An expression that evaluates to an integer. + * lgNomEntries - The log-base-2 of nominal entries, which sets the number of buckets for the + TupleSketch. + * mode - The aggregation mode for numeric summaries (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala index 0baff38674762..ff565baaa38d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/tupleUnionAgg.scala @@ -357,6 +357,13 @@ abstract class TupleUnionAggBase[S <: Summary] `child` should be a binary TupleSketch representation created with a double type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with a double type summary, to union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_double(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary)); @@ -386,6 +393,14 @@ object TupleUnionAggDoubleExpressionBuilder extends ExpressionBuilder { `child` should be a binary TupleSketch representation created with an integer type summary. `lgNomEntries` is the log-base-2 of nominal entries for the union operation. Default is 12. `mode` is the aggregation mode for numeric summaries during union (sum, min, max, alwaysone). Default is sum. """, + arguments = """ + Arguments: + * child - The binary TupleSketch representation, created with an integer type summary, to + union. + An expression that evaluates to a binary. + * lgNomEntries - The log-base-2 of nominal entries for the union operation. + * mode - The aggregation mode for numeric summaries during union (sum, min, max, alwaysone). + """, examples = """ Examples: > SELECT tuple_sketch_estimate_integer(_FUNC_(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 9c72f822143f6..03bb84c2623ea 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -36,6 +36,11 @@ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( usage = "_FUNC_(expr) - Returns the negated value of `expr`.", + arguments = """ + Arguments: + * expr - The expression to negate. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -109,6 +114,11 @@ case class UnaryMinus( @ExpressionDescription( usage = "_FUNC_(expr) - Returns the value of `expr`.", + arguments = """ + Arguments: + * expr - The input expression whose value is returned. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -140,6 +150,11 @@ case class UnaryPositive(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the absolute value of the numeric or interval value.", + arguments = """ + Arguments: + * expr - The numeric or interval value to take the absolute value of. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT _FUNC_(-1); @@ -388,6 +403,13 @@ object BinaryArithmetic { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`+`expr2`.", + arguments = """ + Arguments: + * expr1 - The first addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + * expr2 - The second addend. + An expression that evaluates to a numeric, interval, date, timestamp, or time. + """, examples = """ Examples: > SELECT 1 _FUNC_ 2; @@ -562,6 +584,13 @@ object Subtract { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`*`expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand. + An expression that evaluates to a numeric or interval. + * expr2 - The second operand. + An expression that evaluates to a numeric or interval. + """, examples = """ Examples: > SELECT 2 _FUNC_ 3; @@ -811,6 +840,13 @@ trait DivModLike extends BinaryArithmetic { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns `expr1`/`expr2`. It always performs floating point division.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric or interval. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -893,6 +929,13 @@ object Divide { // scalastyle:off line.size.limit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long.", + arguments = """ + Arguments: + * expr1 - The dividend to be divided. + An expression that evaluates to an integral, decimal, or interval. + * expr2 - The divisor to divide by. + An expression that evaluates to an integral, decimal, or interval. + """, examples = """ Examples: > SELECT 3 _FUNC_ 2; @@ -976,6 +1019,13 @@ object IntegralDivide { @ExpressionDescription( usage = "expr1 % expr2, or mod(expr1, expr2) - Returns the remainder after `expr1`/`expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT 2 % 1.8; @@ -1064,6 +1114,13 @@ object Remainder { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the positive value of `expr1` mod `expr2`.", + arguments = """ + Arguments: + * expr1 - The dividend. + An expression that evaluates to a numeric. + * expr2 - The divisor. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(10, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala index 5a73fd830e037..1f943ed7edb08 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitmapExpressions.scala @@ -30,6 +30,11 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType, LongT @ExpressionDescription( usage = "_FUNC_(child) - Returns the bucket number for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bucket number for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(123); @@ -63,6 +68,11 @@ case class BitmapBucketNumber(child: Expression) @ExpressionDescription( usage = "_FUNC_(child) - Returns the bit position for the given input child expression.", + arguments = """ + Arguments: + * child - The input expression to compute the bit position for. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -148,6 +158,11 @@ case class BitmapCount(child: Expression) _FUNC_(child) - Returns a bitmap with the positions of the bits set from all the values from the child expression. The child expression will most likely be bitmap_bit_position(). """, + arguments = """ + Arguments: + * child - The expression whose values set the bit positions in the bitmap. + An expression that evaluates to a long. + """, // scalastyle:off line.size.limit examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 5fac0a93bf9bf..7fcc150cf05c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -29,6 +29,13 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise AND. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise AND. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -78,6 +85,13 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise OR. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise OR. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -127,6 +141,13 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet */ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of `expr1` and `expr2`.", + arguments = """ + Arguments: + * expr1 - The first operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + * expr2 - The second operand of the bitwise exclusive OR. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT 3 _FUNC_ 5; @@ -174,6 +195,11 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme */ @ExpressionDescription( usage = "_FUNC_ expr - Returns the result of bitwise NOT of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the bitwise NOT of. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_ 0; @@ -216,6 +242,11 @@ case class BitwiseNot(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the number of bits that are set in the argument expr as an" + " unsigned 64-bit integer, or NULL if the argument is NULL.", + arguments = """ + Arguments: + * expr - The argument whose set bits are counted. + An expression that evaluates to an integral or boolean. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -266,6 +297,13 @@ object BitwiseGetUtil { The positions are numbered from right to left, starting at zero. The position argument cannot be negative. """, + arguments = """ + Arguments: + * expr - The expression to read the bit from. + An expression that evaluates to an integral. + * pos - The position of the bit to return, numbered from right to left starting at zero. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(11, 0); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala index c3db6fca6a861..bf1bc66ca8b19 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala @@ -145,6 +145,7 @@ case class ResolvedCollation(collationName: String) extends LeafExpression { arguments = """ Arguments: * expr - String expression to perform collation on. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ab56386869a02..195113695f1ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -101,6 +101,11 @@ trait BinaryArrayExpressionWithImplicitCast spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the default settings, the function returns null for null input. """, + arguments = """ + Arguments: + * expr - The array or map whose size is returned. + An expression that evaluates to an array or map. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -157,6 +162,11 @@ object Size { */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the size of an array. The function returns null for null input.", + arguments = """ + Arguments: + * expr - The array to return the size of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a')); @@ -279,6 +289,12 @@ case class MapContainsKey(left: Expression, right: Expression) _FUNC_(a1, a2, ...) - Returns a merged array of structs in which the N-th struct contains all N-th values of input arrays. """, + arguments = """ + Arguments: + * a1 - The first array to merge. + An expression that evaluates to an array. + * a2 - The second array to merge. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(2, 3, 4)); @@ -1044,6 +1060,13 @@ case class MapSort(base: Expression) elements for double/float type. Null elements will be placed at the beginning of the returned array in ascending order or at the end of the returned array in descending order. """, + arguments = """ + Arguments: + * array - The array to sort. + An expression that evaluates to an array. + * ascendingOrder - Whether to sort in ascending order; false sorts in descending order. + An expression that evaluates to a boolean. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true); @@ -1244,6 +1267,11 @@ case class SortArray(base: Expression, ascendingOrder: Expression) */ @ExpressionDescription( usage = "_FUNC_(array) - Returns a random permutation of the given array.", + arguments = """ + Arguments: + * array - The array to return a random permutation of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, 3, 5)); @@ -1344,6 +1372,11 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None) extends U @ExpressionDescription( usage = """_FUNC_(expr) - Returns a reversed string, a binary value with bytes in reverse order, or an array with reverse order of elements.""", + arguments = """ + Arguments: + * expr - The string, binary value, or array to reverse. + An expression that evaluates to a string, binary, or array. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -1451,6 +1484,13 @@ case class Reverse(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, value) - Returns true if the array contains the value.", + arguments = """ + Arguments: + * array - The array to search. + An expression that evaluates to an array. + * value - The value to check for membership in the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -1731,6 +1771,13 @@ trait ArrayPendBase extends RuntimeReplaceable Null element is also prepended to the array. But if the array passed is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to prepend the element to. + An expression that evaluates to an array. + * element - The element to add at the beginning of the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1766,6 +1813,13 @@ case class ArrayPrepend(left: Expression, right: Expression) extends ArrayPendBa Null element is also appended into the array. But if the array passed, is NULL output is NULL """, + arguments = """ + Arguments: + * array - The array to append the element to. + An expression that evaluates to an array. + * element - The element to add at the end of the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), 'd'); @@ -1794,6 +1848,13 @@ case class ArrayAppend(left: Expression, right: Expression) extends ArrayPendBas // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(a1, a2) - Returns true if a1 contains at least a non-null element present also in a2. If the arrays have no common element and they are both non-empty and either of them contains a null element null is returned, false otherwise.", + arguments = """ + Arguments: + * a1 - The first array to compare. + An expression that evaluates to an array. + * a2 - The second array to compare. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5)); @@ -2022,6 +2083,15 @@ case class ArraysOverlap(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(x, start, length) - Subsets array x starting from index start (array indices start at 1, or starting from the end if start is negative) with the specified length.", + arguments = """ + Arguments: + * x - The array to take a subset of. + An expression that evaluates to an array. + * start - The 1-based start index, or from the end if negative. + An expression that evaluates to an integer. + * length - The number of elements to take. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 2, 2); @@ -2121,6 +2191,15 @@ case class Slice(x: Expression, start: Expression, length: Expression) _FUNC_(array, delimiter[, nullReplacement]) - Concatenates the elements of the given array using the delimiter and an optional string to replace nulls. If no value is set for nullReplacement, any null value is filtered.""", + arguments = """ + Arguments: + * array - The array whose elements are concatenated. + An expression that evaluates to an array. + * delimiter - The delimiter placed between concatenated elements. + An expression that evaluates to a string. + * nullReplacement - The string used to replace null elements. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(array('hello', 'world'), ' '); @@ -2316,6 +2395,11 @@ case class ArrayJoin( usage = """ _FUNC_(array) - Returns the minimum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the minimum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2389,6 +2473,11 @@ case class ArrayMin(child: Expression) usage = """ _FUNC_(array) - Returns the maximum value in the array. NaN is greater than any non-NaN elements for double/float type. NULL elements are skipped.""", + arguments = """ + Arguments: + * array - The array to find the maximum value of. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 20, null, 3)); @@ -2469,6 +2558,13 @@ case class ArrayMax(child: Expression) _FUNC_(array, element) - Returns the (1-based) index of the first matching element of the array as long, or 0 if no match is found. """, + arguments = """ + Arguments: + * array - The array to search within. + An expression that evaluates to an array. + * element - The element to find the position of. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(312, 773, 708, 708), 708); @@ -2571,6 +2667,13 @@ case class ArrayPosition(left: Expression, right: Expression) _FUNC_(array, index) - Returns element of array at given (0-based) index. If the index points outside of the array boundaries, then this function returns NULL. """, + arguments = """ + Arguments: + * array - The array to retrieve an element from. + An expression that evaluates to an array. + * index - The 0-based index of the element to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0); @@ -2619,6 +2722,13 @@ case class Get(left: Expression, right: Expression) _FUNC_(map, key) - Returns value for given key. The function returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array to retrieve the element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -2872,6 +2982,13 @@ case class ElementAt( _FUNC_(map, key) - Returns value for given key. The function always returns NULL if the key is not contained in the map. """, + arguments = """ + Arguments: + * array - The array or map to retrieve an element from. + An expression that evaluates to an array or map. + * index - The 1-based index of the array element, or the key of the map entry, to return. + An expression that evaluates to an integer for an array, or the key type for a map. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 2); @@ -3883,6 +4000,13 @@ object Sequence { */ @ExpressionDescription( usage = "_FUNC_(element, count) - Returns the array containing element count times.", + arguments = """ + Arguments: + * element - The element to repeat. + An expression of any type. + * count - The number of times to repeat the element. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -4007,6 +4131,13 @@ case class ArrayRepeat(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(array, element) - Remove all elements that equal to element from array.", + arguments = """ + Arguments: + * array - The array to remove elements from. + An expression that evaluates to an array. + * element - The element to remove from the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3), 3); @@ -4219,6 +4350,11 @@ trait ArraySetLike { */ @ExpressionDescription( usage = "_FUNC_(array) - Removes duplicate values from the array.", + arguments = """ + Arguments: + * array - The array to remove duplicate values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3)); @@ -4417,6 +4553,13 @@ trait ArrayBinaryLike _FUNC_(array1, array2) - Returns an array of the elements in the union of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to union. + An expression that evaluates to an array. + * array2 - The second array to union. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4596,6 +4739,13 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi _FUNC_(array1, array2) - Returns an array of the elements in the intersection of array1 and array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The first array to intersect. + An expression that evaluates to an array. + * array2 - The second array to intersect. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -4830,6 +4980,13 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina _FUNC_(array1, array2) - Returns an array of the elements in array1 but not in array2, without duplicates. """, + arguments = """ + Arguments: + * array1 - The array to take elements from. + An expression that evaluates to an array. + * array2 - The array of elements to exclude. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); @@ -5043,6 +5200,15 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL Index above array size appends the array, or prepends the array if index is negative, with 'null' elements. """, + arguments = """ + Arguments: + * x - The array to insert the value into. + An expression that evaluates to an array. + * pos - The 1-based index at which to insert the value. + An expression that evaluates to an integer. + * val - The value to insert into the array. + An expression of the same type as the array elements. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 5, 5); @@ -5391,6 +5557,11 @@ case class ArrayInsert( @ExpressionDescription( usage = "_FUNC_(array) - Removes null values from the array.", + arguments = """ + Arguments: + * array - The array to remove null values from. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 1c601b29002e3..122272ba43339 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -299,6 +299,13 @@ object CreateMap { usage = """ _FUNC_(keys, values) - Creates a map with a pair of the given key/value arrays. All elements in keys should not be null""", + arguments = """ + Arguments: + * keys - The array providing the keys of the map. + An expression that evaluates to an array. + * values - The array providing the values of the map. + An expression that evaluates to an array. + """, examples = """ Examples: > SELECT _FUNC_(array(1.0, 3.0), array('2', '4')); @@ -566,6 +573,15 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression with // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for `pairDelim` and ':' for `keyValueDelim`. Both `pairDelim` and `keyValueDelim` are treated as regular expressions.", + arguments = """ + Arguments: + * text - The text to split into key/value pairs. + An expression that evaluates to a string. + * pairDelim - The delimiter regular expression separating pairs, defaults to ','. + An expression that evaluates to a string. + * keyValueDelim - The delimiter regular expression separating key and value, defaults to ':'. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index e7c6a1091861b..f966b04bec263 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -316,6 +316,13 @@ case class CurrentBatchTimestamp( */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to add to the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -356,6 +363,13 @@ case class DateAdd(startDate: Expression, days: Expression) */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` before `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date. + An expression that evaluates to a date. + * num_days - The number of days to subtract from the start date. + An expression that evaluates to an integer, short, or byte. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30', 1); @@ -548,6 +562,11 @@ trait GetDateField extends UnaryExpression with ImplicitCastInputTypes { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of year of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-04-09'); @@ -569,6 +588,11 @@ case class DayOfYear(child: Expression) extends GetDateField { @ExpressionDescription( usage = "_FUNC_(days) - Create date from the number of days since 1970-01-01.", + arguments = """ + Arguments: + * days - The number of days since 1970-01-01. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -597,6 +621,11 @@ case class DateFromUnixDate(child: Expression) extends UnaryExpression @ExpressionDescription( usage = "_FUNC_(date) - Returns the number of days since 1970-01-01.", + arguments = """ + Arguments: + * date - The date to convert to the number of days since 1970-01-01. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE("1970-01-02")); @@ -666,6 +695,11 @@ abstract class IntegralToTimestampBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(seconds) - Creates timestamp from the number of seconds (can be fractional) since UTC epoch.", + arguments = """ + Arguments: + * seconds - The number of seconds since the UTC epoch. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(1230219000); @@ -802,6 +836,11 @@ case class SecondsToTimestamp(child: Expression) extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(milliseconds) - Creates timestamp from the number of milliseconds since UTC epoch.", + arguments = """ + Arguments: + * milliseconds - The number of milliseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123); @@ -826,6 +865,11 @@ case class MillisToTimestamp(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(microseconds) - Creates timestamp from the number of microseconds since UTC epoch.", + arguments = """ + Arguments: + * microseconds - The number of microseconds since the UTC epoch. + An expression that evaluates to an integral. + """, examples = """ Examples: > SELECT _FUNC_(1230219000123123); @@ -850,6 +894,11 @@ case class MicrosToTimestamp(child: Expression) // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(nanoseconds) - Creates timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from the number of nanoseconds since UTC epoch.", + arguments = """ + Arguments: + * nanoseconds - The number of nanoseconds since the UTC epoch. + An expression that evaluates to an integral or decimal. + """, examples = """ Examples: > SET spark.sql.timestampNanosTypes.enabled=true; @@ -965,6 +1014,11 @@ abstract class TimestampToLongBase extends UnaryExpression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to seconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -998,6 +1052,11 @@ case class CastTimestampNTZToLong(child: Expression) extends TimestampToLongBase // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of precision.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to milliseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1018,6 +1077,11 @@ case class UnixMillis(child: Expression) extends TimestampToLongBase { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp) - Returns the number of microseconds since 1970-01-01 00:00:00 UTC.", + arguments = """ + Arguments: + * timestamp - The timestamp to convert to microseconds since the epoch. + An expression that evaluates to a timestamp. + """, examples = """ Examples: > SELECT _FUNC_(TIMESTAMP('1970-01-01 00:00:01Z')); @@ -1089,6 +1153,11 @@ case class UnixNanos(child: Expression) // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1118,6 +1187,11 @@ case class YearOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the quarter from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31'); @@ -1140,6 +1214,11 @@ case class Quarter(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the month component of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2016-07-30'); @@ -1161,6 +1240,11 @@ case class Month(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of month of the date/timestamp.", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of month from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1183,6 +1267,11 @@ case class DayOfMonth(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (1 = Sunday, 2 = Monday, ..., 7 = Saturday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1205,6 +1294,11 @@ case class DayOfWeek(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday).", + arguments = """ + Arguments: + * date - The date or timestamp to extract the day of the week from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30'); @@ -1227,6 +1321,11 @@ case class WeekDay(child: Expression) extends GetDateField { // scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the week of the year of the given date. A week is considered to start on a Monday and week 1 is the first week with >3 days.", + arguments = """ + Arguments: + * date - The date to extract the week of the year from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1249,6 +1348,11 @@ case class WeekOfYear(child: Expression) extends GetDateField { // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated month name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated month name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2008-02-20'); @@ -1271,6 +1375,11 @@ case class MonthName(child: Expression) extends GetDateField with DefaultStringP // scalastyle:off line.contains.tab @ExpressionDescription( usage = "_FUNC_(date) - Returns the three-letter abbreviated day name from the given date.", + arguments = """ + Arguments: + * date - The date to extract the abbreviated day name from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_(DATE('2008-02-20')); @@ -1298,8 +1407,10 @@ case class DayName(child: Expression) extends GetDateField with DefaultStringPro arguments = """ Arguments: * timestamp - A date, time, timestamp or string to be converted to the given format. + An expression that evaluates to a timestamp or time. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1439,9 +1550,11 @@ object DateFormatClass { arguments = """ Arguments: * timeExp - A date/timestamp or string which is returned as a UNIX timestamp. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1493,9 +1606,11 @@ case class ToUnixTimestamp( arguments = """ Arguments: * timeExp - A date/timestamp or string. If not provided, this defaults to current time. + An expression that evaluates to a string, date, or timestamp. * fmt - Date/time format pattern to follow. Ignored if `timeExp` is not a string. Default value is "yyyy-MM-dd HH:mm:ss". See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1578,8 +1693,10 @@ case class GetTimestamp( arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp without time zone. + An expression that evaluates to a string, date, or timestamp. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1615,6 +1732,7 @@ object ParseToTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp with local time zone. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1653,6 +1771,7 @@ object ParseToTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -1874,8 +1993,10 @@ abstract class UnixTime extends ToTimestamp { arguments = """ Arguments: * unix_time - UNIX Timestamp to be converted to the provided format. + An expression that evaluates to a long. * fmt - Date/time format pattern to follow. See Datetime Patterns for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1953,6 +2074,11 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ */ @ExpressionDescription( usage = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", + arguments = """ + Arguments: + * date - The date whose month's last day is returned. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-01-12'); @@ -2000,6 +2126,13 @@ case class LastDay(startDate: Expression) When both of the input parameters are not NULL and day_of_week is an invalid input, the function throws SparkIllegalArgumentException if `spark.sql.ansi.enabled` is set to true, otherwise NULL. """, + arguments = """ + Arguments: + * start_date - The date after which to find the next occurrence of the given day of week. + An expression that evaluates to a date. + * day_of_week - The name of the day of week to find (e.g. "Mon", "Tuesday"). + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2015-01-14', 'TU'); @@ -2353,6 +2486,13 @@ sealed trait UTCTimestamp extends BinaryExpression with ImplicitCastInputTypes { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 03:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in UTC. + An expression that evaluates to a timestamp. + * timezone - The target time zone to render the timestamp in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2387,6 +2527,13 @@ case class FromUTCTimestamp(left: Expression, right: Expression) extends UTCTime // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp, timezone) - Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield '2017-07-14 01:40:00.0'.", + arguments = """ + Arguments: + * timestamp - The timestamp to interpret as a time in the given time zone. + An expression that evaluates to a timestamp. + * timezone - The time zone the timestamp is interpreted as being in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -2426,6 +2573,13 @@ abstract class AddMonthsBase extends BinaryExpression with ImplicitCastInputType // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(start_date, num_months) - Returns the date that is `num_months` after `start_date`.", + arguments = """ + Arguments: + * start_date - The starting date to add months to. + An expression that evaluates to a date. + * num_months - The number of months to add to the start date. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('2016-08-31', 1); @@ -2519,6 +2673,15 @@ case class TimestampAddYMInterval( are the last day of month, time of day will be ignored. Otherwise, the difference is calculated based on 31 days per month, and rounded to 8 digits unless roundOff=false. """, + arguments = """ + Arguments: + * timestamp1 - The first timestamp to compare. + An expression that evaluates to a timestamp. + * timestamp2 - The second timestamp to compare. + An expression that evaluates to a timestamp. + * roundOff - Whether to round off the result to 8 decimal places. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); @@ -2586,8 +2749,10 @@ case class MonthsBetween( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2662,8 +2827,10 @@ case class ParseToDate( arguments = """ Arguments: * date_str - A string to be parsed to date. + An expression that evaluates to a string, date, or timestamp. * fmt - Date format pattern to follow. See Datetime Patterns for valid date and time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -2705,6 +2872,7 @@ object TryToDateExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * timestamp_str - A string to be parsed to timestamp. + An expression that evaluates to a string, date, timestamp, or numeric. * fmt - Timestamp format pattern to follow. See Datetime Patterns for valid date and time format patterns. """, @@ -2864,11 +3032,13 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes { arguments = """ Arguments: * date - date value or valid date string + An expression that evaluates to a date. * fmt - the format representing the unit to be truncated to - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in - "QUARTER" - truncate to the first date of the quarter that the `date` falls in - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in - "WEEK" - truncate to the Monday of the week that the `date` falls in + An expression that evaluates to a string. """, examples = """ Examples: @@ -2933,7 +3103,9 @@ case class TruncDate(date: Expression, format: Expression) - "SECOND" - zero out the second fraction part - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - everything remains + An expression that evaluates to a string. * ts - datetime value or valid timestamp string + An expression that evaluates to a timestamp. """, examples = """ Examples: @@ -2993,6 +3165,13 @@ case class TruncTimestamp( */ @ExpressionDescription( usage = "_FUNC_(endDate, startDate) - Returns the number of days from `startDate` to `endDate`.", + arguments = """ + Arguments: + * endDate - The end date to count days up to. + An expression that evaluates to a date. + * startDate - The start date to count days from. + An expression that evaluates to a date. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-31', '2009-07-30'); @@ -3031,8 +3210,11 @@ case class DateDiff(endDate: Expression, startDate: Expression) arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. """, examples = """ Examples: @@ -3252,13 +3434,19 @@ case class MakeTimestampLTZNanos( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3306,13 +3494,19 @@ object MakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 """, @@ -3363,13 +3557,19 @@ object TryMakeTimestampNTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3430,13 +3630,19 @@ object MakeTimestampLTZExpressionBuilder extends ExpressionBuilder { arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * timezone - the time zone identifier. For example, CET, UTC and etc. * date - a date to represent, from 0001-01-01 to 9999-12-31 * time - a local time to represent, from 00:00:00 to 23:59:59.999999 @@ -3702,14 +3908,20 @@ case class TryMakeTimestamp( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. The value can be either an integer like 13 , or a fraction like 13.123. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -3852,13 +4064,19 @@ case class MakeTimestampFromDateTime( arguments = """ Arguments: * year - the year to represent, from 1 to 9999 + An expression that evaluates to an integer. * month - the month-of-year to represent, from 1 (January) to 12 (December) + An expression that evaluates to an integer. * day - the day-of-month to represent, from 1 to 31 + An expression that evaluates to an integer. * hour - the hour-of-day to represent, from 0 to 23 + An expression that evaluates to an integer. * min - the minute-of-hour to represent, from 0 to 59 + An expression that evaluates to an integer. * sec - the second-of-minute and its micro-fraction to represent, from 0 to 60. If the sec argument equals to 60, the seconds field is set to 0 and 1 minute is added to the final timestamp. + An expression that evaluates to a decimal. * date - a date expression * time - a time expression (optional). Default is 00:00:00. * timezone - the time zone identifier (optional). For example, CET, UTC and etc. @@ -4250,8 +4468,11 @@ object SubtractDates { Arguments: * sourceTz - the time zone for the input timestamp. If it is missed, the current session time zone is used as the source time zone. + An expression that evaluates to a string. * targetTz - the time zone to which the input timestamp should be converted + An expression that evaluates to a string. * sourceTs - a timestamp without time zone + An expression that evaluates to a timestamp. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala index 3e6f6dbe22e07..2a9a140ef2a99 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala @@ -52,6 +52,11 @@ import org.apache.spark.util.ArrayImplicits._ */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the MD5 checksum of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -94,6 +99,13 @@ case class Md5(child: Expression) _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. """, + arguments = """ + Arguments: + * expr - The expression to compute the SHA-2 checksum of. + An expression that evaluates to a binary. + * bitLength - The bit length of the SHA-2 result (224, 256, 384, or 512). + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark', 256); @@ -160,6 +172,11 @@ case class Sha2(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the SHA-1 hash of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); @@ -195,6 +212,11 @@ case class Sha1(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint.", + arguments = """ + Arguments: + * expr - The expression to compute the cyclic redundancy check value of. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index f87b4f70d298a..524e27321f4fa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -393,6 +393,13 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction { */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Transforms elements in an array using the function.", + arguments = """ + Arguments: + * expr - The array whose elements are transformed. + An expression that evaluates to an array. + * func - The function applied to each element. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x + 1); @@ -513,6 +520,13 @@ case class ArrayTransform( equal to, or greater than the second element. If the comparator function returns null, the function will fail and raise an error. """, + arguments = """ + Arguments: + * expr - The input array to sort. + An expression that evaluates to an array. + * func - The comparator function used to determine the sort order. + A lambda function returning an integer. + """, examples = """ Examples: > SELECT _FUNC_(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); @@ -700,6 +714,13 @@ case class MapFilter( */ @ExpressionDescription( usage = "_FUNC_(expr, func) - Filters the input array using the given predicate.", + arguments = """ + Arguments: + * expr - The array to filter. + An expression that evaluates to an array. + * func - The predicate used to filter the array. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1); @@ -838,6 +859,13 @@ case class ArrayFilter( */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for one or more elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -972,6 +1000,13 @@ object ArrayExists { */ @ExpressionDescription(usage = "_FUNC_(expr, pred) - Tests whether a predicate holds for all elements in the array.", + arguments = """ + Arguments: + * expr - The array to test. + An expression that evaluates to an array. + * pred - The predicate tested against the array elements. + A lambda function returning a boolean. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0); @@ -1093,6 +1128,17 @@ case class ArrayForAll( elements in the array, and reduces this to a single state. The final state is converted into the final result by applying a finish function. """, + arguments = """ + Arguments: + * expr - The input array to reduce. + An expression that evaluates to an array. + * start - The initial state to start the reduction from. + An expression of any type. + * merge - The binary operator applied to the current state and each array element. + A lambda function. + * finish - The function that converts the final state into the result. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), 0, (acc, x) -> acc + x); @@ -1652,6 +1698,15 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(left, right, func) - Merges the two given arrays, element-wise, into a single array using function. If one array is shorter, nulls are appended at the end to match the length of the longer array, before applying function.", + arguments = """ + Arguments: + * left - The first array to merge. + An expression that evaluates to an array. + * right - The second array to merge. + An expression that evaluates to an array. + * func - The function applied element-wise to merge the arrays. + A lambda function. + """, examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 3e4d6772c4fc4..29522cbcc15c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -233,6 +233,7 @@ case class DivideInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -316,6 +317,7 @@ case class TryMakeInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative * weeks - the number of weeks, positive or negative * days - the number of days, positive or negative @@ -474,9 +476,13 @@ case class MakeInterval( arguments = """ Arguments: * days - the number of days, positive or negative + An expression that evaluates to an integer. * hours - the number of hours, positive or negative + An expression that evaluates to an integer. * mins - the number of minutes, positive or negative + An expression that evaluates to an integer. * secs - the number of seconds with the fractional part in microsecond precision. + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -557,7 +563,9 @@ case class MakeDTInterval( arguments = """ Arguments: * years - the number of years, positive or negative + An expression that evaluates to an integer. * months - the number of months, positive or negative + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index b4f41fda42f00..a6792d80ef341 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -42,6 +42,13 @@ import org.apache.spark.unsafe.types.UTF8String */ @ExpressionDescription( usage = "_FUNC_(json_txt, path) - Extracts a json object from `path`.", + arguments = """ + Arguments: + * json_txt - The JSON text to extract from. + An expression that evaluates to a string. + * path - The path identifying the JSON object to extract. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":"b"}', '$.a'); @@ -449,6 +456,12 @@ object JsonToStructs { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, options]) - Returns a JSON string with a given struct value", + arguments = """ + Arguments: + * expr - The struct value to convert to a JSON string. + An expression that evaluates to a struct, array, map, or variant. + * options - Options controlling how the JSON string is produced. + """, examples = """ Examples: > SELECT _FUNC_(named_struct('a', 1, 'b', 2)); @@ -611,6 +624,7 @@ case class SchemaOfJson( Arguments: * jsonArray - A JSON array. `NULL` is returned in case of any other valid JSON string, `NULL` or an invalid JSON. + An expression that evaluates to a string. """, examples = """ Examples: @@ -657,6 +671,7 @@ case class LengthOfJsonArray(child: Expression) * json_object - A JSON object. If a valid JSON object is given, all the keys of the outermost object will be returned as an array. If it is any other valid JSON string, an invalid JSON string or an empty string, the function returns null. + An expression that evaluates to a string. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala index 01481050f2805..ba6c68ab95fcc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/kllExpressions.scala @@ -33,6 +33,11 @@ import org.apache.spark.unsafe.types.UTF8String usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -61,6 +66,11 @@ case class KllSketchToStringBigint(child: Expression) extends KllSketchToStringB usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -89,6 +99,11 @@ case class KllSketchToStringFloat(child: Expression) extends KllSketchToStringBa usage = """ _FUNC_(expr) - Returns human readable summary information about this sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return summary information about. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(_FUNC_(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -127,6 +142,11 @@ abstract class KllSketchToStringBase usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -155,6 +175,11 @@ case class KllSketchGetNBigint(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -183,6 +208,11 @@ case class KllSketchGetNFloat(child: Expression) extends KllSketchGetNBase { usage = """ _FUNC_(expr) - Returns the number of items collected in the sketch. """, + arguments = """ + Arguments: + * expr - The sketch to return the number of collected items from. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -221,6 +251,13 @@ abstract class KllSketchGetNBase usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_bigint(_FUNC_(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -252,6 +289,13 @@ case class KllSketchMergeBigint(left: Expression, right: Expression) extends Kll usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_float(_FUNC_(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -283,6 +327,13 @@ case class KllSketchMergeFloat(left: Expression, right: Expression) extends KllS usage = """ _FUNC_(left, right) - Merges two sketch buffers together into one. """, + arguments = """ + Arguments: + * left - The first sketch buffer to merge. + An expression that evaluates to a binary. + * right - The second sketch buffer to merge. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT LENGTH(kll_sketch_to_string_double(_FUNC_(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -327,6 +378,13 @@ abstract class KllSketchMergeBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -359,6 +417,13 @@ case class KllSketchGetQuantileBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -391,6 +456,13 @@ case class KllSketchGetQuantileFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the quantile from. + An expression that evaluates to a binary. + * right - The input rank, or array of ranks, to compute the quantile for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); @@ -523,6 +595,13 @@ abstract class KllSketchGetQuantileBase or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a long or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col); @@ -551,6 +630,13 @@ case class KllSketchGetRankBigint(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a float or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col); @@ -579,6 +665,13 @@ case class KllSketchGetRankFloat(left: Expression, right: Expression) or an array. In the latter case, the function will return an array of results of equal length to the input array. """, + arguments = """ + Arguments: + * left - The sketch buffer to extract the rank from. + An expression that evaluates to a binary. + * right - The input quantile, or array of quantiles, to compute the rank for. + An expression that evaluates to a double or array. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala index 5bba89b3dad51..9613bc25d65f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/maskExpressions.scala @@ -38,10 +38,15 @@ import org.apache.spark.unsafe.types.UTF8String arguments = """ Arguments: * input - string value to mask. Supported types: STRING, VARCHAR, CHAR + An expression that evaluates to a string. * upperChar - character to replace upper-case characters with. Specify NULL to retain original character. Default value: 'X' + An expression that evaluates to a string. Must be a constant. * lowerChar - character to replace lower-case characters with. Specify NULL to retain original character. Default value: 'x' + An expression that evaluates to a string. Must be a constant. * digitChar - character to replace digit characters with. Specify NULL to retain original character. Default value: 'n' + An expression that evaluates to a string. Must be a constant. * otherChar - character to replace all other characters with. Specify NULL to retain original character. Default value: NULL + An expression that evaluates to a string. Must be a constant. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 7bd166250db01..5fecba3fc54f0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -192,6 +192,11 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") _FUNC_(expr) - Returns the inverse cosine (a.k.a. arc cosine) of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -210,6 +215,11 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS" _FUNC_(expr) - Returns the inverse sine (a.k.a. arc sine) the arc sin of `expr`, as if computed by `java.lang.Math._FUNC_`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -228,6 +238,11 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN" _FUNC_(expr) - Returns the inverse tangent (a.k.a. arc tangent) of `expr`, as if computed by `java.lang.Math._FUNC_` """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -241,6 +256,11 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN" @ExpressionDescription( usage = "_FUNC_(expr) - Returns the cube root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the cube root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(27.0); @@ -308,6 +328,13 @@ trait CeilFloorExpressionBuilderBase extends ExpressionBuilder { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr[, scale]) - Returns the smallest number after rounding up that is not smaller than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round up. + An expression that evaluates to a numeric. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -349,6 +376,7 @@ case class RoundCeil(child: Expression, scale: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -368,6 +396,7 @@ case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -392,6 +421,7 @@ case class Sec(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -408,6 +438,11 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic cosine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic cosine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -436,6 +471,15 @@ case class Acosh(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`.", + arguments = """ + Arguments: + * num - The number to convert. + An expression that evaluates to a string. + * from_base - The base the number is currently in. + An expression that evaluates to an integer. + * to_base - The base to convert the number to. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('100', 2, 10); @@ -501,6 +545,11 @@ case class Conv( @ExpressionDescription( usage = "_FUNC_(expr) - Returns e to the power of `expr`.", + arguments = """ + Arguments: + * expr - The exponent to raise e to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -517,6 +566,11 @@ case class Exp(child: Expression) extends UnaryMathExpression(StrictMath.exp, "E @ExpressionDescription( usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.", + arguments = """ + Arguments: + * expr - The exponent used to compute exp(expr) - 1. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -564,6 +618,13 @@ case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLO // scalastyle:off line.size.limit @ExpressionDescription( usage = " _FUNC_(expr[, scale]) - Returns the largest number after rounding down that is not greater than `expr`. An optional `scale` parameter can be specified to control the rounding behavior.", + arguments = """ + Arguments: + * expr - The expression to round down. + An expression that evaluates to a numeric. + * scale - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(-0.1); @@ -630,6 +691,11 @@ object Factorial { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null.", + arguments = """ + Arguments: + * expr - The integer between 0 and 20 to compute the factorial of. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_(5); @@ -676,6 +742,11 @@ case class Factorial(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the natural logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(1); @@ -690,6 +761,11 @@ case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LO @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.", + arguments = """ + Arguments: + * expr - The expression to compute the base-2 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2); @@ -715,6 +791,11 @@ case class Log2(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 10.", + arguments = """ + Arguments: + * expr - The expression to compute the base-10 logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10); @@ -728,6 +809,11 @@ case class Log10(child: Expression) extends UnaryLogExpression(StrictMath.log10, @ExpressionDescription( usage = "_FUNC_(expr) - Returns log(1 + `expr`).", + arguments = """ + Arguments: + * expr - The expression used to compute log(1 + expr). + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -743,6 +829,11 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.", + arguments = """ + Arguments: + * expr - The value to round to the closest mathematical integer. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(12.3456); @@ -759,6 +850,11 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND @ExpressionDescription( usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive.", + arguments = """ + Arguments: + * expr - The expression whose sign is returned. + An expression that evaluates to a double or interval. + """, examples = """ Examples: > SELECT _FUNC_(40); @@ -782,6 +878,7 @@ case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "S arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -801,6 +898,7 @@ case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -824,6 +922,7 @@ case class Csc(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -840,6 +939,11 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic sine of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic sine of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -859,6 +963,11 @@ case class Asinh(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the square root of `expr`.", + arguments = """ + Arguments: + * expr - The expression to compute the square root of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(4); @@ -877,6 +986,7 @@ case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT" arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -896,6 +1006,7 @@ case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") { arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -920,6 +1031,7 @@ case class Cot(child: Expression) arguments = """ Arguments: * expr - hyperbolic angle + An expression that evaluates to a double. """, examples = """ Examples: @@ -936,6 +1048,11 @@ case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH" usage = """ _FUNC_(expr) - Returns inverse hyperbolic tangent of `expr`. """, + arguments = """ + Arguments: + * expr - The expression to compute the inverse hyperbolic tangent of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -961,6 +1078,7 @@ case class Atanh(child: Expression) arguments = """ Arguments: * expr - angle in radians + An expression that evaluates to a double. """, examples = """ Examples: @@ -980,6 +1098,7 @@ case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegre arguments = """ Arguments: * expr - angle in degrees + An expression that evaluates to a double. """, examples = """ Examples: @@ -997,6 +1116,11 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the string representation of the long value `expr` represented in binary.", + arguments = """ + Arguments: + * expr - The long value to represent in binary. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(13); @@ -1112,6 +1236,11 @@ object Hex { */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts `expr` to hexadecimal.", + arguments = """ + Arguments: + * expr - The expression to convert to hexadecimal. + An expression that evaluates to a long, binary, or string. + """, examples = """ Examples: > SELECT _FUNC_(17); @@ -1162,6 +1291,11 @@ case class Hex(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Converts hexadecimal `expr` to binary.", + arguments = """ + Arguments: + * expr - The hexadecimal expression to convert to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); @@ -1226,7 +1360,9 @@ case class Unhex(child: Expression, failOnError: Boolean = false) arguments = """ Arguments: * exprY - coordinate on y-axis + An expression that evaluates to a double. * exprX - coordinate on x-axis + An expression that evaluates to a double. """, examples = """ Examples: @@ -1253,6 +1389,13 @@ case class Atan2(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Raises `expr1` to the power of `expr2`.", + arguments = """ + Arguments: + * expr1 - The base to be raised to a power. + An expression that evaluates to a double. + * expr2 - The exponent to raise the base to. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(2, 3); @@ -1314,6 +1457,13 @@ sealed trait BitShiftOperation */ @ExpressionDescription( usage = "base _FUNC_ exp - Bitwise left shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted left. + An expression that evaluates to an integer or long. + * exp - The number of positions to shift left by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftleft(2, 1); @@ -1343,6 +1493,13 @@ case class ShiftLeft(left: Expression, right: Expression) extends BitShiftOperat */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise (signed) right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftright(4, 1); @@ -1371,6 +1528,13 @@ case class ShiftRight(left: Expression, right: Expression) extends BitShiftOpera */ @ExpressionDescription( usage = "base _FUNC_ expr - Bitwise unsigned right shift.", + arguments = """ + Arguments: + * base - The value whose bits are shifted right without sign extension. + An expression that evaluates to an integer or long. + * expr - The number of positions to shift right by. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT shiftrightunsigned(4, 1); @@ -1395,6 +1559,13 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) extends BitSh // scalastyle:off nonascii @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns sqrt(`expr1`\u00B2 + `expr2`\u00B2).", + arguments = """ + Arguments: + * expr1 - The first value whose square is summed. + An expression that evaluates to a double. + * expr2 - The second value whose square is summed. + An expression that evaluates to a double. + """, // scalastyle:on nonascii examples = """ Examples: @@ -1418,6 +1589,13 @@ case class Hypot(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(base, expr) - Returns the logarithm of `expr` with `base`.", + arguments = """ + Arguments: + * base - The base of the logarithm. + An expression that evaluates to a double. + * expr - The expression to compute the logarithm of. + An expression that evaluates to a double. + """, examples = """ Examples: > SELECT _FUNC_(10, 100); @@ -1701,6 +1879,13 @@ abstract class RoundBase(child: Expression, scale: Expression, // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1732,6 +1917,13 @@ case class Round( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode.", + arguments = """ + Arguments: + * expr - The expression to round. + An expression that evaluates to a numeric. + * d - The number of decimal places to round to. + An expression that evaluates to an integer. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(2.5, 0); @@ -1826,6 +2018,17 @@ object WidthBucket { `value` would be assigned in an equiwidth histogram with `num_bucket` buckets, in the range `min_value` to `max_value`." """, + arguments = """ + Arguments: + * value - The value to assign to a bucket. + An expression that evaluates to a double or interval. + * min_value - The minimum value of the histogram range. + An expression that evaluates to a double or interval. + * max_value - The maximum value of the histogram range. + An expression that evaluates to a double or interval. + * num_bucket - The number of equiwidth buckets in the histogram. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(5.3, 0.2, 10.6, 5); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 23e8b343e431e..651b1a231279d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -154,6 +154,11 @@ object RaiseError { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_( expr ) - Throws a USER_RAISED_EXCEPTION with `expr` as message.", + arguments = """ + Arguments: + * expr - The message for the raised exception. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('custom error message'); @@ -177,6 +182,13 @@ object RaiseErrorExpressionBuilder extends ExpressionBuilder { */ @ExpressionDescription( usage = "_FUNC_(expr [, message]) - Throws an exception if `expr` is not true.", + arguments = """ + Arguments: + * expr - The expression that is expected to be true. + An expression that evaluates to a boolean. + * message - The message for the exception thrown when the expression is not true. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(0 < 1); @@ -412,15 +424,21 @@ case class CurrentUser() arguments = """ Arguments: * expr - The binary value to encrypt. + An expression that evaluates to a binary. * key - The passphrase to use to encrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * iv - Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or ''. 16-byte array for CBC mode. 12-byte array for GCM mode. + An expression that evaluates to a binary. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -495,13 +513,18 @@ case class AesEncrypt( arguments = """ Arguments: * expr - The binary value to decrypt. + An expression that evaluates to a binary. * key - The passphrase to use to decrypt the data. + An expression that evaluates to a binary. * mode - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, GCM, CBC. + An expression that evaluates to a string. * padding - Specifies how to pad messages whose length is not a multiple of the block size. Valid values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS for CBC. + An expression that evaluates to a string. * aad - Optional additional authenticated data. Only supported for GCM mode. This can be any free-form input and must be provided for both encryption and decryption. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -561,6 +584,19 @@ case class AesDecrypt( @ExpressionDescription( usage = "_FUNC_(expr, key[, mode[, padding[, aad]]]) - This is a special version of `aes_decrypt` that performs the same operation, but returns a NULL value instead of raising an error if the decryption cannot be performed.", + arguments = """ + Arguments: + * expr - The encrypted value to decrypt. + An expression that evaluates to a binary. + * key - The key used to decrypt the value. + An expression that evaluates to a binary. + * mode - The block cipher mode used for decryption. + An expression that evaluates to a string. + * padding - The padding scheme used for decryption. + An expression that evaluates to a string. + * aad - The additional authenticated data. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_(unhex('6E7CA17BBB468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 1d50d8987ccee..eb8e67eca3f14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -188,6 +188,13 @@ private case class TypedNullLiteral(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not equal to the other expression. + An expression of any orderable type. + * expr2 - The value compared against the first expression. + An expression of any orderable type. + """, examples = """ Examples: > SELECT _FUNC_(2, 2); @@ -219,6 +226,11 @@ case class NullIf(left: Expression, right: Expression, replacement: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns null if `expr` is equal to zero, or `expr` otherwise.", + arguments = """ + Arguments: + * expr - The expression that returns null when equal to zero. + An expression that evaluates to a numeric. + """, examples = """ Examples: > SELECT _FUNC_(0); @@ -313,6 +325,11 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression, replace */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if `expr` is NaN, or false otherwise.", + arguments = """ + Arguments: + * expr - The expression to test for NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double)); @@ -361,6 +378,13 @@ case class IsNaN(child: Expression) extends UnaryExpression */ @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise.", + arguments = """ + Arguments: + * expr1 - The value returned when it is not NaN. + An expression that evaluates to a double or float. + * expr2 - The value returned when the first expression is NaN. + An expression that evaluates to a double or float. + """, examples = """ Examples: > SELECT _FUNC_(cast('NaN' as double), 123); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala index 0a6d23977d2f6..6389a9e0b5660 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala @@ -127,6 +127,13 @@ abstract class ToNumberBase(left: Expression, right: Expression, errorOnFail: Bo negative number with wrapping angled brackets. ('<1>'). """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -162,6 +169,13 @@ case class ToNumber(left: Expression, right: Expression) Returns NULL if the string 'expr' does not match the expected format. The format follows the same semantics as the to_number function. """, + arguments = """ + Arguments: + * expr - The string to convert to a number. + An expression that evaluates to a string. + * fmt - The format that specifies how to parse the string as a number. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('454', '999'); @@ -223,6 +237,13 @@ case class TryToNumber(left: Expression, right: Expression) 'hex': a string in the hexadecimal format. 'utf-8': the input binary is decoded to UTF-8 string. """, + arguments = """ + Arguments: + * expr - The expression to convert to a string. + An expression that evaluates to a numeric, date, timestamp, time, or binary. + * format - The format that specifies how to render the value as a string. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(454, '999'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 7767e97d34ac5..d9f9c7fec6dab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -306,6 +306,11 @@ trait PredicateHelper extends AliasHelper with Logging { @ExpressionDescription( usage = "_FUNC_ expr - Logical not.", + arguments = """ + Arguments: + * expr - The boolean expression to negate. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT _FUNC_ true; @@ -826,6 +831,13 @@ object InSet { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical AND.", + arguments = """ + Arguments: + * expr1 - The first boolean operand. + An expression that evaluates to a boolean. + * expr2 - The second boolean operand. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ true; @@ -922,6 +934,13 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical OR.", + arguments = """ + Arguments: + * expr1 - The left boolean operand of the logical OR. + An expression that evaluates to a boolean. + * expr2 - The right boolean operand of the logical OR. + An expression that evaluates to a boolean. + """, examples = """ Examples: > SELECT true _FUNC_ false; @@ -1091,6 +1110,7 @@ object Equality { * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1139,6 +1159,7 @@ case class EqualTo(left: Expression, right: Expression) * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1202,6 +1223,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp * expr1, expr2 - the two expressions must be same type or can be casted to a common type, and must be a type that can be used in equality comparison. Map type is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1238,6 +1260,7 @@ case class EqualNull(left: Expression, right: Expression, replacement: Expressio and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1273,6 +1296,7 @@ case class LessThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1308,6 +1332,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: @@ -1344,6 +1369,7 @@ case class GreaterThan(left: Expression, right: Expression) and must be a type that can be ordered. For example, map type is not orderable, so it is not supported. For complex types such array/struct, the data types of fields must be orderable. + An expression of any orderable type. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index adaaceb2e4396..b52d09fc9c709 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -92,6 +92,11 @@ private[catalyst] object ExpressionWithRandomSeed { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1).", + arguments = """ + Arguments: + * seed - The seed for the random number generator. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -146,6 +151,11 @@ object Rand { // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.""", + arguments = """ + Arguments: + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(); @@ -205,6 +215,15 @@ object Randn { one or both of these are floating-point numbers, then the result will also be a floating-point number. """, + arguments = """ + Arguments: + * min - The lower bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * max - The upper bound of the range of random values. + An expression that evaluates to a numeric. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_(10, 20, 0) > 0 AS result; @@ -329,6 +348,13 @@ object Uniform { optional. The string length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). """, + arguments = """ + Arguments: + * length - The length of the random string to generate. + An expression that evaluates to an integer. Must be a constant. + * seed - The seed used to produce reproducible random results. + An expression that evaluates to an integer or long. Must be a constant. + """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 3bd0352f6f07c..9f24c512bd335 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -127,6 +127,7 @@ private[catalyst] object StringRegexExpression { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)\ @@ -140,6 +141,7 @@ private[catalyst] object StringRegexExpression { enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -251,6 +253,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * pattern - a string expression. The pattern is a string which is matched literally and case-insensitively, with exception to the following special symbols:

_ matches any one character in the input (similar to . in posix regular expressions)

@@ -264,6 +267,7 @@ case class Like(left: Expression, right: Expression, escapeChar: Char) enabled, the pattern to match "\abc" should be "\abc".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * escape - an character added since Spark 3.0. The default escape character is the '\'. If an escape character precedes a special symbol or another escape character, the following character is matched literally. It is invalid to escape any other character. @@ -481,7 +485,9 @@ case class NotLikeAny(child: Expression, patterns: Seq[UTF8String]) extends Like arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * regexp - a string expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL parser, see the unescaping rules at String Literal. @@ -573,14 +579,17 @@ case class RLike(left: Expression, right: Expression) extends StringRegexExpress arguments = """ Arguments: * str - a string expression to split. + An expression that evaluates to a string. * regex - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. * limit - an integer expression which controls the number of times the regex is applied. * limit > 0: The resulting array's length will not be more than `limit`, and the resulting array's last entry will contain all input beyond the last matched regex. * limit <= 0: `regex` will be applied as many times as possible, and the resulting array can be of any size. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -654,6 +663,7 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) arguments = """ Arguments: * str - a string expression to search for a regular expression pattern match. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -664,9 +674,12 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * rep - a string expression to replace matched substrings. + An expression that evaluates to a string. * position - a positive integer literal that indicates the position within `str` to begin searching. The default is 1. If position is greater than the number of characters in `str`, the result is `str`. + An expression that evaluates to an integer. Must be a constant. """, examples = """ Examples: @@ -876,6 +889,7 @@ abstract class RegExpExtractBase arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -886,11 +900,13 @@ abstract class RegExpExtractBase if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex maybe contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -949,6 +965,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -959,11 +976,13 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. * idx - an integer expression that representing the group index. The regex may contains multiple groups. `idx` indicates which regex group to extract. The group index should be non-negative. The minimum value of `idx` is 0, which means matching the entire regular expression. If `idx` is not specified, the default group index value is 1. The `idx` parameter is the Java regex Matcher group() method index. + An expression that evaluates to an integer. """, examples = """ Examples: @@ -1018,8 +1037,10 @@ case class RegExpExtractAll(subject: Expression, regexp: Expression, idx: Expres arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1057,7 +1078,9 @@ case class RegExpCount(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1097,6 +1120,7 @@ case class RegExpSubStr(left: Expression, right: Expression) arguments = """ Arguments: * str - a string expression. + An expression that evaluates to a string. * regexp - a string representing a regular expression. The regex string should be a Java regular expression.

Since Spark 2.0, string literals (including regex patterns) are unescaped in our SQL @@ -1107,6 +1131,10 @@ case class RegExpSubStr(left: Expression, right: Expression) if the config is enabled, the `regexp` that can match "\abc" is "^\abc$".

It's recommended to use a raw string literal (with the `r` prefix) to avoid escaping special characters in the pattern string if exists. + An expression that evaluates to a string. + * idx - the group index of the regular expression to search for. The default value is 0 + if not specified. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala index fe774a62fcc6d..556b9603e9785 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/st/stExpressions.scala @@ -134,6 +134,7 @@ case class ST_AsBinary(geo: Expression, endianness: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOGRAPHY value. + An expression that evaluates to a binary. """, examples = """ Examples: @@ -179,7 +180,9 @@ case class ST_GeogFromWKB(wkb: Expression) arguments = """ Arguments: * wkb - A BINARY value in WKB format, representing a GEOMETRY value. + An expression that evaluates to a binary. * srid - The optional SRID value of the geometry. Default is 0. + An expression that evaluates to an integer. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 66c4a39ce8233..2aed0d9c3abe4 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -60,6 +60,11 @@ import org.apache.spark.util.ArrayImplicits._ // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values.", + arguments = """ + Arguments: + * sep - The separator placed between the concatenated values. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_(' ', 'Spark', 'SQL'); @@ -452,6 +457,11 @@ trait String2StringExpression extends ImplicitCastInputTypes { */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to uppercase.", + arguments = """ + Arguments: + * str - The string to convert to uppercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -485,6 +495,11 @@ case class Upper(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` with all characters changed to lowercase.", + arguments = """ + Arguments: + * str - The string to convert to lowercase. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('SparkSql'); @@ -593,6 +608,13 @@ case class BinaryPredicate(override val prettyName: String, left: Expression, ri Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to search within. + An expression that evaluates to a string or binary. + * right - The expression to search for inside the left expression. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -635,6 +657,13 @@ case class Contains(left: Expression, right: Expression) extends StringPredicate Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The string to test. + An expression that evaluates to a string or binary. + * right - The prefix to check for at the start of the string. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'Spark'); @@ -684,6 +713,13 @@ case class StartsWith(left: Expression, right: Expression) extends StringPredica Returns NULL if either input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or BINARY type. """, + arguments = """ + Arguments: + * left - The expression to test. + An expression that evaluates to a string or binary. + * right - The suffix expression to check the left expression ends with. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 'SQL'); @@ -735,6 +771,7 @@ case class EndsWith(left: Expression, right: Expression) extends StringPredicate arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -783,6 +820,7 @@ case class IsValidUTF8(input: Expression) extends RuntimeReplaceable with Implic arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -829,6 +867,7 @@ case class MakeValidUTF8(input: Expression) extends RuntimeReplaceable with Impl arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -877,6 +916,7 @@ case class ValidateUTF8(input: Expression) extends RuntimeReplaceable with Impli arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. """, examples = """ Examples: @@ -928,9 +968,12 @@ case class TryValidateUTF8(input: Expression) extends RuntimeReplaceable with Im arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * search - a string expression. If `search` is not found in `str`, `str` is returned unchanged. + An expression that evaluates to a string. * replace - a string expression. If `replace` is not specified or is an empty string, nothing replaces the string that is removed from `str`. + An expression that evaluates to a string. """, examples = """ Examples: @@ -1013,6 +1056,17 @@ object Overlay { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, replace, pos[, len]) - Replace `input` with `replace` that starts at `pos` and is of length `len`.", + arguments = """ + Arguments: + * input - The string to have part of its content replaced. + An expression that evaluates to a string or binary. + * replace - The replacement content to insert. + An expression that evaluates to a string or binary. + * pos - The 1-based start position of the replacement. + An expression that evaluates to an integer. + * len - The number of characters to replace. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL' PLACING '_' FROM 6); @@ -1167,6 +1221,15 @@ object StringTranslate { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(input, from, to) - Translates the `input` string by replacing the characters present in the `from` string with the corresponding characters in the `to` string.", + arguments = """ + Arguments: + * input - The string to translate. + An expression that evaluates to a string. + * from - The characters to be replaced. + An expression that evaluates to a string. + * to - The corresponding replacement characters. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('AaBbCc', 'abc', '123'); @@ -1247,6 +1310,13 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac _FUNC_(str, str_array) - Returns the index (1-based) of the given string (`str`) in the comma-delimited list (`str_array`). Returns 0, if the string was not found or if the given string (`str`) contains a comma. """, + arguments = """ + Arguments: + * str - The string to search for. + An expression that evaluates to a string. + * str_array - The comma-delimited list to search within. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('ab','abc,b,ab,c,def'); @@ -1413,7 +1483,9 @@ object StringTrim { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. * BOTH, FROM - these are keywords to specify trimming string characters from both ends of the string * LEADING, FROM - these are keywords to specify trimming string characters from the left @@ -1487,7 +1559,9 @@ case class StringTrim(srcStr: Expression, trimStr: Option[Expression] = None) arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1546,7 +1620,9 @@ object StringTrimLeft { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1611,7 +1687,9 @@ object StringTrimRight { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * trimStr - the trim string characters to trim, the default value is a single space + An expression that evaluates to a string. """, examples = """ Examples: @@ -1662,6 +1740,17 @@ case class StringTrimRight(srcStr: Expression, trimStr: Option[Expression] = Non If `start` is not specified, it defaults to 1. `occurrence` must be a positive integer and defaults to 1. """, + arguments = """ + Arguments: + * str - The string to search within. + An expression that evaluates to a string. + * substr - The substring to search for. + An expression that evaluates to a string. + * start - The 1-based position from which to start the search. + An expression that evaluates to an integer. + * occurrence - Which occurrence of the substring to find. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('SparkSQL', 'SQL'); @@ -1809,6 +1898,15 @@ case class StringInstrWithOccurrence( (counting from the right) is returned. The function substring_index performs a case-sensitive match when searching for `delim`. """, + arguments = """ + Arguments: + * str - The string to take the substring from. + An expression that evaluates to a string. + * delim - The delimiter to count occurrences of. + An expression that evaluates to a string. + * count - The number of delimiter occurrences that bound the returned substring. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('www.apache.org', '.', 2); @@ -1860,6 +1958,15 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`. The given `pos` and return value are 1-based. """, + arguments = """ + Arguments: + * substr - The substring to search for. + An expression that evaluates to a string. + * str - The string to search within. + An expression that evaluates to a string. + * pos - The 1-based position from which to start the search. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('bar', 'foobarbar'); @@ -1988,6 +2095,15 @@ trait PadExpressionBuilderBase extends ExpressionBuilder { If `pad` is not specified, `str` will be padded to the left with space characters if it is a character string, and with zeros if it is a byte sequence. """, + arguments = """ + Arguments: + * str - The string to be left-padded. + An expression that evaluates to a string or binary. + * len - The target length of the resulting string. + An expression that evaluates to an integer. + * pad - The string to pad with on the left. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2073,6 +2189,15 @@ case class BinaryPad(funcName: String, str: Expression, len: Expression, pad: Ex If `pad` is not specified, `str` will be padded to the right with space characters if it is a character string, and with zeros if it is a binary string. """, + arguments = """ + Arguments: + * str - The string to right-pad. + An expression that evaluates to a string or binary. + * len - The length to pad the string to. + An expression that evaluates to an integer. + * pad - The string used to pad on the right. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('hi', 5, '??'); @@ -2135,6 +2260,12 @@ case class StringRPad( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.", + arguments = """ + Arguments: + * strfmt - The printf-style format string. + An expression that evaluates to a string. + * obj - The value to be formatted into the string. + """, examples = """ Examples: > SELECT _FUNC_("Hello World %d %s", 100, "days"); @@ -2253,6 +2384,11 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC _FUNC_(str) - Returns `str` with the first letter of each word in uppercase. All other letters are in lowercase. Words are delimited by white space. """, + arguments = """ + Arguments: + * str - The string to capitalize the first letter of each word in. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('sPark sql'); @@ -2288,6 +2424,13 @@ case class InitCap(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.", + arguments = """ + Arguments: + * str - The string to repeat. + An expression that evaluates to a string. + * n - The number of times to repeat the string. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('123', 2); @@ -2328,6 +2471,11 @@ case class StringRepeat(str: Expression, times: Expression) */ @ExpressionDescription( usage = "_FUNC_(n) - Returns a string consisting of `n` spaces.", + arguments = """ + Arguments: + * n - The number of spaces to produce. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT concat(_FUNC_(2), '1'); @@ -2370,6 +2518,15 @@ case class StringSpace(child: Expression) _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`. """, + arguments = """ + Arguments: + * str - The string or byte array to take the substring from. + An expression that evaluates to a string or binary. + * pos - The 1-based starting position of the substring. + An expression that evaluates to an integer. + * len - The length of the substring to return. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 5); @@ -2444,6 +2601,13 @@ case class Substring(str: Expression, pos: Expression, len: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the rightmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the rightmost characters from. + An expression that evaluates to a string. + * len - The number of characters to take from the right. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2484,6 +2648,13 @@ case class Right(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, len) - Returns the leftmost `len`(`len` can be string type) characters from the string `str`,if `len` is less or equal than 0 the result is an empty string.", + arguments = """ + Arguments: + * str - The string to take the leftmost characters from. + An expression that evaluates to a string or binary. + * len - The number of characters to take from the left. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 3); @@ -2523,6 +2694,11 @@ case class Left(str: Expression, len: Expression) extends RuntimeReplaceable // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the character length of string data or number of bytes of binary data. The length of string data includes the trailing spaces. The length of binary data includes binary zeros.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL '); @@ -2570,6 +2746,11 @@ case class Length(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the bit length of string data or number of bits of binary data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the bit length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2615,6 +2796,11 @@ case class BitLength(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the byte length of string data or number of bytes of binary " + "data.", + arguments = """ + Arguments: + * expr - The string or binary value to measure the byte length of. + An expression that evaluates to a string or binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -2662,6 +2848,15 @@ case class OctetLength(child: Expression) @ExpressionDescription( usage = """ _FUNC_(str1, str2[, threshold]) - Returns the Levenshtein distance between the two given strings. If threshold is set and distance more than it, return -1.""", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + * threshold - The maximum distance to compute; if exceeded, -1 is returned. + An expression that evaluates to an integer. + """, examples = """ Examples: > SELECT _FUNC_('kitten', 'sitting'); @@ -2817,6 +3012,13 @@ case class Levenshtein( // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str1, str2) - Returns the Jaro-Winkler similarity between the two given strings. The result is a double between 0 and 1, where 1 means identical.", + arguments = """ + Arguments: + * str1 - The first string to compare. + An expression that evaluates to a string. + * str2 - The second string to compare. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('MARTHA', 'MARHTA'); @@ -2855,6 +3057,11 @@ case class JaroWinkler(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns Soundex code of the string.", + arguments = """ + Arguments: + * str - The string to compute the Soundex code of. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Miller'); @@ -2886,6 +3093,11 @@ case class SoundEx(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns the numeric value of the first character of `str`.", + arguments = """ + Arguments: + * str - The string whose first character's numeric value is returned. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('222'); @@ -2919,6 +3131,11 @@ case class Ascii(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr) - Returns the ASCII character having the binary equivalent to `expr`. If n is larger than 256 the result is equivalent to chr(n % 256)", + arguments = """ + Arguments: + * expr - The number to convert to its ASCII character. + An expression that evaluates to a long. + """, examples = """ Examples: > SELECT _FUNC_(65); @@ -2952,6 +3169,11 @@ case class Chr(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(bin) - Converts the argument from a binary `bin` to a base 64 string.", + arguments = """ + Arguments: + * bin - The binary value to encode as a base 64 string. + An expression that evaluates to a binary. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL'); @@ -3007,6 +3229,11 @@ object Base64 { */ @ExpressionDescription( usage = "_FUNC_(str) - Converts the argument from a base 64 string `str` to a binary.", + arguments = """ + Arguments: + * str - The base 64 string to decode to binary. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('U3BhcmsgU1FM'); @@ -3156,7 +3383,9 @@ object Decode { arguments = """ Arguments: * bin - a binary expression to decode + An expression that evaluates to a binary. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to decode `bin` into a STRING. It is case insensitive. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3252,7 +3481,9 @@ object StringDecode { arguments = """ Arguments: * str - a string expression + An expression that evaluates to a string. * charset - one of the charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32' to encode `str` into a BINARY. It is case insensitive. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3332,6 +3563,13 @@ object Encode { By default, the binary format for conversion is "hex" if `fmt` is omitted. The function returns NULL if at least one of the input parameters is NULL. """, + arguments = """ + Arguments: + * str - The string to convert to a binary value. + An expression that evaluates to a string. + * fmt - The format describing how to interpret the string (e.g. "hex", "utf-8", "base64"). + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('abc', 'utf-8'); @@ -3445,6 +3683,13 @@ case class ToBinary( `expr2` also accept a user specified format. This is supposed to function like MySQL's FORMAT. """, + arguments = """ + Arguments: + * expr1 - The number to format. + An expression that evaluates to a numeric. + * expr2 - The number of decimal places to round to, or a format string. + An expression that evaluates to an integer or string. + """, examples = """ Examples: > SELECT _FUNC_(12332.123456, 4); @@ -3611,10 +3856,13 @@ case class FormatNumber(x: Expression, d: Expression) arguments = """ Arguments: * str - A STRING expression to be parsed. + An expression that evaluates to a string. * lang - An optional STRING expression with a language code from ISO 639 Alpha-2 (e.g. 'DE'), Alpha-3, or a language subtag of up to 8 characters. + An expression that evaluates to a string. * country - An optional STRING expression with a country code from ISO 3166 alpha-2 country code or a UN M.49 numeric-3 area code. + An expression that evaluates to a string. """, examples = """ Examples: @@ -3717,6 +3965,15 @@ case class StringSplitSQL( throws an error. If `partNum` is negative, the parts are counted backward from the end of the string. If the `delimiter` is an empty string, the `str` is not split. """, + arguments = """ + Arguments: + * str - The string to split. + An expression that evaluates to a string. + * delimiter - The string used to split the input into parts. + An expression that evaluates to a string. + * partNum - The 1-based index of the split part to return. + An expression that evaluates to an integer. + """, examples = """ Examples: @@ -3781,6 +4038,11 @@ case class Empty2Null(child: Expression) extends UnaryExpression with String2Str This checksum function is widely applied on credit card numbers and government identification numbers to distinguish valid numbers from mistyped, incorrect numbers. """, + arguments = """ + Arguments: + * str - The string of digits to validate against the Luhn algorithm. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('8112189876'); @@ -3818,6 +4080,11 @@ case class Luhncheck(input: Expression) extends RuntimeReplaceable with Implicit // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str) - Returns `str` enclosed by single quotes and each instance of single quote in it is preceded by a backslash.", + arguments = """ + Arguments: + * str - The string to enclose in single quotes and escape. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('Don\'t'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 1c9d6b335e8d9..827cfbda957d8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -61,8 +61,10 @@ trait TimeExpression extends Expression { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -188,8 +190,10 @@ private[expressions] object NanosTimestampCast { arguments = """ Arguments: * str - A string to be parsed to time. + An expression that evaluates to a string. * format - Time format pattern to follow. See Datetime Patterns for valid time format patterns. + An expression that evaluates to a string. """, examples = """ Examples: @@ -263,6 +267,11 @@ case class MinutesOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the minute of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the minute component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -330,6 +339,11 @@ case class HoursOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the hour of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the hour component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -422,6 +436,11 @@ case class SecondsOfTime(child: Expression) sub-microsecond digits. If `expr` is a TIME type (since 4.1.0), it returns the second of the time-of-day. """, + arguments = """ + Arguments: + * expr - The expression to extract the second component from. + An expression that evaluates to a timestamp or time. + """, examples = """ Examples: > SELECT _FUNC_('2018-02-14 12:58:59'); @@ -583,8 +602,11 @@ case class CurrentTime( arguments = """ Arguments: * hour - the hour to represent, from 0 to 23 + An expression that evaluates to an integer. * minute - the minute to represent, from 0 to 59 + An expression that evaluates to an integer. * second - the second to represent, from 0 to 59.999999 + An expression that evaluates to a decimal. """, examples = """ Examples: @@ -706,8 +728,11 @@ case class SubtractTimes(left: Expression, right: Expression) - "SECOND" - "MILLISECOND" - "MICROSECOND" + An expression that evaluates to a string. * start - a starting TIME expression + An expression that evaluates to a time. * end - an ending TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -770,7 +795,9 @@ case class TimeDiff( - "SECOND" - zero out the fraction part of seconds - "MILLISECOND" - zero out the microseconds - "MICROSECOND" - zero out the nanoseconds + An expression that evaluates to a string. * time - a TIME expression + An expression that evaluates to a time. """, examples = """ Examples: @@ -834,6 +861,7 @@ abstract class TimeFromBase extends UnaryExpression with RuntimeReplaceable with Arguments: * seconds - seconds since midnight (0 to 86399.999999). Supports decimals for fractional seconds. + An expression that evaluates to a numeric. """, examples = """ Examples: @@ -864,6 +892,7 @@ case class TimeFromSeconds(child: Expression) extends TimeFromBase { arguments = """ Arguments: * millis - milliseconds since midnight (0 to 86399999) + An expression that evaluates to an integral. """, examples = """ Examples: @@ -893,6 +922,7 @@ case class TimeFromMillis(child: Expression) extends TimeFromBase { arguments = """ Arguments: * micros - microseconds since midnight (0 to 86399999999) + An expression that evaluates to an integral. """, examples = """ Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala index b51cb74a5a8fa..b0bd67701be85 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala @@ -38,7 +38,8 @@ import org.apache.spark.unsafe.types.UTF8String """, arguments = """ Arguments: - str - a string expression to be translated + * str - a string expression to be translated + An expression that evaluates to a string. """, examples = """ Examples: @@ -80,6 +81,7 @@ case class UrlEncode(child: Expression) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -123,6 +125,7 @@ case class UrlDecode(child: Expression, failOnError: Boolean = true) arguments = """ Arguments: * str - a string expression to decode + An expression that evaluates to a string. """, examples = """ Examples: @@ -165,6 +168,15 @@ object UrlCodec { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - This is a special version of `parse_url` that performs the same operation, but returns a NULL value instead of raising an error if the parsing cannot be performed.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); @@ -197,6 +209,15 @@ case class TryParseUrl(params: Seq[Expression], replacement: Expression) */ @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.", + arguments = """ + Arguments: + * url - The URL to extract a part from. + An expression that evaluates to a string. + * partToExtract - The part of the URL to extract. + An expression that evaluates to a string. + * key - The key to extract when returning a query parameter value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index 0f8995ce829fe..d06b7a22bd2e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -576,6 +576,11 @@ abstract class ParseJsonExpressionBuilderBase(failOnError: Boolean) extends Expr // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Throw an exception when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); @@ -590,6 +595,11 @@ object ParseJsonExpressionBuilder extends ParseJsonExpressionBuilderBase(true) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(jsonStr) - Parse a JSON string as a Variant value. Return NULL when the string is not valid JSON value.", + arguments = """ + Arguments: + * jsonStr - The JSON string to parse as a Variant value. + An expression that evaluates to a string. + """, examples = """ Examples: > SELECT _FUNC_('{"a":1,"b":0.8}'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 800b38ea32223..0e1cafbb24c01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -78,6 +78,13 @@ abstract class XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('1','a/b'); @@ -99,6 +106,13 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -121,6 +135,13 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -143,6 +164,13 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -165,6 +193,13 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -187,6 +222,13 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression to evaluate against the XML. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -210,6 +252,13 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the node whose text is returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('bcc','a/c'); @@ -233,6 +282,13 @@ case class XPathString(xml: Expression, path: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", + arguments = """ + Arguments: + * xml - The XML string to evaluate. + An expression that evaluates to a string. + * xpath - The XPath expression matching the nodes whose values are returned. + An expression that evaluates to a string. Must be a constant. + """, examples = """ Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 92e9dc1a2e59d..c9479152d2511 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1204,6 +1204,12 @@ abstract class DDLSuite extends QueryTest with DDLSuiteBase { Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") :: Row( """Extended Usage: + | Arguments: + | * expr1 - The first operand of the bitwise exclusive OR. + | An expression that evaluates to an integral. + | * expr2 - The second operand of the bitwise exclusive OR. + | An expression that evaluates to an integral. + | | Examples: | > SELECT 3 ^ 5; | 6