From 7da0f441e68c175a85bfc610edf7a2ff9b1e7387 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Tue, 1 Sep 2026 13:45:17 +0000 Subject: [PATCH 1/3] Fix variables in mixed SQLPage expressions --- CHANGELOG.md | 1 + src/webserver/database/sql.rs | 59 +++- src/webserver/database/sql/rewrite.rs | 332 ++++++++++-------- .../set_variable_lost_in_mixed_expression.sql | 3 + 4 files changed, 255 insertions(+), 140 deletions(-) create mode 100644 tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c67a750f..d728c7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - `sqlpage.fetch_with_meta` now correctly documents server JSON responses sent under `json_body`, not `body`. - Datagrid rows with an icon or image no longer display an unnecessary en-dash placeholder, and an explicitly empty description remains empty. - Tooltip title text is now inhertis the same colour as the tooltip text. + - Fixed `SET` and request variables becoming `NULL` when used after a database expression and a `sqlpage.*` function inside `concat`, `coalesce`, or a JSON constructor. - Charts can display reference lines. A row with a `yline` is drawn as a line across the chart at that value of the y axis, and a row with an `xline` marks a position on the x axis. Adding `yline_end` or `xline_end` makes a line a band, and the row's `label` and `color` set its text and its color. Reference lines are rows, so a chart can have as many of them as the query returns. Each one follows its own axis, so on a `horizontal` bar chart a `yline` is drawn down the chart rather than across it. They are not added to the total of a `stacked` chart, and are not filled in an `area` chart. - Chart data rows can set their own `color`, painting a single bar, slice or point instead of the whole series. It applies to `bar`, `column`, `rangeBar`, `pie`, `treemap`, `scatter` and `bubble` charts, and to the markers of a `line` or an `area` chart. - Updated the bundled Tabler icon sprite from v3.44.0 to [v3.46.0](https://github.com/tabler/tabler-icons/releases/tag/v3.46.0). This adds 18 icons, including `play-bug`, `remote-control`, `rocking-chair`, `run-sprint`, `tabs`, `treasure-chest`, `vault`, and `yarn`, and includes the upstream fixes listed in the release notes. diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index ef550c17..73f62c3f 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -551,6 +551,35 @@ mod tests { ); } + #[test] + fn numbered_bindings_keep_source_projection_order() { + let query = rewrite_database( + "select $a as a, sqlpage.url_encode(upper(col || sqlpage.url_encode($b))) as b, $c as c from t", + ); + assert_eq!( + query.bindings.as_ref(), + [ + variable("a"), + call(SqlPageFunctionName::url_encode, [variable("b")]), + variable("c"), + ] + ); + } + + #[test] + fn numbered_bindings_keep_source_argument_order() { + let query = rewrite_database( + "select coalesce(upper(sqlpage.url_encode($a)), sqlpage.url_encode(upper(sqlpage.url_encode($b)))) from t", + ); + assert_eq!( + query.bindings.as_ref(), + [ + call(SqlPageFunctionName::url_encode, [variable("a")]), + call(SqlPageFunctionName::url_encode, [variable("b")]), + ] + ); + } + #[test] fn database_cannot_order_by_computed_column() { let FileStatement::Error(error) = @@ -739,14 +768,14 @@ mod tests { "select coalesce(upper(sqlpage.url_encode($prefix)), sqlpage.url_encode(value)) as result from t" ), DatabaseQuery { - sql: "SELECT value AS \"__sqlpage_input_0\", upper($1) AS \"__sqlpage_input_1\" FROM t".into(), + sql: "SELECT upper($1) AS \"__sqlpage_input_0\", value AS \"__sqlpage_input_1\" FROM t".into(), bindings: Box::new([call(SqlPageFunctionName::url_encode, [variable("prefix")])]), row_input_json: Box::new([false, false]), computed_columns: Box::new([OutputColumn { name: "result".into(), value: coalesce([ - row(1), - call(SqlPageFunctionName::url_encode, [row(0)]), + row(0), + call(SqlPageFunctionName::url_encode, [row(1)]), ]), }]), json_columns: Box::new([]), @@ -754,6 +783,30 @@ mod tests { ); } + #[test] + fn database_fragment_promoted_to_row_input_keeps_source_variables() { + assert_eq!( + rewrite_database("select concat(1 + 1, sqlpage.request_method(), $x) as result from t"), + DatabaseQuery { + sql: "SELECT 1 + 1 AS \"__sqlpage_input_0\" FROM t".into(), + bindings: Box::new([]), + row_input_json: Box::new([false]), + computed_columns: Box::new([OutputColumn { + name: "result".into(), + value: SqlPageExpr::Concat { + arguments: Box::new([ + row(0), + call(SqlPageFunctionName::request_method, []), + variable("x"), + ]), + null_behavior: ConcatNullBehavior::IgnoreNull, + }, + }]), + json_columns: Box::new([]), + } + ); + } + #[test] fn predicate_call_is_standalone_while_projection_call_is_per_row() { assert_eq!( diff --git a/src/webserver/database/sql/rewrite.rs b/src/webserver/database/sql/rewrite.rs index 819de120..33d7bb74 100644 --- a/src/webserver/database/sql/rewrite.rs +++ b/src/webserver/database/sql/rewrite.rs @@ -45,12 +45,17 @@ use sqlx::any::AnyKind; const SQLPAGE_INPUT_PREFIX: &str = "__sqlpage_input_"; -/// Mutable state used while rewriting one database query. -struct QueryRewriter<'a> { +/// State used while partitioning source expressions between `SQLPage` and the database. +struct QueryPartitioner<'a> { database: &'a DbInfo, - bindings: Vec, row_input_json: Vec, private_projection: Vec, +} + +/// State used after partitioning to lower `SQLPage` values into database bindings. +struct DatabaseLowerer<'a> { + database: &'a DbInfo, + bindings: Vec, error: Option, } @@ -80,7 +85,14 @@ impl sqlparser::ast::Visitor for ComputedAliasFinder<'_> { // Keeping the owned AST inline avoids one heap allocation for every ordinary // projected expression. The enum is short-lived inside the rewriter. #[allow(clippy::large_enum_variant)] -enum RewrittenProjection { +enum PartitionedProjection { + Database(SqlExpr), + PerRow(RowExpr), +} + +/// A partitioned projection after its database fragments have been lowered. +#[allow(clippy::large_enum_variant)] +enum LoweredProjection { Database(SqlExpr), PerRow(RowExpr), } @@ -91,7 +103,8 @@ trait ExprEnvironment { type Input; fn use_database_expr( - rewriter: &mut QueryRewriter<'_>, + &mut self, + database: &DbInfo, expression: SqlExpr, ) -> anyhow::Result>; } @@ -99,13 +112,17 @@ trait ExprEnvironment { /// Rejects database-owned inputs because no returned row is available. struct StandaloneEnvironment; /// Projects database-owned inputs into the current returned row. -struct RowEnvironment; +struct RowEnvironment<'a> { + row_input_json: &'a mut Vec, + private_projection: &'a mut Vec, +} impl ExprEnvironment for StandaloneEnvironment { type Input = NoRowInput; fn use_database_expr( - _rewriter: &mut QueryRewriter<'_>, + &mut self, + _database: &DbInfo, expression: SqlExpr, ) -> anyhow::Result { if let SqlExpr::Function(function) = &expression @@ -122,15 +139,23 @@ impl ExprEnvironment for StandaloneEnvironment { } } -impl ExprEnvironment for RowEnvironment { +impl ExprEnvironment for RowEnvironment<'_> { type Input = RowInputId; fn use_database_expr( - rewriter: &mut QueryRewriter<'_>, + &mut self, + _database: &DbInfo, expression: SqlExpr, ) -> anyhow::Result { - let id = rewriter.add_row_input(expression)?; - Ok(SqlPageExpr::Input(id)) + let decode_as_json = is_json_expression(&expression); + let index = self.row_input_json.len(); + let name = format!("{SQLPAGE_INPUT_PREFIX}{index}"); + self.private_projection.push(SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::with_quote('"', name), + }); + self.row_input_json.push(decode_as_json); + Ok(SqlPageExpr::Input(RowInputId::new(index))) } } @@ -152,25 +177,20 @@ pub(super) fn rewrite_query( semicolon: bool, ) -> anyhow::Result { let source_span = source_span(&statement); - let mut rewriter = QueryRewriter { + let mut partitioner = QueryPartitioner { database, - bindings: Vec::new(), row_input_json: Vec::new(), private_projection: Vec::new(), - error: None, }; - if let Some(single_row) = rewrite_single_row(&mut statement, &mut rewriter)? { + if let Some(single_row) = rewrite_single_row(&mut statement, &mut partitioner)? { return Ok(Query { body: QueryBody::SingleRow(single_row), source_span, }); } - let computed_columns = rewrite_top_level_projection(&mut statement, &mut rewriter)?; - - let _ = statement.visit(&mut rewriter); - if let Some(error) = rewriter.error { - return Err(error); - } + let mut lowerer = DatabaseLowerer::new(database); + let computed_columns = + rewrite_top_level_projection(&mut statement, &mut partitioner, &mut lowerer)?; if let SqlStatement::Query(query) = &mut statement && let SetExpr::Select(select) = query.body.as_mut() @@ -180,14 +200,15 @@ pub(super) fn rewrite_query( expr: SqlExpr::value(Value::Null), alias: Ident::with_quote('"', format!("{SQLPAGE_INPUT_PREFIX}anchor")), }); - rewriter.row_input_json.push(false); + partitioner.row_input_json.push(false); } let json_columns = extract_json_columns(&statement, database.database_type) .into_iter() .filter(|name| !name.starts_with(SQLPAGE_INPUT_PREFIX)) .collect(); - let bindings = rewriter.finish_bindings(&mut statement)?; + lowerer.lower_statement(&mut statement)?; + let bindings = lowerer.finish_bindings(&mut statement)?; let sql = format!( "{statement}{semicolon}", semicolon = if semicolon { ";" } else { "" } @@ -197,7 +218,7 @@ pub(super) fn rewrite_query( body: QueryBody::Database(DatabaseQuery { sql, bindings, - row_input_json: rewriter.row_input_json.into_boxed_slice(), + row_input_json: partitioner.row_input_json.into_boxed_slice(), computed_columns: computed_columns.into_boxed_slice(), json_columns, }), @@ -209,7 +230,8 @@ pub(super) fn rewrite_query( /// and appends their private database inputs as a trailing suffix. fn rewrite_top_level_projection( statement: &mut SqlStatement, - rewriter: &mut QueryRewriter<'_>, + partitioner: &mut QueryPartitioner<'_>, + lowerer: &mut DatabaseLowerer<'_>, ) -> anyhow::Result>> { let SqlStatement::Query(query) = statement else { return Ok(Vec::new()); @@ -224,8 +246,8 @@ fn rewrite_top_level_projection( } let (mut database_projection, computed_columns) = - rewrite_projection_items(std::mem::take(&mut select.projection), rewriter)?; - database_projection.append(&mut rewriter.private_projection); + rewrite_projection_items(std::mem::take(&mut select.projection), partitioner, lowerer)?; + database_projection.append(&mut partitioner.private_projection); select.projection = database_projection; reject_computed_alias_references("WHERE", select.selection.as_ref(), &computed_columns)?; @@ -245,7 +267,8 @@ fn rewrite_top_level_projection( fn rewrite_projection_items( projection: Vec, - rewriter: &mut QueryRewriter<'_>, + partitioner: &mut QueryPartitioner<'_>, + lowerer: &mut DatabaseLowerer<'_>, ) -> anyhow::Result<(Vec, Vec>)> { let mut database_projection = Vec::with_capacity(projection.len()); let mut computed_columns = Vec::new(); @@ -261,8 +284,13 @@ fn rewrite_projection_items( let name = alias .as_ref() .map_or_else(|| expression.to_string(), |alias| alias.value.clone()); - match rewriter.rewrite_projection(expression)? { - RewrittenProjection::Database(expression) => { + let first_private_input = partitioner.private_projection.len(); + let projection = partitioner.rewrite_projection(expression)?; + match lowerer.lower_projection( + projection, + &mut partitioner.private_projection[first_private_input..], + )? { + LoweredProjection::Database(expression) => { database_projection.push(match alias { Some(alias) => SelectItem::ExprWithAlias { expr: expression, @@ -271,7 +299,7 @@ fn rewrite_projection_items( None => SelectItem::UnnamedExpr(expression), }); } - RewrittenProjection::PerRow(value) => { + LoweredProjection::PerRow(value) => { computed_columns.push(OutputColumn { name, value }); } } @@ -384,7 +412,7 @@ fn references_computed_projection( /// avoiding both a database round trip and an intermediate row-expression tree. fn rewrite_single_row( statement: &mut SqlStatement, - rewriter: &mut QueryRewriter<'_>, + partitioner: &mut QueryPartitioner<'_>, ) -> anyhow::Result> { if !has_single_row_shape(statement) { return Ok(None); @@ -411,7 +439,7 @@ fn rewrite_single_row( }; columns.push(OutputColumn { name: alias.value, - value: build_sqlpage_expr::(rewriter, expr)?, + value: build_sqlpage_expr(partitioner.database, &mut StandaloneEnvironment, expr)?, }); } Ok(Some(SingleRowQuery { @@ -501,22 +529,23 @@ fn can_build_standalone(expression: &SqlExpr) -> anyhow::Result { } } -impl QueryRewriter<'_> { +impl QueryPartitioner<'_> { /// Splits a projected expression at SQLPage-supported operations while /// leaving opaque database operations in the SQL AST. - fn rewrite_projection(&mut self, expression: SqlExpr) -> anyhow::Result { + fn rewrite_projection(&mut self, expression: SqlExpr) -> anyhow::Result { + if !projection_is_per_row(&expression)? { + return Ok(PartitionedProjection::Database(expression)); + } match expression { SqlExpr::Function(function) => { if recognize_sqlpage_function(&function)?.is_some() { - return build_sqlpage_expr::(self, SqlExpr::Function(function)) - .map(RewrittenProjection::PerRow); - } - if let Some(kind) = emulated_function(&function) { - return self.rewrite_emulated_projection(function, kind); + return self + .build_row_expr(SqlExpr::Function(function)) + .map(PartitionedProjection::PerRow); } - let mut expression = SqlExpr::Function(function); - self.rewrite_database_expression(&mut expression)?; - Ok(RewrittenProjection::Database(expression)) + let kind = emulated_function(&function) + .expect("per-row function ownership was already classified"); + self.rewrite_emulated_projection(function, kind) } SqlExpr::BinaryOp { left, @@ -524,37 +553,23 @@ impl QueryRewriter<'_> { right, } => { let left = self.rewrite_projection(*left)?; + let left = self.projection_into_row_expr(left)?; let right = self.rewrite_projection(*right)?; - match (left, right) { - (RewrittenProjection::Database(left), RewrittenProjection::Database(right)) => { - Ok(RewrittenProjection::Database(SqlExpr::BinaryOp { - left: Box::new(left), - op: BinaryOperator::StringConcat, - right: Box::new(right), - })) - } - (left, right) => Ok(RewrittenProjection::PerRow(SqlPageExpr::Concat { - arguments: vec![ - self.projection_into_row_expr(left)?, - self.projection_into_row_expr(right)?, - ] - .into_boxed_slice(), - null_behavior: self.database.database_type.concat_operator_null_behavior(), - })), - } + let right = self.projection_into_row_expr(right)?; + Ok(PartitionedProjection::PerRow(SqlPageExpr::Concat { + arguments: vec![left, right].into_boxed_slice(), + null_behavior: self.database.database_type.concat_operator_null_behavior(), + })) } SqlExpr::Nested(expression) => match self.rewrite_projection(*expression)? { - RewrittenProjection::Database(expression) => Ok(RewrittenProjection::Database( + PartitionedProjection::Database(expression) => Ok(PartitionedProjection::Database( SqlExpr::Nested(Box::new(expression)), )), - RewrittenProjection::PerRow(expression) => { - Ok(RewrittenProjection::PerRow(expression)) + PartitionedProjection::PerRow(expression) => { + Ok(PartitionedProjection::PerRow(expression)) } }, - mut expression => { - self.rewrite_database_expression(&mut expression)?; - Ok(RewrittenProjection::Database(expression)) - } + _ => unreachable!("per-row expression ownership was already classified"), } } @@ -562,35 +577,16 @@ impl QueryRewriter<'_> { &mut self, function: Function, kind: EmulatedFunction, - ) -> anyhow::Result { - let (arguments, original) = take_expression_arguments(function)?; + ) -> anyhow::Result { + let (arguments, _) = take_expression_arguments(function)?; let mut rewritten = Vec::with_capacity(arguments.len()); - let mut has_per_row = false; for argument in arguments { let argument = self.rewrite_projection(argument)?; - has_per_row |= matches!(argument, RewrittenProjection::PerRow(_)); - rewritten.push(argument); - } - if !has_per_row { - let arguments = rewritten - .into_iter() - .map(|argument| match argument { - RewrittenProjection::Database(expression) => expression, - RewrittenProjection::PerRow(_) => unreachable!(), - }) - .collect(); - return Ok(RewrittenProjection::Database(rebuild_function( - original, arguments, - ))); + rewritten.push(self.projection_into_row_expr(argument)?); } - - let arguments = rewritten - .into_iter() - .map(|argument| self.projection_into_row_expr(argument)) - .collect::>>()?; - Ok(RewrittenProjection::PerRow(build_emulated( + Ok(PartitionedProjection::PerRow(build_emulated( kind, - arguments, + rewritten, self.database.database_type, )?)) } @@ -599,18 +595,106 @@ impl QueryRewriter<'_> { /// while preserving an already per-row expression unchanged. fn projection_into_row_expr( &mut self, - projection: RewrittenProjection, + projection: PartitionedProjection, ) -> anyhow::Result { match projection { - RewrittenProjection::Database(expression) => { - build_sqlpage_expr::(self, expression) + PartitionedProjection::Database(expression) => self.build_row_expr(expression), + PartitionedProjection::PerRow(expression) => Ok(expression), + } + } + + fn build_row_expr(&mut self, expression: SqlExpr) -> anyhow::Result { + build_sqlpage_expr( + self.database, + &mut RowEnvironment { + row_input_json: &mut self.row_input_json, + private_projection: &mut self.private_projection, + }, + expression, + ) + } +} + +/// Determines ownership without mutating the AST, so database fragments can +/// be promoted to row inputs in source order once a per-row parent is known. +fn projection_is_per_row(expression: &SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Function(function) => { + if recognize_sqlpage_function(function)?.is_some() { + return Ok(true); + } + if emulated_function(function).is_none() { + return Ok(false); } - RewrittenProjection::PerRow(expression) => Ok(expression), + let FunctionArguments::List(arguments) = &function.args else { + anyhow::bail!("Unsupported arguments to {}", function.name); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + anyhow::bail!("Unsupported arguments to {}", function.name); + } + for argument in &arguments.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { + anyhow::bail!("Named and wildcard function arguments are not supported"); + }; + if projection_is_per_row(expression)? { + return Ok(true); + } + } + Ok(false) } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(projection_is_per_row(left)? || projection_is_per_row(right)?), + SqlExpr::Nested(expression) => projection_is_per_row(expression), + _ => Ok(false), + } +} + +impl<'a> DatabaseLowerer<'a> { + fn new(database: &'a DbInfo) -> Self { + Self { + database, + bindings: Vec::new(), + error: None, + } + } + + /// Accepts a terminal partition result and lowers its database-owned + /// fragments. The lowered AST cannot return to recursive partitioning. + fn lower_projection( + &mut self, + projection: PartitionedProjection, + private_inputs: &mut [SelectItem], + ) -> anyhow::Result { + let projection = match projection { + PartitionedProjection::Database(mut expression) => { + self.lower_expression(&mut expression)?; + LoweredProjection::Database(expression) + } + PartitionedProjection::PerRow(expression) => { + for input in private_inputs { + let _ = input.visit(self); + self.take_error()?; + } + LoweredProjection::PerRow(expression) + } + }; + Ok(projection) } - fn rewrite_database_expression(&mut self, expression: &mut SqlExpr) -> anyhow::Result<()> { + fn lower_expression(&mut self, expression: &mut SqlExpr) -> anyhow::Result<()> { let _ = expression.visit(self); + self.take_error() + } + + fn lower_statement(&mut self, statement: &mut SqlStatement) -> anyhow::Result<()> { + let _ = statement.visit(self); + self.take_error() + } + + fn take_error(&mut self) -> anyhow::Result<()> { self.error.take().map_or(Ok(()), Err) } @@ -624,19 +708,6 @@ impl QueryRewriter<'_> { cast_placeholder(placeholder, self.database) } - fn add_row_input(&mut self, mut expression: SqlExpr) -> anyhow::Result { - let decode_as_json = is_json_expression(&expression); - self.rewrite_database_expression(&mut expression)?; - let index = self.row_input_json.len(); - let name = format!("{SQLPAGE_INPUT_PREFIX}{index}"); - self.private_projection.push(SelectItem::ExprWithAlias { - expr: expression, - alias: Ident::with_quote('"', name), - }); - self.row_input_json.push(decode_as_json); - Ok(RowInputId::new(index)) - } - fn finish_bindings( &mut self, statement: &mut SqlStatement, @@ -708,7 +779,7 @@ impl VisitorMut for PositionalBindingFinalizer { } } -impl VisitorMut for QueryRewriter<'_> { +impl VisitorMut for DatabaseLowerer<'_> { type Break = (); fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { @@ -730,7 +801,7 @@ impl VisitorMut for QueryRewriter<'_> { SqlExpr::Function(function) => match recognize_sqlpage_function(function) { Ok(Some(_)) => { let owned = std::mem::replace(expression, SqlExpr::value(Value::Null)); - match build_sqlpage_expr::(self, owned) { + match build_sqlpage_expr(self.database, &mut StandaloneEnvironment, owned) { Ok(value) => Some(self.add_binding(value)), Err(error) => { self.error = Some(error.context( @@ -783,7 +854,8 @@ impl VisitorMut for QueryRewriter<'_> { /// environment determines whether opaque database fragments are illegal or /// become private row inputs. fn build_sqlpage_expr( - rewriter: &mut QueryRewriter<'_>, + database: &DbInfo, + environment: &mut Environment, expression: SqlExpr, ) -> anyhow::Result> { match expression { @@ -796,14 +868,14 @@ fn build_sqlpage_expr( Value::Boolean(value) => Ok(SqlPageExpr::Literal(JsonValue::Bool(value))), Value::Null => Ok(SqlPageExpr::Literal(JsonValue::Null)), _ => { - Environment::use_database_expr(rewriter, SqlExpr::Value(ValueWithSpan::from(value))) + environment.use_database_expr(database, SqlExpr::Value(ValueWithSpan::from(value))) } }, SqlExpr::Identifier(identifier) => { if let Some(variable) = variable_from_ident(&identifier) { Ok(SqlPageExpr::Variable(variable)) } else { - Environment::use_database_expr(rewriter, SqlExpr::Identifier(identifier)) + environment.use_database_expr(database, SqlExpr::Identifier(identifier)) } } SqlExpr::Function(function) => { @@ -811,7 +883,7 @@ fn build_sqlpage_expr( let (arguments, _) = take_expression_arguments(function)?; let arguments = arguments .into_iter() - .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .map(|argument| build_sqlpage_expr(database, environment, argument)) .collect::>>()?; Ok(SqlPageExpr::Call { function: function_name, @@ -821,11 +893,11 @@ fn build_sqlpage_expr( let (arguments, _) = take_expression_arguments(function)?; let arguments = arguments .into_iter() - .map(|argument| build_sqlpage_expr::(rewriter, argument)) + .map(|argument| build_sqlpage_expr(database, environment, argument)) .collect::>>()?; - build_emulated(kind, arguments, rewriter.database.database_type) + build_emulated(kind, arguments, database.database_type) } else { - Environment::use_database_expr(rewriter, SqlExpr::Function(function)) + environment.use_database_expr(database, SqlExpr::Function(function)) } } SqlExpr::BinaryOp { @@ -834,17 +906,14 @@ fn build_sqlpage_expr( right, } => Ok(SqlPageExpr::Concat { arguments: vec![ - build_sqlpage_expr::(rewriter, *left)?, - build_sqlpage_expr::(rewriter, *right)?, + build_sqlpage_expr(database, environment, *left)?, + build_sqlpage_expr(database, environment, *right)?, ] .into_boxed_slice(), - null_behavior: rewriter - .database - .database_type - .concat_operator_null_behavior(), + null_behavior: database.database_type.concat_operator_null_behavior(), }), - SqlExpr::Nested(expression) => build_sqlpage_expr::(rewriter, *expression), - expression => Environment::use_database_expr(rewriter, expression), + SqlExpr::Nested(expression) => build_sqlpage_expr(database, environment, *expression), + expression => environment.use_database_expr(database, expression), } } @@ -967,17 +1036,6 @@ fn take_expression_arguments(mut function: Function) -> anyhow::Result<(Vec) -> SqlExpr { - let FunctionArguments::List(arguments) = &mut function.args else { - unreachable!() - }; - arguments.args = expressions - .into_iter() - .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) - .collect(); - SqlExpr::Function(function) -} - fn make_function(name: &str, expressions: Vec) -> SqlExpr { SqlExpr::Function(Function { name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), diff --git a/tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql b/tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql new file mode 100644 index 00000000..b18564a3 --- /dev/null +++ b/tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql @@ -0,0 +1,3 @@ +set x = 'x'; +set y = concat(1 + 1, sqlpage.request_method(), $x); +select '2GETx' as expected, $y as actual; From a3c5467cd90298a2ea1b781147b7af0842805efa Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Tue, 1 Sep 2026 15:11:28 +0000 Subject: [PATCH 2/3] Skip numeric fixture assertion on Oracle --- ...ion.sql => set_variable_lost_in_mixed_expression_nooracle.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/sql_test_files/data/{set_variable_lost_in_mixed_expression.sql => set_variable_lost_in_mixed_expression_nooracle.sql} (100%) diff --git a/tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql b/tests/sql_test_files/data/set_variable_lost_in_mixed_expression_nooracle.sql similarity index 100% rename from tests/sql_test_files/data/set_variable_lost_in_mixed_expression.sql rename to tests/sql_test_files/data/set_variable_lost_in_mixed_expression_nooracle.sql From 7150d7afaf9d484e78a1f768405845dd8123c520 Mon Sep 17 00:00:00 2001 From: Ophir Lojkine Date: Tue, 1 Sep 2026 16:39:21 +0000 Subject: [PATCH 3/3] Refactor SQL rewrite pipeline --- CHANGELOG.md | 15 +- Cargo.lock | 2 +- Cargo.toml | 2 +- examples/official-site/extensions-to-sql.md | 12 +- src/webserver/database/execute_queries.rs | 18 +- src/webserver/database/sql.rs | 106 +- src/webserver/database/sql/rewrite.rs | 1177 ----------------- .../sql/rewrite/computed_column_validation.rs | 136 ++ .../database/sql/rewrite/database_lowering.rs | 233 ++++ src/webserver/database/sql/rewrite/mod.rs | 152 +++ .../sql/rewrite/projection_partitioning.rs | 232 ++++ .../sql/rewrite/sqlpage_expression.rs | 325 +++++ .../sql/rewrite/static_simple_select.rs | 86 ++ src/webserver/database/sql/statement.rs | 6 +- 14 files changed, 1221 insertions(+), 1281 deletions(-) delete mode 100644 src/webserver/database/sql/rewrite.rs create mode 100644 src/webserver/database/sql/rewrite/computed_column_validation.rs create mode 100644 src/webserver/database/sql/rewrite/database_lowering.rs create mode 100644 src/webserver/database/sql/rewrite/mod.rs create mode 100644 src/webserver/database/sql/rewrite/projection_partitioning.rs create mode 100644 src/webserver/database/sql/rewrite/sqlpage_expression.rs create mode 100644 src/webserver/database/sql/rewrite/static_simple_select.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d728c7d1..9ce5788f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # CHANGELOG.md +## v0.46.1 + +- Fixed a regression introduced in v0.46 that could replace a variable with `NULL` while building a value that also used database expressions and `sqlpage.*` functions. For example, this API request could lose `john.doe` and produce a URL ending at `https://api.example.com/`: + + ```sql + SET user_id = 'john.doe'; + SET api_request = json_object( + 'timeout_ms', CAST(sqlpage.environment_variable('API_TIMEOUT') AS INTEGER), + 'url', concat(sqlpage.environment_variable('API_URL'), '/', $user_id) + ); + ``` + + SQLPage now keeps the variable value, producing `https://api.example.com/john.doe` as expected. + ## v0.46 - Removed unnecessary `CAST` around request variables: @@ -24,7 +38,6 @@ - `sqlpage.fetch_with_meta` now correctly documents server JSON responses sent under `json_body`, not `body`. - Datagrid rows with an icon or image no longer display an unnecessary en-dash placeholder, and an explicitly empty description remains empty. - Tooltip title text is now inhertis the same colour as the tooltip text. - - Fixed `SET` and request variables becoming `NULL` when used after a database expression and a `sqlpage.*` function inside `concat`, `coalesce`, or a JSON constructor. - Charts can display reference lines. A row with a `yline` is drawn as a line across the chart at that value of the y axis, and a row with an `xline` marks a position on the x axis. Adding `yline_end` or `xline_end` makes a line a band, and the row's `label` and `color` set its text and its color. Reference lines are rows, so a chart can have as many of them as the query returns. Each one follows its own axis, so on a `horizontal` bar chart a `yline` is drawn down the chart rather than across it. They are not added to the total of a `stacked` chart, and are not filled in an `area` chart. - Chart data rows can set their own `color`, painting a single bar, slice or point instead of the whole series. It applies to `bar`, `column`, `rangeBar`, `pie`, `treemap`, `scatter` and `bubble` charts, and to the markers of a `line` or an `area` chart. - Updated the bundled Tabler icon sprite from v3.44.0 to [v3.46.0](https://github.com/tabler/tabler-icons/releases/tag/v3.46.0). This adds 18 icons, including `play-bug`, `remote-control`, `rocking-chair`, `run-sprint`, `tabs`, `treasure-chest`, `vault`, and `yarn`, and includes the upstream fixes listed in the release notes. diff --git a/Cargo.lock b/Cargo.lock index a0954474..4453f45c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4553,7 +4553,7 @@ dependencies = [ [[package]] name = "sqlpage" -version = "0.46.0" +version = "0.46.1" dependencies = [ "actix-http", "actix-multipart", diff --git a/Cargo.toml b/Cargo.toml index 0f474188..092efa2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sqlpage" -version = "0.46.0" +version = "0.46.1" edition = "2024" description = "Build data user interfaces entirely in SQL. A web server that takes .sql files and formats the query result using pre-made configurable professional-looking components." keywords = ["web", "sql", "framework"] diff --git a/examples/official-site/extensions-to-sql.md b/examples/official-site/extensions-to-sql.md index f49d71ad..7ef679bf 100644 --- a/examples/official-site/extensions-to-sql.md +++ b/examples/official-site/extensions-to-sql.md @@ -42,9 +42,11 @@ To be static and simple, a statement must satisfy all of the following: - No `FROM`, `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, `LIMIT`/`FETCH`, `WITH`, `DISTINCT`, `TOP`, windowing, locks, or other clauses. - Each selected item is of the form `value AS alias`. -- Each `value` is either: - - a literal (single-quoted string, number, boolean, or `NULL`), or - - a variable (like `$name`, `:message`) +- Each `value` can be evaluated without a database row. This includes: + - literals (single-quoted strings, numbers, booleans, or `NULL`), + - variables (like `$name` or `:message`), + - `sqlpage.*` functions whose arguments can also be evaluated without a database row, + - and combinations of those values using `||`, `concat`, `coalesce`, or JSON constructors. That’s it. If any part is more complex, it is not a static simple select and will be sent to the database. @@ -53,14 +55,12 @@ That’s it. If any part is more complex, it is not a static simple select and w ```sql SELECT 'text' AS component, 'Hello' AS contents; SELECT 'text' AS component, $name AS contents; +SELECT 'text' AS component, 'Hello ' || $name AS contents; ``` #### Examples that are NOT static (sent to the database) ```sql --- Has string concatenation -select 'from' as component, 'handle_form.sql?id=' || $id as action; - -- Has WHERE select 'text' as component, $alert_message as contents where $should_alert; diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs index 9154afeb..810cba2d 100644 --- a/src/webserver/database/execute_queries.rs +++ b/src/webserver/database/execute_queries.rs @@ -10,8 +10,8 @@ use tracing::Instrument; use super::csv_import::run_csv_import; use super::error_highlighting::{display_stmt_db_error, display_stmt_error, is_positioned_error}; use super::sql::{ - DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, - SqlFile, + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SourceSpan, SqlFile, + StaticSimpleSelect, }; use super::sqlpage_expr::{NoInputs, RowExpr, RowInputs}; use crate::dynamic_component::parse_dynamic_rows; @@ -201,8 +201,8 @@ pub fn stream_query_results_with_conn<'a>( run_csv_import(connection, csv_import, request).await.with_context(|| format!("Failed to import the CSV file {:?} into the table {:?}", csv_import.uploaded_file, csv_import.table_name))?; }, FileStatement::Query(statement) => match &statement.body { - QueryBody::SingleRow(query) => { - let row = execute_single_row(query, request, db_connection) + QueryBody::StaticSimpleSelect(query) => { + let row = execute_static_simple_select(query, request, db_connection) .await .map_err(|error| with_stmt_position(source_file, statement.source_span, error))?; for item in parse_dynamic_rows(DbItem::Row(row)) { @@ -330,8 +330,8 @@ pub fn stop_at_first_error( .take_until(error_rx) } -async fn execute_single_row( - query: &SingleRowQuery, +async fn execute_static_simple_select( + query: &StaticSimpleSelect, req: &ExecutionContext, db_connection: &mut DbConn, ) -> anyhow::Result { @@ -397,11 +397,11 @@ async fn execute_scalar_query<'a>( source_file: &Path, ) -> anyhow::Result> { let QueryBody::Database(database_query) = &statement.body else { - let QueryBody::SingleRow(single_row) = &statement.body else { + let QueryBody::StaticSimpleSelect(static_select) = &statement.body else { unreachable!() }; - ensure_scalar_column_count(single_row.columns.len())?; - let row = execute_single_row(single_row, request, db_connection).await?; + ensure_scalar_column_count(static_select.columns.len())?; + let row = execute_static_simple_select(static_select, request, db_connection).await?; return scalar_value_from_row(DbItem::Row(row)); }; let query = bind_query(database_query, request, db_connection).await?; diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index 73f62c3f..6c99b274 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -29,8 +29,10 @@ use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::Token::{self, EOF, SemiColon}; use sqlparser::tokenizer::{Location, Span, TokenWithSpan, Tokenizer}; +#[cfg(test)] +use super::SupportedDatabase; use super::csv_import::extract_csv_copy_statement; -use super::{Database, DbInfo, SupportedDatabase}; +use super::{Database, DbInfo}; use crate::AppState; use crate::file_cache::AsyncFromStrWithState; use crate::webserver::database::error_highlighting::quote_source_with_highlight; @@ -43,7 +45,7 @@ mod statement; pub(super) use statement::SourceLocation; pub use statement::SqlFile; pub(super) use statement::{ - DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SingleRowQuery, SourceSpan, + DatabaseQuery, FileStatement, OutputColumn, Query, QueryBody, SourceSpan, StaticSimpleSelect, VariableName, }; @@ -211,83 +213,6 @@ fn syntax_error(error: ParserError, parser: &Parser<'_>, sql: &str) -> FileState FileStatement::Error(anyhow::Error::from(error).context(message)) } -const SQLPAGE_FUNCTION_NAMESPACE: &str = "sqlpage"; - -pub(super) fn is_sqlpage_func(parts: &[ObjectNamePart]) -> bool { - matches!( - parts, - [ - ObjectNamePart::Identifier(Ident { - value, - quote_style: None, - .. - }), - ObjectNamePart::Identifier(Ident { quote_style: None, .. }) - ] if value.eq_ignore_ascii_case(SQLPAGE_FUNCTION_NAMESPACE) - ) -} - -pub(super) fn extract_json_columns( - statement: &Statement, - database: SupportedDatabase, -) -> Vec { - if matches!( - database, - SupportedDatabase::Postgres | SupportedDatabase::Mssql - ) { - return Vec::new(); - } - let Statement::Query(query) = statement else { - return Vec::new(); - }; - let SetExpr::Select(select) = query.body.as_ref() else { - return Vec::new(); - }; - select - .projection - .iter() - .filter_map(|item| match item { - SelectItem::ExprWithAlias { expr, alias } if is_json_expression(expr) => { - Some(alias.value.clone()) - } - _ => None, - }) - .collect() -} - -pub(super) fn is_json_expression(expression: &Expr) -> bool { - match expression { - Expr::Function(function) => { - let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { - return false; - }; - [ - "json_object", - "json_array", - "json_build_object", - "json_build_array", - "to_json", - "to_jsonb", - "json_agg", - "jsonb_agg", - "json_arrayagg", - "json_objectagg", - "json_group_array", - "json_group_object", - "json", - "jsonb", - ] - .iter() - .any(|candidate| name.value.eq_ignore_ascii_case(candidate)) - } - Expr::Cast { data_type, .. } => matches!( - data_type, - sqlparser::ast::DataType::JSON | sqlparser::ast::DataType::JSONB - ), - _ => false, - } -} - fn expression_to_query(expression: Expr) -> Statement { if let Expr::Subquery(query) = expression { return Statement::Query(query); @@ -697,9 +622,9 @@ mod tests { } #[test] - fn standalone_projection_has_no_database_query() { + fn static_simple_select_has_no_database_query() { let FileStatement::Query(Query { - body: QueryBody::SingleRow(query), + body: QueryBody::StaticSimpleSelect(query), .. }) = one("select sqlpage.url_encode('a b') as value") else { @@ -712,7 +637,7 @@ mod tests { fn concat_operator_uses_backend_null_behavior_in_sqlpage_expressions() { for database_type in [SupportedDatabase::Oracle, SupportedDatabase::Mssql] { let FileStatement::Query(Query { - body: QueryBody::SingleRow(query), + body: QueryBody::StaticSimpleSelect(query), .. }) = one_for(database_type, "select '/' || null as path") else { @@ -752,7 +677,7 @@ mod tests { #[test] fn unquoted_sqlpage_names_are_case_insensitive() { let FileStatement::Query(Query { - body: QueryBody::SingleRow(query), + body: QueryBody::StaticSimpleSelect(query), .. }) = one("select SQLPAGE.URL_ENCODE('a b') as value") else { @@ -807,6 +732,21 @@ mod tests { ); } + #[test] + fn private_row_input_json_flags_follow_row_input_ids() { + let query = rewrite_database( + "select concat(to_json(value), sqlpage.url_encode(other)) as result from t", + ); + assert_eq!(query.row_input_json.as_ref(), [true, false]); + let SqlPageExpr::Concat { arguments, .. } = &query.computed_columns[0].value else { + panic!("expected a concatenated per-row expression"); + }; + assert_eq!( + arguments.as_ref(), + [row(0), call(SqlPageFunctionName::url_encode, [row(1)])] + ); + } + #[test] fn predicate_call_is_standalone_while_projection_call_is_per_row() { assert_eq!( diff --git a/src/webserver/database/sql/rewrite.rs b/src/webserver/database/sql/rewrite.rs deleted file mode 100644 index 33d7bb74..00000000 --- a/src/webserver/database/sql/rewrite.rs +++ /dev/null @@ -1,1177 +0,0 @@ -//! Compiler pass that partitions parsed SQL between the database and `SQLPage`. -//! -//! `SQLPage` files contain expressions owned by two runtimes: the configured -//! database handles ordinary SQL, while variables and `sqlpage.*` functions are -//! evaluated by `SQLPage`. This module makes that boundary explicit. Values -//! needed before a database statement become typed bindings; computed -//! projections run once per returned row, with any database-owned dependencies -//! appended to the SQL projection as a private trailing column suffix. Queries -//! that require no database work become single-row `SQLPage` plans instead. The -//! distinct standalone and row expression types prevent pre-query work from -//! depending on a row that does not exist yet. -//! -//! Rewriting also renders backend-appropriate placeholders and casts, preserves -//! selected DBMS semantics for operations `SQLPage` emulates, records which -//! returned values need JSON decoding, and rejects grouping, ordering, or other -//! relational clauses that would incorrectly depend on post-database computed -//! columns. Its output is the immutable query representation consumed by -//! `execute_queries`; no request values are resolved and no functions are -//! executed here. - -use std::ops::ControlFlow; -use std::str::FromStr as _; - -use anyhow::{Context as _, anyhow}; -use serde_json::Value as JsonValue; -use sqlparser::ast::{ - BinaryOperator, CastKind, CharacterLength, DataType, Expr as SqlExpr, Function, FunctionArg, - FunctionArgExpr, FunctionArgumentList, FunctionArguments, GroupByExpr, Ident, ObjectName, - ObjectNamePart, OrderByKind, SelectItem, SetExpr, Statement as SqlStatement, Value, - ValueWithSpan, VisitMut, VisitorMut, -}; -use sqlparser::tokenizer::Span; - -use super::dialect::{PlaceholderStyle, placeholder_style}; -use super::statement::{ - DatabaseQuery, OutputColumn, Query, QueryBody, SingleRowQuery, SourceLocation, SourceSpan, -}; -use super::{extract_json_columns, is_json_expression, is_sqlpage_func}; -use crate::webserver::database::sqlpage_expr::{ - NoRowInput, RowExpr, RowInputId, SqlPageExpr, StandaloneExpr, VariableRef, VariableSource, -}; -use crate::webserver::database::sqlpage_functions::functions::SqlPageFunctionName; -use crate::webserver::database::{DbInfo, SupportedDatabase}; -use sqlx::any::AnyKind; - -const SQLPAGE_INPUT_PREFIX: &str = "__sqlpage_input_"; - -/// State used while partitioning source expressions between `SQLPage` and the database. -struct QueryPartitioner<'a> { - database: &'a DbInfo, - row_input_json: Vec, - private_projection: Vec, -} - -/// State used after partitioning to lower `SQLPage` values into database bindings. -struct DatabaseLowerer<'a> { - database: &'a DbInfo, - bindings: Vec, - error: Option, -} - -struct ComputedAliasFinder<'a> { - computed_columns: &'a [OutputColumn], -} - -impl sqlparser::ast::Visitor for ComputedAliasFinder<'_> { - type Break = (); - - fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { - let SqlExpr::Identifier(identifier) = expression else { - return ControlFlow::Continue(()); - }; - if self - .computed_columns - .iter() - .any(|column| identifier.value.eq_ignore_ascii_case(&column.name)) - { - return ControlFlow::Break(()); - } - ControlFlow::Continue(()) - } -} - -/// Result of partitioning one projected expression. -// Keeping the owned AST inline avoids one heap allocation for every ordinary -// projected expression. The enum is short-lived inside the rewriter. -#[allow(clippy::large_enum_variant)] -enum PartitionedProjection { - Database(SqlExpr), - PerRow(RowExpr), -} - -/// A partitioned projection after its database fragments have been lowered. -#[allow(clippy::large_enum_variant)] -enum LoweredProjection { - Database(SqlExpr), - PerRow(RowExpr), -} - -/// Defines how a SQL expression crossing into a SQLPage-owned expression is -/// represented at a particular evaluation site. -trait ExprEnvironment { - type Input; - - fn use_database_expr( - &mut self, - database: &DbInfo, - expression: SqlExpr, - ) -> anyhow::Result>; -} - -/// Rejects database-owned inputs because no returned row is available. -struct StandaloneEnvironment; -/// Projects database-owned inputs into the current returned row. -struct RowEnvironment<'a> { - row_input_json: &'a mut Vec, - private_projection: &'a mut Vec, -} - -impl ExprEnvironment for StandaloneEnvironment { - type Input = NoRowInput; - - fn use_database_expr( - &mut self, - _database: &DbInfo, - expression: SqlExpr, - ) -> anyhow::Result { - if let SqlExpr::Function(function) = &expression - && let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() - { - return Err(anyhow!( - "{} is not a supported sqlpage function and cannot be evaluated before the query", - name.value - )); - } - Err(anyhow!( - "{expression} is a database expression, but its value is required before the query can run" - )) - } -} - -impl ExprEnvironment for RowEnvironment<'_> { - type Input = RowInputId; - - fn use_database_expr( - &mut self, - _database: &DbInfo, - expression: SqlExpr, - ) -> anyhow::Result { - let decode_as_json = is_json_expression(&expression); - let index = self.row_input_json.len(); - let name = format!("{SQLPAGE_INPUT_PREFIX}{index}"); - self.private_projection.push(SelectItem::ExprWithAlias { - expr: expression, - alias: Ident::with_quote('"', name), - }); - self.row_input_json.push(decode_as_json); - Ok(SqlPageExpr::Input(RowInputId::new(index))) - } -} - -#[derive(Clone, Copy)] -/// SQL operations whose semantics are implemented by the shared `SQLPage` -/// expression evaluator. -enum EmulatedFunction { - Concat, - Coalesce, - JsonObject, - JsonArray, -} - -/// Rewrites one parsed statement into database SQL plus the `SQLPage` -/// expressions evaluated around it. -pub(super) fn rewrite_query( - mut statement: SqlStatement, - database: &DbInfo, - semicolon: bool, -) -> anyhow::Result { - let source_span = source_span(&statement); - let mut partitioner = QueryPartitioner { - database, - row_input_json: Vec::new(), - private_projection: Vec::new(), - }; - if let Some(single_row) = rewrite_single_row(&mut statement, &mut partitioner)? { - return Ok(Query { - body: QueryBody::SingleRow(single_row), - source_span, - }); - } - let mut lowerer = DatabaseLowerer::new(database); - let computed_columns = - rewrite_top_level_projection(&mut statement, &mut partitioner, &mut lowerer)?; - - if let SqlStatement::Query(query) = &mut statement - && let SetExpr::Select(select) = query.body.as_mut() - && select.projection.is_empty() - { - select.projection.push(SelectItem::ExprWithAlias { - expr: SqlExpr::value(Value::Null), - alias: Ident::with_quote('"', format!("{SQLPAGE_INPUT_PREFIX}anchor")), - }); - partitioner.row_input_json.push(false); - } - - let json_columns = extract_json_columns(&statement, database.database_type) - .into_iter() - .filter(|name| !name.starts_with(SQLPAGE_INPUT_PREFIX)) - .collect(); - lowerer.lower_statement(&mut statement)?; - let bindings = lowerer.finish_bindings(&mut statement)?; - let sql = format!( - "{statement}{semicolon}", - semicolon = if semicolon { ";" } else { "" } - ); - - Ok(Query { - body: QueryBody::Database(DatabaseQuery { - sql, - bindings, - row_input_json: partitioner.row_input_json.into_boxed_slice(), - computed_columns: computed_columns.into_boxed_slice(), - json_columns, - }), - source_span, - }) -} - -/// Removes SQLPage-owned projection expressions from the database projection -/// and appends their private database inputs as a trailing suffix. -fn rewrite_top_level_projection( - statement: &mut SqlStatement, - partitioner: &mut QueryPartitioner<'_>, - lowerer: &mut DatabaseLowerer<'_>, -) -> anyhow::Result>> { - let SqlStatement::Query(query) = statement else { - return Ok(Vec::new()); - }; - let SetExpr::Select(select) = query.body.as_mut() else { - return Ok(Vec::new()); - }; - if select.distinct.is_some() && select.projection.iter().any(select_item_contains_sqlpage) { - anyhow::bail!( - "SQLPage-computed projections cannot be used with SELECT DISTINCT because DISTINCT must be evaluated by the database" - ); - } - - let (mut database_projection, computed_columns) = - rewrite_projection_items(std::mem::take(&mut select.projection), partitioner, lowerer)?; - database_projection.append(&mut partitioner.private_projection); - select.projection = database_projection; - - reject_computed_alias_references("WHERE", select.selection.as_ref(), &computed_columns)?; - reject_computed_group_by_references(&select.group_by, &computed_columns)?; - reject_computed_alias_references("HAVING", select.having.as_ref(), &computed_columns)?; - reject_computed_alias_references("QUALIFY", select.qualify.as_ref(), &computed_columns)?; - reject_computed_aliases_in_expressions("CLUSTER BY", &select.cluster_by, &computed_columns)?; - reject_computed_aliases_in_expressions( - "DISTRIBUTE BY", - &select.distribute_by, - &computed_columns, - )?; - reject_computed_ordering_references("SORT BY", &select.sort_by, &computed_columns)?; - reject_computed_order_by_references(query.order_by.as_ref(), &computed_columns)?; - Ok(computed_columns) -} - -fn rewrite_projection_items( - projection: Vec, - partitioner: &mut QueryPartitioner<'_>, - lowerer: &mut DatabaseLowerer<'_>, -) -> anyhow::Result<(Vec, Vec>)> { - let mut database_projection = Vec::with_capacity(projection.len()); - let mut computed_columns = Vec::new(); - for item in projection { - let (expression, alias) = match item { - SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias)), - SelectItem::UnnamedExpr(expr) => (expr, None), - item => { - database_projection.push(item); - continue; - } - }; - let name = alias - .as_ref() - .map_or_else(|| expression.to_string(), |alias| alias.value.clone()); - let first_private_input = partitioner.private_projection.len(); - let projection = partitioner.rewrite_projection(expression)?; - match lowerer.lower_projection( - projection, - &mut partitioner.private_projection[first_private_input..], - )? { - LoweredProjection::Database(expression) => { - database_projection.push(match alias { - Some(alias) => SelectItem::ExprWithAlias { - expr: expression, - alias, - }, - None => SelectItem::UnnamedExpr(expression), - }); - } - LoweredProjection::PerRow(value) => { - computed_columns.push(OutputColumn { name, value }); - } - } - } - Ok((database_projection, computed_columns)) -} - -fn reject_computed_order_by_references( - order_by: Option<&sqlparser::ast::OrderBy>, - computed_columns: &[OutputColumn], -) -> anyhow::Result<()> { - if let Some(order_by) = order_by - && let OrderByKind::Expressions(expressions) = &order_by.kind - { - reject_computed_ordering_references("ORDER BY", expressions, computed_columns)?; - } - Ok(()) -} - -fn reject_computed_ordering_references( - clause: &str, - expressions: &[sqlparser::ast::OrderByExpr], - computed_columns: &[OutputColumn], -) -> anyhow::Result<()> { - if expressions - .iter() - .any(|ordering| references_computed_projection(&ordering.expr, computed_columns, true)) - { - anyhow::bail!( - "{clause} cannot reference a SQLPage-computed column because ordering is performed by the database" - ); - } - Ok(()) -} - -fn reject_computed_group_by_references( - group_by: &GroupByExpr, - computed_columns: &[OutputColumn], -) -> anyhow::Result<()> { - let GroupByExpr::Expressions(expressions, _) = group_by else { - return Ok(()); - }; - if expressions - .iter() - .any(|expression| references_computed_projection(expression, computed_columns, true)) - { - anyhow::bail!( - "GROUP BY cannot reference a SQLPage-computed column because grouping is performed by the database" - ); - } - Ok(()) -} - -fn reject_computed_alias_references( - clause: &str, - expression: Option<&SqlExpr>, - computed_columns: &[OutputColumn], -) -> anyhow::Result<()> { - if expression.is_some_and(|expression| { - references_computed_projection(expression, computed_columns, false) - }) { - anyhow::bail!( - "{clause} cannot reference a SQLPage-computed column because it is evaluated by the database" - ); - } - Ok(()) -} - -fn reject_computed_aliases_in_expressions( - clause: &str, - expressions: &[SqlExpr], - computed_columns: &[OutputColumn], -) -> anyhow::Result<()> { - if expressions - .iter() - .any(|expression| references_computed_projection(expression, computed_columns, false)) - { - anyhow::bail!( - "{clause} cannot reference a SQLPage-computed column because it is evaluated by the database" - ); - } - Ok(()) -} - -fn references_computed_projection( - expression: &SqlExpr, - computed_columns: &[OutputColumn], - reject_ordinal: bool, -) -> bool { - if computed_columns.is_empty() { - return false; - } - if reject_ordinal - && matches!( - expression, - SqlExpr::Value(ValueWithSpan { - value: Value::Number(_, _), - .. - }) - ) - { - return true; - } - - let mut finder = ComputedAliasFinder { computed_columns }; - sqlparser::ast::Visit::visit(expression, &mut finder).is_break() -} - -/// Rewrites a guaranteed one-row query directly as standalone expressions, -/// avoiding both a database round trip and an intermediate row-expression tree. -fn rewrite_single_row( - statement: &mut SqlStatement, - partitioner: &mut QueryPartitioner<'_>, -) -> anyhow::Result> { - if !has_single_row_shape(statement) { - return Ok(None); - } - let SqlStatement::Query(query) = statement else { - return Ok(None); - }; - let SetExpr::Select(select) = query.body.as_mut() else { - return Ok(None); - }; - for item in &select.projection { - let SelectItem::ExprWithAlias { expr, .. } = item else { - return Ok(None); - }; - if !can_build_standalone(expr)? { - return Ok(None); - } - } - - let mut columns = Vec::with_capacity(select.projection.len()); - for item in std::mem::take(&mut select.projection) { - let SelectItem::ExprWithAlias { expr, alias } = item else { - unreachable!("projection shape was checked") - }; - columns.push(OutputColumn { - name: alias.value, - value: build_sqlpage_expr(partitioner.database, &mut StandaloneEnvironment, expr)?, - }); - } - Ok(Some(SingleRowQuery { - columns: columns.into_boxed_slice(), - })) -} - -fn has_single_row_shape(statement: &SqlStatement) -> bool { - let SqlStatement::Query(query) = statement else { - return false; - }; - if query.with.is_some() - || query.order_by.is_some() - || query.limit_clause.is_some() - || query.fetch.is_some() - || !query.locks.is_empty() - || query.for_clause.is_some() - || query.settings.is_some() - || query.format_clause.is_some() - || !query.pipe_operators.is_empty() - { - return false; - } - let SetExpr::Select(select) = query.body.as_ref() else { - return false; - }; - select.distinct.is_none() - && select.top.is_none() - && select.into.is_none() - && select.from.is_empty() - && select.lateral_views.is_empty() - && select.selection.is_none() - && select.group_by == GroupByExpr::Expressions(vec![], vec![]) - && select.cluster_by.is_empty() - && select.distribute_by.is_empty() - && select.sort_by.is_empty() - && select.having.is_none() - && select.named_window.is_empty() - && select.qualify.is_none() - && select.prewhere.is_none() - && select.connect_by.is_empty() - && select.optimizer_hints.is_empty() - && select.select_modifiers.is_none() - && select.exclude.is_none() -} - -/// Checks standalone support without consuming or cloning the AST, allowing -/// callers to select a rewrite path before moving any nodes. -fn can_build_standalone(expression: &SqlExpr) -> anyhow::Result { - match expression { - SqlExpr::Value(ValueWithSpan { - value: - Value::Boolean(_) - | Value::Number(_, _) - | Value::SingleQuotedString(_) - | Value::Null - | Value::Placeholder(_), - .. - }) => Ok(true), - SqlExpr::Identifier(identifier) => Ok(variable_from_ident(identifier).is_some()), - SqlExpr::Function(function) => { - if recognize_sqlpage_function(function)?.is_none() - && emulated_function(function).is_none() - { - return Ok(false); - } - let FunctionArguments::List(arguments) = &function.args else { - return Ok(false); - }; - for argument in &arguments.args { - let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { - return Ok(false); - }; - if !can_build_standalone(expression)? { - return Ok(false); - } - } - Ok(true) - } - SqlExpr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => Ok(can_build_standalone(left)? && can_build_standalone(right)?), - SqlExpr::Nested(expression) => can_build_standalone(expression), - _ => Ok(false), - } -} - -impl QueryPartitioner<'_> { - /// Splits a projected expression at SQLPage-supported operations while - /// leaving opaque database operations in the SQL AST. - fn rewrite_projection(&mut self, expression: SqlExpr) -> anyhow::Result { - if !projection_is_per_row(&expression)? { - return Ok(PartitionedProjection::Database(expression)); - } - match expression { - SqlExpr::Function(function) => { - if recognize_sqlpage_function(&function)?.is_some() { - return self - .build_row_expr(SqlExpr::Function(function)) - .map(PartitionedProjection::PerRow); - } - let kind = emulated_function(&function) - .expect("per-row function ownership was already classified"); - self.rewrite_emulated_projection(function, kind) - } - SqlExpr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => { - let left = self.rewrite_projection(*left)?; - let left = self.projection_into_row_expr(left)?; - let right = self.rewrite_projection(*right)?; - let right = self.projection_into_row_expr(right)?; - Ok(PartitionedProjection::PerRow(SqlPageExpr::Concat { - arguments: vec![left, right].into_boxed_slice(), - null_behavior: self.database.database_type.concat_operator_null_behavior(), - })) - } - SqlExpr::Nested(expression) => match self.rewrite_projection(*expression)? { - PartitionedProjection::Database(expression) => Ok(PartitionedProjection::Database( - SqlExpr::Nested(Box::new(expression)), - )), - PartitionedProjection::PerRow(expression) => { - Ok(PartitionedProjection::PerRow(expression)) - } - }, - _ => unreachable!("per-row expression ownership was already classified"), - } - } - - fn rewrite_emulated_projection( - &mut self, - function: Function, - kind: EmulatedFunction, - ) -> anyhow::Result { - let (arguments, _) = take_expression_arguments(function)?; - let mut rewritten = Vec::with_capacity(arguments.len()); - for argument in arguments { - let argument = self.rewrite_projection(argument)?; - rewritten.push(self.projection_into_row_expr(argument)?); - } - Ok(PartitionedProjection::PerRow(build_emulated( - kind, - rewritten, - self.database.database_type, - )?)) - } - - /// Converts a database-owned projection fragment into a typed row input, - /// while preserving an already per-row expression unchanged. - fn projection_into_row_expr( - &mut self, - projection: PartitionedProjection, - ) -> anyhow::Result { - match projection { - PartitionedProjection::Database(expression) => self.build_row_expr(expression), - PartitionedProjection::PerRow(expression) => Ok(expression), - } - } - - fn build_row_expr(&mut self, expression: SqlExpr) -> anyhow::Result { - build_sqlpage_expr( - self.database, - &mut RowEnvironment { - row_input_json: &mut self.row_input_json, - private_projection: &mut self.private_projection, - }, - expression, - ) - } -} - -/// Determines ownership without mutating the AST, so database fragments can -/// be promoted to row inputs in source order once a per-row parent is known. -fn projection_is_per_row(expression: &SqlExpr) -> anyhow::Result { - match expression { - SqlExpr::Function(function) => { - if recognize_sqlpage_function(function)?.is_some() { - return Ok(true); - } - if emulated_function(function).is_none() { - return Ok(false); - } - let FunctionArguments::List(arguments) = &function.args else { - anyhow::bail!("Unsupported arguments to {}", function.name); - }; - if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { - anyhow::bail!("Unsupported arguments to {}", function.name); - } - for argument in &arguments.args { - let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { - anyhow::bail!("Named and wildcard function arguments are not supported"); - }; - if projection_is_per_row(expression)? { - return Ok(true); - } - } - Ok(false) - } - SqlExpr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => Ok(projection_is_per_row(left)? || projection_is_per_row(right)?), - SqlExpr::Nested(expression) => projection_is_per_row(expression), - _ => Ok(false), - } -} - -impl<'a> DatabaseLowerer<'a> { - fn new(database: &'a DbInfo) -> Self { - Self { - database, - bindings: Vec::new(), - error: None, - } - } - - /// Accepts a terminal partition result and lowers its database-owned - /// fragments. The lowered AST cannot return to recursive partitioning. - fn lower_projection( - &mut self, - projection: PartitionedProjection, - private_inputs: &mut [SelectItem], - ) -> anyhow::Result { - let projection = match projection { - PartitionedProjection::Database(mut expression) => { - self.lower_expression(&mut expression)?; - LoweredProjection::Database(expression) - } - PartitionedProjection::PerRow(expression) => { - for input in private_inputs { - let _ = input.visit(self); - self.take_error()?; - } - LoweredProjection::PerRow(expression) - } - }; - Ok(projection) - } - - fn lower_expression(&mut self, expression: &mut SqlExpr) -> anyhow::Result<()> { - let _ = expression.visit(self); - self.take_error() - } - - fn lower_statement(&mut self, statement: &mut SqlStatement) -> anyhow::Result<()> { - let _ = statement.visit(self); - self.take_error() - } - - fn take_error(&mut self) -> anyhow::Result<()> { - self.error.take().map_or(Ok(()), Err) - } - - fn add_binding(&mut self, value: StandaloneExpr) -> SqlExpr { - let sequence = self.bindings.len(); - self.bindings.push(value); - let placeholder = match placeholder_style(self.database.kind) { - PlaceholderStyle::Numbered { prefix } => format!("{prefix}{}", sequence + 1), - PlaceholderStyle::Positional { .. } => format!("${}", sequence + 1), - }; - cast_placeholder(placeholder, self.database) - } - - fn finish_bindings( - &mut self, - statement: &mut SqlStatement, - ) -> anyhow::Result> { - let PlaceholderStyle::Positional { token } = placeholder_style(self.database.kind) else { - return Ok(std::mem::take(&mut self.bindings).into_boxed_slice()); - }; - let mut finalizer = PositionalBindingFinalizer { - token, - binding_count: self.bindings.len(), - order: Vec::with_capacity(self.bindings.len()), - error: None, - }; - let _ = statement.visit(&mut finalizer); - if let Some(error) = finalizer.error { - return Err(error); - } - let mut bindings = std::mem::take(&mut self.bindings) - .into_iter() - .map(Some) - .collect::>(); - finalizer - .order - .into_iter() - .map(|index| { - bindings[index].take().ok_or_else(|| { - anyhow!("Generated binding placeholder ${} was repeated", index + 1) - }) - }) - .collect::>>() - } -} - -struct PositionalBindingFinalizer { - token: &'static str, - binding_count: usize, - order: Vec, - error: Option, -} - -impl VisitorMut for PositionalBindingFinalizer { - type Break = (); - - fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { - let SqlExpr::Value(ValueWithSpan { - value: Value::Placeholder(name), - span, - }) = expression - else { - return ControlFlow::Continue(()); - }; - if *span != Span::empty() { - return ControlFlow::Continue(()); - } - let Some(index) = name - .strip_prefix('$') - .and_then(|number| number.parse::().ok()) - .and_then(|number| number.checked_sub(1)) - else { - return ControlFlow::Continue(()); - }; - if index >= self.binding_count { - self.error = Some(anyhow!("Invalid generated binding placeholder {name}")); - return ControlFlow::Break(()); - } - self.order.push(index); - self.token.clone_into(name); - ControlFlow::Continue(()) - } -} - -impl VisitorMut for DatabaseLowerer<'_> { - type Break = (); - - fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { - if self.error.is_some() { - return ControlFlow::Break(()); - } - - let replacement = match expression { - SqlExpr::Value(ValueWithSpan { - value: Value::Placeholder(_), - span, - }) if *span == Span::empty() => None, - SqlExpr::Value(ValueWithSpan { - value: Value::Placeholder(_), - .. - }) - | SqlExpr::Identifier(_) => variable_from_expr(expression) - .map(|variable| self.add_binding(SqlPageExpr::Variable(variable))), - SqlExpr::Function(function) => match recognize_sqlpage_function(function) { - Ok(Some(_)) => { - let owned = std::mem::replace(expression, SqlExpr::value(Value::Null)); - match build_sqlpage_expr(self.database, &mut StandaloneEnvironment, owned) { - Ok(value) => Some(self.add_binding(value)), - Err(error) => { - self.error = Some(error.context( - "A SQLPage function used by the database could not be evaluated before the query", - )); - None - } - } - } - Ok(None) => None, - Err(error) => { - self.error = Some(error); - None - } - }, - SqlExpr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } if self.database.database_type == SupportedDatabase::Mssql => { - let left = std::mem::replace(left.as_mut(), SqlExpr::value(Value::Null)); - let right = std::mem::replace(right.as_mut(), SqlExpr::value(Value::Null)); - Some(make_function("CONCAT", vec![left, right])) - } - SqlExpr::Cast { - kind: kind @ CastKind::DoubleColon, - .. - } if ![ - SupportedDatabase::Postgres, - SupportedDatabase::Duckdb, - SupportedDatabase::Snowflake, - SupportedDatabase::Generic, - ] - .contains(&self.database.database_type) => - { - *kind = CastKind::Cast; - None - } - _ => None, - }; - - if let Some(replacement) = replacement { - *expression = replacement; - } - ControlFlow::Continue(()) - } -} - -/// Consumes an AST expression into the shared `SQLPage` expression type. The -/// environment determines whether opaque database fragments are illegal or -/// become private row inputs. -fn build_sqlpage_expr( - database: &DbInfo, - environment: &mut Environment, - expression: SqlExpr, -) -> anyhow::Result> { - match expression { - SqlExpr::Value(ValueWithSpan { value, .. }) => match value { - Value::Placeholder(name) => Ok(SqlPageExpr::Variable(variable_from_placeholder(name))), - Value::SingleQuotedString(text) => Ok(SqlPageExpr::Literal(JsonValue::String(text))), - Value::Number(number, _) => Ok(SqlPageExpr::Literal(JsonValue::Number( - number.parse().context("Invalid numeric SQL literal")?, - ))), - Value::Boolean(value) => Ok(SqlPageExpr::Literal(JsonValue::Bool(value))), - Value::Null => Ok(SqlPageExpr::Literal(JsonValue::Null)), - _ => { - environment.use_database_expr(database, SqlExpr::Value(ValueWithSpan::from(value))) - } - }, - SqlExpr::Identifier(identifier) => { - if let Some(variable) = variable_from_ident(&identifier) { - Ok(SqlPageExpr::Variable(variable)) - } else { - environment.use_database_expr(database, SqlExpr::Identifier(identifier)) - } - } - SqlExpr::Function(function) => { - if let Some(function_name) = recognize_sqlpage_function(&function)? { - let (arguments, _) = take_expression_arguments(function)?; - let arguments = arguments - .into_iter() - .map(|argument| build_sqlpage_expr(database, environment, argument)) - .collect::>>()?; - Ok(SqlPageExpr::Call { - function: function_name, - arguments: arguments.into_boxed_slice(), - }) - } else if let Some(kind) = emulated_function(&function) { - let (arguments, _) = take_expression_arguments(function)?; - let arguments = arguments - .into_iter() - .map(|argument| build_sqlpage_expr(database, environment, argument)) - .collect::>>()?; - build_emulated(kind, arguments, database.database_type) - } else { - environment.use_database_expr(database, SqlExpr::Function(function)) - } - } - SqlExpr::BinaryOp { - left, - op: BinaryOperator::StringConcat, - right, - } => Ok(SqlPageExpr::Concat { - arguments: vec![ - build_sqlpage_expr(database, environment, *left)?, - build_sqlpage_expr(database, environment, *right)?, - ] - .into_boxed_slice(), - null_behavior: database.database_type.concat_operator_null_behavior(), - }), - SqlExpr::Nested(expression) => build_sqlpage_expr(database, environment, *expression), - expression => environment.use_database_expr(database, expression), - } -} - -fn build_emulated( - kind: EmulatedFunction, - arguments: Vec>, - database: SupportedDatabase, -) -> anyhow::Result> { - Ok(match kind { - EmulatedFunction::Concat => SqlPageExpr::Concat { - arguments: arguments.into_boxed_slice(), - null_behavior: database.concat_function_null_behavior(), - }, - EmulatedFunction::Coalesce => SqlPageExpr::Coalesce(arguments.into_boxed_slice()), - EmulatedFunction::JsonArray => SqlPageExpr::JsonArray(arguments.into_boxed_slice()), - EmulatedFunction::JsonObject => { - if !arguments.len().is_multiple_of(2) { - anyhow::bail!("JSON_OBJECT requires an even number of arguments"); - } - let mut arguments = arguments.into_iter(); - let mut entries = Vec::with_capacity(arguments.len() / 2); - while let Some(key) = arguments.next() { - let value = arguments.next().expect("argument count was checked"); - entries.push((key, value)); - } - SqlPageExpr::JsonObject(entries.into_boxed_slice()) - } - }) -} - -/// Recognizes and validates an unquoted `sqlpage.` call. A recognized -/// call is either rewritten or rejected and can never reach database SQL. -fn recognize_sqlpage_function(function: &Function) -> anyhow::Result> { - let ObjectName(parts) = &function.name; - if !is_sqlpage_func(parts) { - return Ok(None); - } - let [ - ObjectNamePart::Identifier(_), - ObjectNamePart::Identifier(name), - ] = parts.as_slice() - else { - unreachable!("is_sqlpage_func checked the name") - }; - if function.uses_odbc_syntax - || !matches!(function.parameters, FunctionArguments::None) - || function.filter.is_some() - || function.null_treatment.is_some() - || function.over.is_some() - || !function.within_group.is_empty() - { - anyhow::bail!( - "Modifiers are not supported on SQLPage function {}", - function.name - ); - } - let FunctionArguments::List(FunctionArgumentList { - duplicate_treatment: None, - clauses, - .. - }) = &function.args - else { - anyhow::bail!( - "Unsupported argument syntax for SQLPage function {}", - function.name - ); - }; - if !clauses.is_empty() { - anyhow::bail!( - "Argument clauses are not supported on SQLPage function {}", - function.name - ); - } - Ok(Some(SqlPageFunctionName::from_str(&name.value)?)) -} - -fn emulated_function(function: &Function) -> Option { - let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { - return None; - }; - if !matches!(function.parameters, FunctionArguments::None) - || function.filter.is_some() - || function.null_treatment.is_some() - || function.over.is_some() - || !function.within_group.is_empty() - { - return None; - } - match name.value.to_ascii_lowercase().as_str() { - "concat" => Some(EmulatedFunction::Concat), - "coalesce" => Some(EmulatedFunction::Coalesce), - "json_object" | "jsonb_object" | "json_build_object" | "jsonb_build_object" => { - Some(EmulatedFunction::JsonObject) - } - "json_array" | "jsonb_array" | "json_build_array" | "jsonb_build_array" => { - Some(EmulatedFunction::JsonArray) - } - _ => None, - } -} - -/// Moves expression arguments out of a function while retaining its emptied -/// AST shell so database-owned functions can be rebuilt without cloning. -fn take_expression_arguments(mut function: Function) -> anyhow::Result<(Vec, Function)> { - let FunctionArguments::List(arguments) = &mut function.args else { - anyhow::bail!("Unsupported arguments to {}", function.name); - }; - if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { - anyhow::bail!("Unsupported arguments to {}", function.name); - } - let arguments = std::mem::take(&mut arguments.args) - .into_iter() - .map(|argument| match argument { - FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) => Ok(expression), - _ => Err(anyhow!( - "Named and wildcard function arguments are not supported" - )), - }) - .collect::>>()?; - Ok((arguments, function)) -} - -fn make_function(name: &str, expressions: Vec) -> SqlExpr { - SqlExpr::Function(Function { - name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new(name))]), - args: FunctionArguments::List(FunctionArgumentList { - args: expressions - .into_iter() - .map(|expression| FunctionArg::Unnamed(FunctionArgExpr::Expr(expression))) - .collect(), - duplicate_treatment: None, - clauses: Vec::new(), - }), - parameters: FunctionArguments::None, - over: None, - filter: None, - null_treatment: None, - within_group: Vec::new(), - uses_odbc_syntax: false, - }) -} - -fn variable_from_expr(expression: &SqlExpr) -> Option { - match expression { - SqlExpr::Value(ValueWithSpan { - value: Value::Placeholder(name), - .. - }) => Some(variable_from_placeholder(name.clone())), - SqlExpr::Identifier(identifier) => variable_from_ident(identifier), - _ => None, - } -} - -fn variable_from_ident(identifier: &Ident) -> Option { - if identifier.quote_style.is_some() { - return None; - } - let prefix = identifier.value.chars().next()?; - matches!(prefix, '$' | ':' | '?').then(|| VariableRef { - name: identifier.value[prefix.len_utf8()..].to_owned(), - source: variable_source(prefix), - }) -} - -fn variable_from_placeholder(mut name: String) -> VariableRef { - let prefix = name.remove(0); - VariableRef { - name, - source: variable_source(prefix), - } -} - -fn variable_source(prefix: char) -> VariableSource { - match prefix { - '$' => VariableSource::SetOrUrl, - ':' => VariableSource::SetOrForm, - _ => VariableSource::Url, - } -} - -/// Wraps a generated placeholder in the backend-specific text cast when the -/// database cannot reliably infer that the parameter is a string. -/// -/// `SQLPage` always binds parameters as strings. Native `PostgreSQL` (which -/// pins the parameter type to `TEXT` when preparing the statement), `MySQL` -/// and `SQL Server` (which convert the bound string to the type expected by -/// the surrounding expression) do not need the cast, and it can even be -/// harmful: on `SQL Server` the parameter is bound as `NVARCHAR(MAX)`, and -/// casting it to a narrow `VARCHAR` mangles non-ASCII values. `SQLite` -/// needs it to keep text affinity in comparisons with numbers. -/// -/// Through ODBC, the decision follows the database behind the driver, since -/// `SQLPage` knows it from the driver's reported name: -/// - `PostgreSQL` keeps the cast: `psqlodbc` provides no parameter type -/// information, and the server then fails on context-free parameters -/// (`could not determine data type of parameter`). -/// - `SQLite` keeps it for the same affinity reasons as native connections. -/// - `MySQL`, `SQL Server` and `DuckDB` drop it, like their native -/// counterparts: the former two convert the string at execution time, and -/// `DuckDB` defaults untyped parameters to `VARCHAR`. -/// - `Oracle`, `Snowflake` and unknown databases keep it conservatively. -fn cast_placeholder(placeholder: String, database: &DbInfo) -> SqlExpr { - let data_type = match database.kind { - AnyKind::Sqlite => DataType::Text, - AnyKind::Postgres | AnyKind::MySql | AnyKind::Mssql => { - return SqlExpr::value(Value::Placeholder(placeholder)); - } - AnyKind::Odbc => match database.database_type { - SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, - SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { - length: 4000, - unit: None, - })), - SupportedDatabase::MySql | SupportedDatabase::Mssql | SupportedDatabase::Duckdb => { - return SqlExpr::value(Value::Placeholder(placeholder)); - } - _ => DataType::Varchar(None), - }, - }; - SqlExpr::Cast { - expr: Box::new(SqlExpr::value(Value::Placeholder(placeholder))), - data_type, - format: None, - kind: CastKind::Cast, - array: false, - } -} - -fn source_span(value: &impl sqlparser::ast::Spanned) -> SourceSpan { - let span = value.span(); - SourceSpan { - start: SourceLocation { - line: usize::try_from(span.start.line).unwrap_or(0), - column: usize::try_from(span.start.column).unwrap_or(0), - }, - end: SourceLocation { - line: usize::try_from(span.end.line).unwrap_or(0), - column: usize::try_from(span.end.column).unwrap_or(0), - }, - } -} - -fn select_item_contains_sqlpage(item: &SelectItem) -> bool { - struct Finder(bool); - impl sqlparser::ast::Visitor for Finder { - type Break = (); - - fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { - if let SqlExpr::Function(function) = expression - && is_sqlpage_func(&function.name.0) - { - self.0 = true; - return ControlFlow::Break(()); - } - ControlFlow::Continue(()) - } - } - let mut finder = Finder(false); - let _ = sqlparser::ast::Visit::visit(item, &mut finder); - finder.0 -} diff --git a/src/webserver/database/sql/rewrite/computed_column_validation.rs b/src/webserver/database/sql/rewrite/computed_column_validation.rs new file mode 100644 index 00000000..e5e0ad12 --- /dev/null +++ b/src/webserver/database/sql/rewrite/computed_column_validation.rs @@ -0,0 +1,136 @@ +//! Rejects database clauses that depend on columns computed later by `SQLPage`. + +use std::ops::ControlFlow; + +use sqlparser::ast::{ + Expr as SqlExpr, GroupByExpr, OrderByKind, Query, Select, SelectItem, SetExpr, Value, + ValueWithSpan, +}; + +use super::super::statement::OutputColumn; +use super::sqlpage_expression::is_sqlpage_func; +use crate::webserver::database::sqlpage_expr::RowExpr; + +pub(super) fn reject_distinct_sqlpage_projection(select: &Select) -> anyhow::Result<()> { + if select.distinct.is_some() && select.projection.iter().any(select_item_contains_sqlpage) { + anyhow::bail!( + "SQLPage-computed projections cannot be used with SELECT DISTINCT because DISTINCT must be evaluated by the database" + ); + } + Ok(()) +} + +pub(super) fn reject_references( + query: &Query, + computed_columns: &[OutputColumn], +) -> anyhow::Result<()> { + let SetExpr::Select(select) = query.body.as_ref() else { + return Ok(()); + }; + let validator = ComputedColumnValidator(computed_columns); + validator.reject("WHERE", select.selection.iter(), false)?; + if let GroupByExpr::Expressions(expressions, _) = &select.group_by { + validator.reject("GROUP BY", expressions, true)?; + } + for (clause, expression) in [ + ("HAVING", select.having.as_ref()), + ("QUALIFY", select.qualify.as_ref()), + ] { + validator.reject(clause, expression, false)?; + } + for (clause, expressions) in [ + ("CLUSTER BY", select.cluster_by.as_slice()), + ("DISTRIBUTE BY", select.distribute_by.as_slice()), + ] { + validator.reject(clause, expressions, false)?; + } + validator.reject( + "SORT BY", + select.sort_by.iter().map(|ordering| &ordering.expr), + true, + )?; + if let Some(order_by) = &query.order_by + && let OrderByKind::Expressions(expressions) = &order_by.kind + { + validator.reject( + "ORDER BY", + expressions.iter().map(|ordering| &ordering.expr), + true, + )?; + } + Ok(()) +} + +struct ComputedColumnValidator<'a>(&'a [OutputColumn]); + +impl ComputedColumnValidator<'_> { + fn reject<'a>( + &self, + clause: &str, + expressions: impl IntoIterator, + reject_ordinal: bool, + ) -> anyhow::Result<()> { + if self.0.is_empty() { + return Ok(()); + } + let references_computed_column = |expression: &SqlExpr| { + if reject_ordinal + && matches!( + expression, + SqlExpr::Value(ValueWithSpan { + value: Value::Number(_, _), + .. + }) + ) + { + return true; + } + let mut finder = ComputedAliasFinder(self.0); + sqlparser::ast::Visit::visit(expression, &mut finder).is_break() + }; + if expressions.into_iter().any(references_computed_column) { + let reason = match clause { + "GROUP BY" => "grouping is performed by the database", + "SORT BY" | "ORDER BY" => "ordering is performed by the database", + _ => "it is evaluated by the database", + }; + anyhow::bail!("{clause} cannot reference a SQLPage-computed column because {reason}"); + } + Ok(()) + } +} + +struct ComputedAliasFinder<'a>(&'a [OutputColumn]); + +impl sqlparser::ast::Visitor for ComputedAliasFinder<'_> { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { + if let SqlExpr::Identifier(identifier) = expression + && self + .0 + .iter() + .any(|column| identifier.value.eq_ignore_ascii_case(&column.name)) + { + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + } +} + +fn select_item_contains_sqlpage(item: &SelectItem) -> bool { + struct SqlPageFunctionFinder; + impl sqlparser::ast::Visitor for SqlPageFunctionFinder { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &SqlExpr) -> ControlFlow { + if let SqlExpr::Function(function) = expression + && is_sqlpage_func(&function.name.0) + { + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + } + } + sqlparser::ast::Visit::visit(item, &mut SqlPageFunctionFinder).is_break() +} diff --git a/src/webserver/database/sql/rewrite/database_lowering.rs b/src/webserver/database/sql/rewrite/database_lowering.rs new file mode 100644 index 00000000..4524a890 --- /dev/null +++ b/src/webserver/database/sql/rewrite/database_lowering.rs @@ -0,0 +1,233 @@ +//! Converts SQLPage-owned values used by database SQL into prepared bindings. + +use std::ops::ControlFlow; + +use anyhow::anyhow; +use sqlparser::ast::{ + BinaryOperator, CastKind, CharacterLength, DataType, Expr as SqlExpr, Function, FunctionArg, + FunctionArgumentList, FunctionArguments, Ident, ObjectName, ObjectNamePart, + Statement as SqlStatement, Value, ValueWithSpan, VisitMut, VisitorMut, +}; +use sqlparser::tokenizer::Span; +use sqlx::any::AnyKind; + +use super::super::dialect::{PlaceholderStyle, placeholder_style}; +use super::sqlpage_expression::{ + StandaloneContext, build_sqlpage_expr, recognize_sqlpage_function, variable_from_expr, +}; +use crate::webserver::database::sqlpage_expr::{SqlPageExpr, StandaloneExpr}; +use crate::webserver::database::{DbInfo, SupportedDatabase}; + +/// Lowers SQLPage-owned values in database AST fragments into bindings. +pub(super) struct DatabaseLowerer<'a> { + database: &'a DbInfo, + bindings: Vec, + error: Option, +} + +impl<'a> DatabaseLowerer<'a> { + pub(super) fn new(database: &'a DbInfo) -> Self { + Self { + database, + bindings: Vec::new(), + error: None, + } + } + + pub(super) fn lower_ast(&mut self, value: &mut impl VisitMut) -> anyhow::Result<()> { + let _ = value.visit(self); + self.error.take().map_or(Ok(()), Err) + } + + fn add_binding(&mut self, value: StandaloneExpr) -> SqlExpr { + let sequence = self.bindings.len(); + self.bindings.push(value); + let placeholder = match placeholder_style(self.database.kind) { + PlaceholderStyle::Numbered { prefix } => format!("{prefix}{}", sequence + 1), + PlaceholderStyle::Positional { .. } => format!("${}", sequence + 1), + }; + cast_placeholder(placeholder, self.database) + } + + pub(super) fn finish_bindings( + self, + statement: &mut SqlStatement, + ) -> anyhow::Result> { + let PlaceholderStyle::Positional { token } = placeholder_style(self.database.kind) else { + return Ok(self.bindings.into_boxed_slice()); + }; + let mut finalizer = PositionalBindingFinalizer { + token, + binding_count: self.bindings.len(), + order: Vec::with_capacity(self.bindings.len()), + error: None, + }; + let _ = statement.visit(&mut finalizer); + finalizer.error.map_or(Ok(()), Err)?; + let mut bindings = self.bindings.into_iter().map(Some).collect::>(); + finalizer + .order + .into_iter() + .map(|index| { + bindings[index].take().ok_or_else(|| { + anyhow!("Generated binding placeholder ${} was repeated", index + 1) + }) + }) + .collect() + } +} + +impl VisitorMut for DatabaseLowerer<'_> { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { + if self.error.is_some() { + return ControlFlow::Break(()); + } + let replacement = match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + span, + }) if *span == Span::empty() => None, + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(_), + .. + }) + | SqlExpr::Identifier(_) => variable_from_expr(expression) + .map(|variable| self.add_binding(SqlPageExpr::Variable(variable))), + SqlExpr::Function(function) => match recognize_sqlpage_function(function) { + Ok(Some(_)) => { + let owned = std::mem::replace(expression, SqlExpr::value(Value::Null)); + match build_sqlpage_expr(self.database, &mut StandaloneContext, owned) { + Ok(value) => Some(self.add_binding(value)), + Err(error) => { + self.error = Some(error.context( + "A SQLPage function used by the database could not be evaluated before the query", + )); + None + } + } + } + Ok(None) => None, + Err(error) => { + self.error = Some(error); + None + } + }, + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } if self.database.database_type == SupportedDatabase::Mssql => { + let left = std::mem::replace(left.as_mut(), SqlExpr::value(Value::Null)); + let right = std::mem::replace(right.as_mut(), SqlExpr::value(Value::Null)); + Some(make_concat(left, right)) + } + SqlExpr::Cast { + kind: kind @ CastKind::DoubleColon, + .. + } if !matches!( + self.database.database_type, + SupportedDatabase::Postgres + | SupportedDatabase::Duckdb + | SupportedDatabase::Snowflake + | SupportedDatabase::Generic + ) => + { + *kind = CastKind::Cast; + None + } + _ => None, + }; + if let Some(replacement) = replacement { + *expression = replacement; + } + ControlFlow::Continue(()) + } +} + +struct PositionalBindingFinalizer { + token: &'static str, + binding_count: usize, + order: Vec, + error: Option, +} + +impl VisitorMut for PositionalBindingFinalizer { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &mut SqlExpr) -> ControlFlow { + let SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(name), + span, + }) = expression + else { + return ControlFlow::Continue(()); + }; + if *span != Span::empty() { + return ControlFlow::Continue(()); + } + let Some(index) = name + .strip_prefix('$') + .and_then(|number| number.parse::().ok()) + .and_then(|number| number.checked_sub(1)) + else { + return ControlFlow::Continue(()); + }; + if index >= self.binding_count { + self.error = Some(anyhow!("Invalid generated binding placeholder {name}")); + return ControlFlow::Break(()); + } + self.order.push(index); + self.token.clone_into(name); + ControlFlow::Continue(()) + } +} + +fn make_concat(left: SqlExpr, right: SqlExpr) -> SqlExpr { + SqlExpr::Function(Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new("CONCAT"))]), + args: FunctionArguments::List(FunctionArgumentList { + args: vec![ + FunctionArg::Unnamed(left.into()), + FunctionArg::Unnamed(right.into()), + ], + duplicate_treatment: None, + clauses: Vec::new(), + }), + parameters: FunctionArguments::None, + over: None, + filter: None, + null_treatment: None, + within_group: Vec::new(), + uses_odbc_syntax: false, + }) +} + +/// Casts generated placeholders where the backend cannot infer string typing. +fn cast_placeholder(placeholder: String, database: &DbInfo) -> SqlExpr { + let data_type = match database.kind { + AnyKind::Sqlite => DataType::Text, + AnyKind::Postgres | AnyKind::MySql | AnyKind::Mssql => { + return SqlExpr::value(Value::Placeholder(placeholder)); + } + AnyKind::Odbc => match database.database_type { + SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, + SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 4000, + unit: None, + })), + SupportedDatabase::MySql | SupportedDatabase::Mssql | SupportedDatabase::Duckdb => { + return SqlExpr::value(Value::Placeholder(placeholder)); + } + _ => DataType::Varchar(None), + }, + }; + SqlExpr::Cast { + expr: Box::new(SqlExpr::value(Value::Placeholder(placeholder))), + data_type, + format: None, + kind: CastKind::Cast, + array: false, + } +} diff --git a/src/webserver/database/sql/rewrite/mod.rs b/src/webserver/database/sql/rewrite/mod.rs new file mode 100644 index 00000000..3a4f7070 --- /dev/null +++ b/src/webserver/database/sql/rewrite/mod.rs @@ -0,0 +1,152 @@ +//! Turns a parsed SQL statement into an immutable execution plan. +//! +//! The pipeline uses the same terms as `SQLPage`'s SQL documentation: +//! 1. A **static simple select** is a restricted `SELECT` that `SQLPage` can +//! evaluate without querying the database. Its selected values are +//! **standalone expressions**: expressions that need no returned row. +//! 2. **Projection partitioning** assigns each selected expression either to +//! the database or to per-row `SQLPage` evaluation. Database values needed by +//! a `SQLPage` expression become private trailing columns called row inputs. +//! 3. **Database lowering** replaces variables and `sqlpage.*` calls needed by +//! database SQL with typed bindings and backend-specific placeholders. +//! 4. **Computed-column validation** rejects relational clauses that refer to a +//! value computed by `SQLPage` after the database has returned its row. +//! +//! Partitioning a source projection must finish before lowering it. This +//! one-way handoff prevents generated database placeholders from returning to +//! `SQLPage` expression parsing. Numbered bindings retain source order; +//! positional bindings are finalized later in rendered SQL order. + +mod computed_column_validation; +mod database_lowering; +mod projection_partitioning; +mod sqlpage_expression; +mod static_simple_select; + +use sqlparser::ast::{ + Expr as SqlExpr, Ident, SelectItem, SetExpr, Statement as SqlStatement, Value, +}; + +use self::database_lowering::DatabaseLowerer; +use self::projection_partitioning::{ + PRIVATE_ROW_INPUT_PREFIX, PartitionedProjection, ProjectionPartitioner, + detect_public_json_columns, +}; +use super::statement::{DatabaseQuery, OutputColumn, Query, QueryBody, SourceLocation, SourceSpan}; +use crate::webserver::database::DbInfo; +use crate::webserver::database::sqlpage_expr::RowExpr; + +/// Rewrites one parsed statement into database SQL plus the `SQLPage` expressions +/// evaluated around it. +pub(super) fn rewrite_query( + mut statement: SqlStatement, + database: &DbInfo, + semicolon: bool, +) -> anyhow::Result { + let source_span = source_span(&statement); + if let Some(static_select) = static_simple_select::try_plan(&mut statement, database)? { + return Ok(Query { + body: QueryBody::StaticSimpleSelect(static_select), + source_span, + }); + } + + let mut partitioner = ProjectionPartitioner::new(database); + let mut lowerer = DatabaseLowerer::new(database); + let computed_columns = + partition_and_lower_top_level_projection(&mut statement, &mut partitioner, &mut lowerer)?; + if let SqlStatement::Query(query) = &mut statement + && let SetExpr::Select(select) = query.body.as_mut() + && select.projection.is_empty() + { + select.projection.push(SelectItem::ExprWithAlias { + expr: SqlExpr::value(Value::Null), + alias: Ident::with_quote('"', format!("{PRIVATE_ROW_INPUT_PREFIX}anchor")), + }); + partitioner.row_input_json.push(false); + } + + let json_columns = detect_public_json_columns(&statement, database.database_type); + lowerer.lower_ast(&mut statement)?; + let bindings = lowerer.finish_bindings(&mut statement)?; + let sql = format!("{statement}{}", if semicolon { ";" } else { "" }); + + Ok(Query { + body: QueryBody::Database(DatabaseQuery { + sql, + bindings, + row_input_json: partitioner.row_input_json.into_boxed_slice(), + computed_columns: computed_columns.into_boxed_slice(), + json_columns, + }), + source_span, + }) +} + +fn partition_and_lower_top_level_projection( + statement: &mut SqlStatement, + partitioner: &mut ProjectionPartitioner<'_>, + lowerer: &mut DatabaseLowerer<'_>, +) -> anyhow::Result>> { + let SqlStatement::Query(query) = statement else { + return Ok(Vec::new()); + }; + let computed_columns = { + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(Vec::new()); + }; + computed_column_validation::reject_distinct_sqlpage_projection(select)?; + + let mut database_projection = Vec::with_capacity(select.projection.len()); + let mut computed_columns = Vec::new(); + for item in std::mem::take(&mut select.projection) { + let (expression, alias) = match item { + SelectItem::ExprWithAlias { expr, alias } => (expr, Some(alias)), + SelectItem::UnnamedExpr(expr) => (expr, None), + item => { + database_projection.push(item); + continue; + } + }; + let name = alias + .as_ref() + .map_or_else(|| expression.to_string(), |alias| alias.value.clone()); + let first_private_input = partitioner.private_projection.len(); + match partitioner.partition_projection(expression)? { + PartitionedProjection::Database(mut expression) => { + lowerer.lower_ast(&mut expression)?; + database_projection.push(match alias { + Some(alias) => SelectItem::ExprWithAlias { + expr: expression, + alias, + }, + None => SelectItem::UnnamedExpr(expression), + }); + } + PartitionedProjection::PerRow(value) => { + for input in &mut partitioner.private_projection[first_private_input..] { + lowerer.lower_ast(input)?; + } + computed_columns.push(OutputColumn { name, value }); + } + } + } + database_projection.append(&mut partitioner.private_projection); + select.projection = database_projection; + computed_columns + }; + computed_column_validation::reject_references(query, &computed_columns)?; + Ok(computed_columns) +} + +fn source_span(value: &impl sqlparser::ast::Spanned) -> SourceSpan { + let span = value.span(); + let location = |location: sqlparser::tokenizer::Location| SourceLocation { + line: usize::try_from(location.line).unwrap_or(0), + column: usize::try_from(location.column).unwrap_or(0), + }; + SourceSpan { + start: location(span.start), + end: location(span.end), + } +} diff --git a/src/webserver/database/sql/rewrite/projection_partitioning.rs b/src/webserver/database/sql/rewrite/projection_partitioning.rs new file mode 100644 index 00000000..96c414d7 --- /dev/null +++ b/src/webserver/database/sql/rewrite/projection_partitioning.rs @@ -0,0 +1,232 @@ +//! Assigns selected expressions to the database or per-row `SQLPage` evaluation. + +use sqlparser::ast::{ + BinaryOperator, DataType, Expr as SqlExpr, FunctionArg, FunctionArgExpr, FunctionArguments, + Ident, ObjectNamePart, SelectItem, SetExpr, Statement as SqlStatement, +}; + +use super::sqlpage_expression::{ + SqlPageExpressionContext, build_emulated, build_sqlpage_expr, emulated_function, + recognize_sqlpage_function, take_expression_arguments, +}; +use crate::webserver::database::sqlpage_expr::{RowExpr, RowInputId, SqlPageExpr}; +use crate::webserver::database::{DbInfo, SupportedDatabase}; + +pub(super) const PRIVATE_ROW_INPUT_PREFIX: &str = "__sqlpage_input_"; + +/// Ownership assigned to one selected expression before database lowering. +#[allow(clippy::large_enum_variant)] +pub(super) enum PartitionedProjection { + /// The database evaluates this expression as part of its `SELECT`. + Database(SqlExpr), + /// `SQLPage` evaluates this expression for every row returned by the database. + PerRow(RowExpr), +} + +/// Partitions selected expressions and records their private database row inputs. +pub(super) struct ProjectionPartitioner<'a> { + database: &'a DbInfo, + pub(super) row_input_json: Vec, + pub(super) private_projection: Vec, +} + +struct PerRowContext<'a> { + row_input_json: &'a mut Vec, + private_projection: &'a mut Vec, +} + +impl SqlPageExpressionContext for PerRowContext<'_> { + type Input = RowInputId; + + fn use_database_expr(&mut self, expression: SqlExpr) -> anyhow::Result { + let index = self.row_input_json.len(); + let decode_as_json = is_json_expression(&expression); + self.private_projection.push(SelectItem::ExprWithAlias { + expr: expression, + alias: Ident::with_quote('"', format!("{PRIVATE_ROW_INPUT_PREFIX}{index}")), + }); + self.row_input_json.push(decode_as_json); + Ok(SqlPageExpr::Input(RowInputId::new(index))) + } +} + +impl<'a> ProjectionPartitioner<'a> { + pub(super) fn new(database: &'a DbInfo) -> Self { + Self { + database, + row_input_json: Vec::new(), + private_projection: Vec::new(), + } + } + + /// Splits a projection at SQLPage-supported operations while leaving opaque + /// database operations in the SQL AST. + pub(super) fn partition_projection( + &mut self, + expression: SqlExpr, + ) -> anyhow::Result { + if !projection_is_per_row(&expression)? { + return Ok(PartitionedProjection::Database(expression)); + } + match expression { + SqlExpr::Function(function) => { + if recognize_sqlpage_function(&function)?.is_some() { + return self + .build_row_expr(SqlExpr::Function(function)) + .map(PartitionedProjection::PerRow); + } + let kind = emulated_function(&function) + .expect("per-row function ownership was already classified"); + let arguments = take_expression_arguments(function)? + .into_iter() + .map(|argument| { + let projection = self.partition_projection(argument)?; + self.projection_into_row_expr(projection) + }) + .collect::>>()?; + Ok(PartitionedProjection::PerRow(build_emulated( + kind, + arguments, + self.database.database_type, + )?)) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => { + let left = self.partition_projection(*left)?; + let left = self.projection_into_row_expr(left)?; + let right = self.partition_projection(*right)?; + let right = self.projection_into_row_expr(right)?; + Ok(PartitionedProjection::PerRow(SqlPageExpr::Concat { + arguments: vec![left, right].into_boxed_slice(), + null_behavior: self.database.database_type.concat_operator_null_behavior(), + })) + } + SqlExpr::Nested(expression) => match self.partition_projection(*expression)? { + PartitionedProjection::Database(expression) => Ok(PartitionedProjection::Database( + SqlExpr::Nested(Box::new(expression)), + )), + PartitionedProjection::PerRow(expression) => { + Ok(PartitionedProjection::PerRow(expression)) + } + }, + _ => unreachable!("per-row expression ownership was already classified"), + } + } + + fn projection_into_row_expr( + &mut self, + projection: PartitionedProjection, + ) -> anyhow::Result { + match projection { + PartitionedProjection::Database(expression) => self.build_row_expr(expression), + PartitionedProjection::PerRow(expression) => Ok(expression), + } + } + + fn build_row_expr(&mut self, expression: SqlExpr) -> anyhow::Result { + build_sqlpage_expr( + self.database, + &mut PerRowContext { + row_input_json: &mut self.row_input_json, + private_projection: &mut self.private_projection, + }, + expression, + ) + } +} + +/// Determines ownership before mutation so promoted row inputs retain source order. +fn projection_is_per_row(expression: &SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Function(function) => { + if recognize_sqlpage_function(function)?.is_some() { + return Ok(true); + } + if emulated_function(function).is_none() { + return Ok(false); + } + let FunctionArguments::List(arguments) = &function.args else { + anyhow::bail!("Unsupported arguments to {}", function.name); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + anyhow::bail!("Unsupported arguments to {}", function.name); + } + for argument in &arguments.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { + anyhow::bail!("Named and wildcard function arguments are not supported"); + }; + if projection_is_per_row(expression)? { + return Ok(true); + } + } + Ok(false) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(projection_is_per_row(left)? || projection_is_per_row(right)?), + SqlExpr::Nested(expression) => projection_is_per_row(expression), + _ => Ok(false), + } +} + +pub(super) fn detect_public_json_columns( + statement: &SqlStatement, + database: SupportedDatabase, +) -> Box<[String]> { + if matches!( + database, + SupportedDatabase::Postgres | SupportedDatabase::Mssql + ) { + return Box::new([]); + } + let SqlStatement::Query(query) = statement else { + return Box::new([]); + }; + let SetExpr::Select(select) = query.body.as_ref() else { + return Box::new([]); + }; + select + .projection + .iter() + .filter_map(|item| match item { + SelectItem::ExprWithAlias { expr, alias } if is_json_expression(expr) => { + (!alias.value.starts_with(PRIVATE_ROW_INPUT_PREFIX)).then(|| alias.value.clone()) + } + _ => None, + }) + .collect() +} + +fn is_json_expression(expression: &SqlExpr) -> bool { + match expression { + SqlExpr::Function(function) => { + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return false; + }; + matches!( + name.value.to_ascii_lowercase().as_str(), + "json_object" + | "json_array" + | "json_build_object" + | "json_build_array" + | "to_json" + | "to_jsonb" + | "json_agg" + | "jsonb_agg" + | "json_arrayagg" + | "json_objectagg" + | "json_group_array" + | "json_group_object" + | "json" + | "jsonb" + ) + } + SqlExpr::Cast { data_type, .. } => matches!(data_type, DataType::JSON | DataType::JSONB), + _ => false, + } +} diff --git a/src/webserver/database/sql/rewrite/sqlpage_expression.rs b/src/webserver/database/sql/rewrite/sqlpage_expression.rs new file mode 100644 index 00000000..8df150b4 --- /dev/null +++ b/src/webserver/database/sql/rewrite/sqlpage_expression.rs @@ -0,0 +1,325 @@ +//! Recognition and construction of expressions evaluated by `SQLPage`. + +use std::str::FromStr as _; + +use anyhow::{Context as _, anyhow}; +use serde_json::Value as JsonValue; +use sqlparser::ast::{ + BinaryOperator, Expr as SqlExpr, Function, FunctionArg, FunctionArgExpr, FunctionArgumentList, + FunctionArguments, Ident, ObjectName, ObjectNamePart, Value, ValueWithSpan, +}; + +use crate::webserver::database::sqlpage_expr::{ + NoRowInput, SqlPageExpr, StandaloneExpr, VariableRef, VariableSource, +}; +use crate::webserver::database::sqlpage_functions::functions::SqlPageFunctionName; +use crate::webserver::database::{DbInfo, SupportedDatabase}; + +const SQLPAGE_FUNCTION_NAMESPACE: &str = "sqlpage"; + +/// Defines how an opaque database expression is represented when it crosses +/// into a SQLPage-owned expression. +pub(super) trait SqlPageExpressionContext { + type Input; + + fn use_database_expr( + &mut self, + expression: SqlExpr, + ) -> anyhow::Result>; +} + +/// Rejects database inputs because no returned row is available. +pub(super) struct StandaloneContext; + +impl SqlPageExpressionContext for StandaloneContext { + type Input = NoRowInput; + + fn use_database_expr(&mut self, expression: SqlExpr) -> anyhow::Result { + if let SqlExpr::Function(function) = &expression + && let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() + { + return Err(anyhow!( + "{} is not a supported sqlpage function and cannot be evaluated before the query", + name.value + )); + } + Err(anyhow!( + "{expression} is a database expression, but its value is required before the query can run" + )) + } +} + +#[derive(Clone, Copy)] +pub(super) enum EmulatedFunction { + Concat, + Coalesce, + JsonObject, + JsonArray, +} + +/// Checks whether `SQLPage` can evaluate a selected value without a database row. +pub(super) fn is_static_simple_select_expression(expression: &SqlExpr) -> anyhow::Result { + match expression { + SqlExpr::Value(ValueWithSpan { + value: + Value::Boolean(_) + | Value::Number(_, _) + | Value::SingleQuotedString(_) + | Value::Null + | Value::Placeholder(_), + .. + }) => Ok(true), + SqlExpr::Identifier(identifier) => Ok(variable_from_ident(identifier).is_some()), + SqlExpr::Function(function) => { + if recognize_sqlpage_function(function)?.is_none() + && emulated_function(function).is_none() + { + return Ok(false); + } + let FunctionArguments::List(arguments) = &function.args else { + return Ok(false); + }; + for argument in &arguments.args { + let FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) = argument else { + return Ok(false); + }; + if !is_static_simple_select_expression(expression)? { + return Ok(false); + } + } + Ok(true) + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => { + Ok(is_static_simple_select_expression(left)? + && is_static_simple_select_expression(right)?) + } + SqlExpr::Nested(expression) => is_static_simple_select_expression(expression), + _ => Ok(false), + } +} + +pub(super) fn build_sqlpage_expr( + database: &DbInfo, + context: &mut Context, + expression: SqlExpr, +) -> anyhow::Result> { + match expression { + SqlExpr::Value(ValueWithSpan { value, .. }) => match value { + Value::Placeholder(name) => Ok(SqlPageExpr::Variable(variable_from_placeholder(name))), + Value::SingleQuotedString(text) => Ok(SqlPageExpr::Literal(JsonValue::String(text))), + Value::Number(number, _) => Ok(SqlPageExpr::Literal(JsonValue::Number( + number.parse().context("Invalid numeric SQL literal")?, + ))), + Value::Boolean(value) => Ok(SqlPageExpr::Literal(JsonValue::Bool(value))), + Value::Null => Ok(SqlPageExpr::Literal(JsonValue::Null)), + _ => context.use_database_expr(SqlExpr::Value(ValueWithSpan::from(value))), + }, + SqlExpr::Identifier(identifier) => variable_from_ident(&identifier) + .map(SqlPageExpr::Variable) + .map_or_else( + || context.use_database_expr(SqlExpr::Identifier(identifier)), + Ok, + ), + SqlExpr::Function(function) => { + if let Some(function_name) = recognize_sqlpage_function(&function)? { + let arguments = take_expression_arguments(function)? + .into_iter() + .map(|argument| build_sqlpage_expr(database, context, argument)) + .collect::>>()?; + Ok(SqlPageExpr::Call { + function: function_name, + arguments: arguments.into_boxed_slice(), + }) + } else if let Some(kind) = emulated_function(&function) { + let arguments = take_expression_arguments(function)? + .into_iter() + .map(|argument| build_sqlpage_expr(database, context, argument)) + .collect::>>()?; + build_emulated(kind, arguments, database.database_type) + } else { + context.use_database_expr(SqlExpr::Function(function)) + } + } + SqlExpr::BinaryOp { + left, + op: BinaryOperator::StringConcat, + right, + } => Ok(SqlPageExpr::Concat { + arguments: vec![ + build_sqlpage_expr(database, context, *left)?, + build_sqlpage_expr(database, context, *right)?, + ] + .into_boxed_slice(), + null_behavior: database.database_type.concat_operator_null_behavior(), + }), + SqlExpr::Nested(expression) => build_sqlpage_expr(database, context, *expression), + expression => context.use_database_expr(expression), + } +} + +pub(super) fn build_emulated( + kind: EmulatedFunction, + arguments: Vec>, + database: SupportedDatabase, +) -> anyhow::Result> { + Ok(match kind { + EmulatedFunction::Concat => SqlPageExpr::Concat { + arguments: arguments.into_boxed_slice(), + null_behavior: database.concat_function_null_behavior(), + }, + EmulatedFunction::Coalesce => SqlPageExpr::Coalesce(arguments.into_boxed_slice()), + EmulatedFunction::JsonArray => SqlPageExpr::JsonArray(arguments.into_boxed_slice()), + EmulatedFunction::JsonObject => { + if !arguments.len().is_multiple_of(2) { + anyhow::bail!("JSON_OBJECT requires an even number of arguments"); + } + let mut arguments = arguments.into_iter(); + let mut entries = Vec::with_capacity(arguments.len() / 2); + while let Some(key) = arguments.next() { + entries.push((key, arguments.next().expect("argument count was checked"))); + } + SqlPageExpr::JsonObject(entries.into_boxed_slice()) + } + }) +} + +/// Recognizes and validates an unquoted `sqlpage.` call. +pub(super) fn recognize_sqlpage_function( + function: &Function, +) -> anyhow::Result> { + let ObjectName(parts) = &function.name; + if !is_sqlpage_func(parts) { + return Ok(None); + } + let ObjectNamePart::Identifier(name) = &parts[1] else { + unreachable!("is_sqlpage_func checked the name") + }; + if function.uses_odbc_syntax + || !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + anyhow::bail!( + "Modifiers are not supported on SQLPage function {}", + function.name + ); + } + let FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + clauses, + .. + }) = &function.args + else { + anyhow::bail!( + "Unsupported argument syntax for SQLPage function {}", + function.name + ); + }; + if !clauses.is_empty() { + anyhow::bail!( + "Argument clauses are not supported on SQLPage function {}", + function.name + ); + } + Ok(Some(SqlPageFunctionName::from_str(&name.value)?)) +} + +pub(super) fn is_sqlpage_func(parts: &[ObjectNamePart]) -> bool { + let [ + ObjectNamePart::Identifier(namespace), + ObjectNamePart::Identifier(name), + ] = parts + else { + return false; + }; + namespace.quote_style.is_none() + && name.quote_style.is_none() + && namespace + .value + .eq_ignore_ascii_case(SQLPAGE_FUNCTION_NAMESPACE) +} + +pub(super) fn emulated_function(function: &Function) -> Option { + let [ObjectNamePart::Identifier(name)] = function.name.0.as_slice() else { + return None; + }; + if !matches!(function.parameters, FunctionArguments::None) + || function.filter.is_some() + || function.null_treatment.is_some() + || function.over.is_some() + || !function.within_group.is_empty() + { + return None; + } + match name.value.to_ascii_lowercase().as_str() { + "concat" => Some(EmulatedFunction::Concat), + "coalesce" => Some(EmulatedFunction::Coalesce), + "json_object" | "jsonb_object" | "json_build_object" | "jsonb_build_object" => { + Some(EmulatedFunction::JsonObject) + } + "json_array" | "jsonb_array" | "json_build_array" | "jsonb_build_array" => { + Some(EmulatedFunction::JsonArray) + } + _ => None, + } +} + +pub(super) fn take_expression_arguments(function: Function) -> anyhow::Result> { + let FunctionArguments::List(arguments) = function.args else { + anyhow::bail!("Unsupported arguments to {}", function.name); + }; + if arguments.duplicate_treatment.is_some() || !arguments.clauses.is_empty() { + anyhow::bail!("Unsupported arguments to {}", function.name); + } + arguments + .args + .into_iter() + .map(|argument| match argument { + FunctionArg::Unnamed(FunctionArgExpr::Expr(expression)) => Ok(expression), + _ => Err(anyhow!( + "Named and wildcard function arguments are not supported" + )), + }) + .collect() +} + +pub(super) fn variable_from_expr(expression: &SqlExpr) -> Option { + match expression { + SqlExpr::Value(ValueWithSpan { + value: Value::Placeholder(name), + .. + }) => Some(variable_from_placeholder(name.clone())), + SqlExpr::Identifier(identifier) => variable_from_ident(identifier), + _ => None, + } +} + +fn variable_from_ident(identifier: &Ident) -> Option { + if identifier.quote_style.is_some() { + return None; + } + let prefix = identifier.value.chars().next()?; + matches!(prefix, '$' | ':' | '?').then(|| VariableRef { + name: identifier.value[prefix.len_utf8()..].to_owned(), + source: variable_source(prefix), + }) +} + +fn variable_from_placeholder(mut name: String) -> VariableRef { + let source = variable_source(name.remove(0)); + VariableRef { name, source } +} + +fn variable_source(prefix: char) -> VariableSource { + match prefix { + '$' => VariableSource::SetOrUrl, + ':' => VariableSource::SetOrForm, + _ => VariableSource::Url, + } +} diff --git a/src/webserver/database/sql/rewrite/static_simple_select.rs b/src/webserver/database/sql/rewrite/static_simple_select.rs new file mode 100644 index 00000000..b207663a --- /dev/null +++ b/src/webserver/database/sql/rewrite/static_simple_select.rs @@ -0,0 +1,86 @@ +//! Plans static simple selects without a database query. + +use sqlparser::ast::{GroupByExpr, SelectItem, SetExpr, Statement as SqlStatement}; + +use super::super::statement::{OutputColumn, StaticSimpleSelect}; +use super::sqlpage_expression::{ + StandaloneContext, build_sqlpage_expr, is_static_simple_select_expression, +}; +use crate::webserver::database::DbInfo; + +/// Plans a documented static simple select for execution without the database. +pub(super) fn try_plan( + statement: &mut SqlStatement, + database: &DbInfo, +) -> anyhow::Result> { + if !has_static_simple_select_shape(statement) { + return Ok(None); + } + let SqlStatement::Query(query) = statement else { + return Ok(None); + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return Ok(None); + }; + for item in &select.projection { + let SelectItem::ExprWithAlias { expr, .. } = item else { + return Ok(None); + }; + if !is_static_simple_select_expression(expr)? { + return Ok(None); + } + } + + let columns = std::mem::take(&mut select.projection) + .into_iter() + .map(|item| { + let SelectItem::ExprWithAlias { expr, alias } = item else { + unreachable!("projection shape was checked") + }; + Ok(OutputColumn { + name: alias.value, + value: build_sqlpage_expr(database, &mut StandaloneContext, expr)?, + }) + }) + .collect::>>()?; + Ok(Some(StaticSimpleSelect { columns })) +} + +fn has_static_simple_select_shape(statement: &SqlStatement) -> bool { + let SqlStatement::Query(query) = statement else { + return false; + }; + if query.with.is_some() + || query.order_by.is_some() + || query.limit_clause.is_some() + || query.fetch.is_some() + || !query.locks.is_empty() + || query.for_clause.is_some() + || query.settings.is_some() + || query.format_clause.is_some() + || !query.pipe_operators.is_empty() + { + return false; + } + let SetExpr::Select(select) = query.body.as_ref() else { + return false; + }; + select.distinct.is_none() + && select.top.is_none() + && select.into.is_none() + && select.from.is_empty() + && select.lateral_views.is_empty() + && select.selection.is_none() + && select.group_by == GroupByExpr::Expressions(vec![], vec![]) + && select.cluster_by.is_empty() + && select.distribute_by.is_empty() + && select.sort_by.is_empty() + && select.having.is_none() + && select.named_window.is_empty() + && select.qualify.is_none() + && select.prewhere.is_none() + && select.connect_by.is_empty() + && select.optimizer_hints.is_empty() + && select.select_modifiers.is_none() + && select.exclude.is_none() +} diff --git a/src/webserver/database/sql/statement.rs b/src/webserver/database/sql/statement.rs index 169a3cce..76ff268d 100644 --- a/src/webserver/database/sql/statement.rs +++ b/src/webserver/database/sql/statement.rs @@ -36,7 +36,7 @@ pub(in crate::webserver::database) struct Query { #[derive(Debug, PartialEq)] pub(in crate::webserver::database) enum QueryBody { Database(DatabaseQuery), - SingleRow(SingleRowQuery), + StaticSimpleSelect(StaticSimpleSelect), } /// A statement executed by the configured database. @@ -62,9 +62,9 @@ impl DatabaseQuery { } } -/// Exactly one row generated without querying the database. +/// Execution plan for a documented static simple select. #[derive(Debug, PartialEq)] -pub(in crate::webserver::database) struct SingleRowQuery { +pub(in crate::webserver::database) struct StaticSimpleSelect { pub columns: Box<[OutputColumn]>, }