fix! EXPOSED-458 Stop sending default and null values in insert state… (

#2295)

* fix! EXPOSED-458 Stop sending default and null values in insert statements

documentation-website/Writerside/topics/Breaking-Changes.md

@@ -1,5 +1,30 @@
 # Breaking Changes
 
+## 0.57.0
+* Insert, Upsert, and Replace statements will no longer implicitly send all default values (except for client-side default values) in every SQL request. 
+  This change will reduce the amount of data Exposed sends to the database and make Exposed rely more on the database's default values. 
+  However, this may uncover previously hidden issues related to actual database default values, which were masked by Exposed's insert/upsert statements.
+  Also from `InsertStatement` was removed protected method `isColumnValuePreferredFromResultSet()` and method `valuesAndDefaults()` was marked as deprecated.
+
+  Let's say you have a table with columns that have default values, and you use an insert statement like this:
+  ```kotlin
+  object TestTable : IntIdTable("test") { 
+    val number = integer("number").default(100)
+    val expression = integer("exp")
+        .defaultExpression(intLiteral(100) + intLiteral(200))
+  }
+
+  TestTable.insert { }
+  ```
+  This insert statement would generate the following SQL in the H2 database:
+  ```sql
+  -- For versions before 0.57.0
+  INSERT INTO TEST ("number", "exp") VALUES (100, (100 + 200))
+
+  -- Starting from version 0.57.0
+  INSERT INTO TEST DEFAULT VALUES
+  ```
+
 ## 0.56.0
 * If the `distinct` parameter of `groupConcat()` is set to `true`, when using Oracle or SQL Server, this will now fail early with an
   `UnsupportedByDialectException`. Previously, the setting would be ignored and SQL function generation would not include a `DISTINCT` clause.

documentation-website/Writerside/topics/DSL-CRUD-operations.topic

@@ -356,6 +356,12 @@
                 it[director] = "Rian Johnson"
             }
         </code-block>
+
+        <note>
+            If the <code>insertValue()</code> for a particular column is used within the <code>onUpdate</code> block,
+            that column must be defined in the <code>insert</code> section, unless it has a client side default value.
+        </note>
+
         <p>If the update operation should be identical to the insert operation except for a few columns,
             then <code>onUpdateExclude</code> should be provided an argument with the specific columns to exclude.
             This parameter could also be used for the reverse case when only a small subset of columns should be updated

exposed-core/api/exposed-core.api

@@ -3093,7 +3093,6 @@ public abstract class org/jetbrains/exposed/sql/statements/BaseBatchInsertStatem
 	public fun prepared (Lorg/jetbrains/exposed/sql/Transaction;Ljava/lang/String;)Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;
 	public fun set (Lorg/jetbrains/exposed/sql/Column;Ljava/lang/Object;)V
 	public fun setArguments (Ljava/util/List;)V
-	protected fun valuesAndDefaults (Ljava/util/Map;)Ljava/util/Map;
 }
 
 public final class org/jetbrains/exposed/sql/statements/BatchDataInconsistentException : java/lang/Exception {
@@ -3136,7 +3135,6 @@ public class org/jetbrains/exposed/sql/statements/BatchUpsertStatement : org/jet
 	public final fun getOnUpdateExclude ()Ljava/util/List;
 	public final fun getWhere ()Lorg/jetbrains/exposed/sql/Op;
 	public fun insertValue (Lorg/jetbrains/exposed/sql/Column;)Lorg/jetbrains/exposed/sql/ExpressionWithColumnType;
-	protected fun isColumnValuePreferredFromResultSet (Lorg/jetbrains/exposed/sql/Column;Ljava/lang/Object;)Z
 	public fun prepareSQL (Lorg/jetbrains/exposed/sql/Transaction;Z)Ljava/lang/String;
 	public fun prepared (Lorg/jetbrains/exposed/sql/Transaction;Ljava/lang/String;)Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;
 	public fun storeUpdateValues (Lkotlin/jvm/functions/Function2;)V
@@ -3199,6 +3197,7 @@ public class org/jetbrains/exposed/sql/statements/InsertStatement : org/jetbrain
 	public synthetic fun <init> (Lorg/jetbrains/exposed/sql/Table;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public synthetic fun arguments ()Ljava/lang/Iterable;
 	public fun arguments ()Ljava/util/List;
+	protected final fun clientDefaultColumns ()Ljava/util/List;
 	protected fun execInsertFunction (Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;)Lkotlin/Pair;
 	public fun executeInternal (Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;Lorg/jetbrains/exposed/sql/Transaction;)Ljava/lang/Integer;
 	public synthetic fun executeInternal (Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;Lorg/jetbrains/exposed/sql/Transaction;)Ljava/lang/Object;
@@ -3218,6 +3217,8 @@ public class org/jetbrains/exposed/sql/statements/InsertStatement : org/jetbrain
 	public fun setArguments (Ljava/util/List;)V
 	public final fun setInsertedCount (I)V
 	protected final fun toSqlString (Ljava/util/List;Z)Ljava/lang/String;
+	protected final fun valuesAndClientDefaults (Ljava/util/Map;)Ljava/util/Map;
+	public static synthetic fun valuesAndClientDefaults$default (Lorg/jetbrains/exposed/sql/statements/InsertStatement;Ljava/util/Map;ILjava/lang/Object;)Ljava/util/Map;
 	protected fun valuesAndDefaults (Ljava/util/Map;)Ljava/util/Map;
 	public static synthetic fun valuesAndDefaults$default (Lorg/jetbrains/exposed/sql/statements/InsertStatement;Ljava/util/Map;ILjava/lang/Object;)Ljava/util/Map;
 }
@@ -3484,7 +3485,6 @@ public class org/jetbrains/exposed/sql/statements/UpsertStatement : org/jetbrain
 	public final fun getOnUpdateExclude ()Ljava/util/List;
 	public final fun getWhere ()Lorg/jetbrains/exposed/sql/Op;
 	public fun insertValue (Lorg/jetbrains/exposed/sql/Column;)Lorg/jetbrains/exposed/sql/ExpressionWithColumnType;
-	protected fun isColumnValuePreferredFromResultSet (Lorg/jetbrains/exposed/sql/Column;Ljava/lang/Object;)Z
 	public fun prepareSQL (Lorg/jetbrains/exposed/sql/Transaction;Z)Ljava/lang/String;
 	public fun prepared (Lorg/jetbrains/exposed/sql/Transaction;Ljava/lang/String;)Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;
 	public fun storeUpdateValues (Lkotlin/jvm/functions/Function2;)V

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Column.kt

@@ -126,7 +126,7 @@ class Column<T>(
             val expressionSQL = currentDialect.dataTypeProvider.processForDefaultValue(defaultValue)
             if (!currentDialect.isAllowedAsColumnDefault(defaultValue)) {
                 val clientDefault = when {
-                    defaultValueFun != null -> " Expression will be evaluated on the client."
+                    defaultValueFun != null && dbDefaultValue == null -> " Expression will be evaluated on the client."
                     !columnType.nullable -> " Column will be created with NULL marker."
                     else -> ""
                 }

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/ColumnType.kt

@@ -374,6 +374,10 @@ open class ColumnWithTransform<Unwrapped, Wrapped>(
     }
 }
 
+internal fun unwrapColumnValues(values: Map<Column<*>, Any?>): Map<Column<*>, Any?> = values.mapValues { (col, value) ->
+    value?.let { (col.columnType as? ColumnWithTransform<Any, Any>)?.unwrapRecursive(it) } ?: value
+}
+
 /**
  * A class that handles the transformation between a source column type and a target type,
  * but also supports transformations involving `null` values.
@@ -1014,7 +1018,11 @@ class BlobColumnType(
         else -> error("Unexpected value of type Blob: $value of ${value::class.qualifiedName}")
     }
 
-    override fun nonNullValueToString(value: ExposedBlob): String = currentDialect.dataTypeProvider.hexToDb(value.hexString())
+    override fun nonNullValueToString(value: ExposedBlob): String {
+        // For H2 Blobs the original dataTypeProvider must be taken (even if H2 in other DB mode)
+        return ((currentDialect as? H2Dialect)?.originalDataTypeProvider ?: currentDialect.dataTypeProvider)
+            .hexToDb(value.hexString())
+    }
 
     override fun readObject(rs: ResultSet, index: Int) = when {
         currentDialect is SQLServerDialect -> rs.getBytes(index)?.let(::ExposedBlob)

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/statements/BaseBatchInsertStatement.kt

@@ -1,11 +1,6 @@
 package org.jetbrains.exposed.sql.statements
 
-import org.jetbrains.exposed.sql.Column
-import org.jetbrains.exposed.sql.EntityIDColumnType
-import org.jetbrains.exposed.sql.ResultRow
-import org.jetbrains.exposed.sql.Table
-import org.jetbrains.exposed.sql.Transaction
-import org.jetbrains.exposed.sql.isAutoInc
+import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.statements.api.PreparedStatementApi
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 
@@ -88,18 +83,26 @@ abstract class BaseBatchInsertStatement(
 
     override var arguments: List<List<Pair<Column<*>, Any?>>>? = null
         get() = field ?: run {
-            val nullableColumns by lazy {
-                allColumnsInDataSet().filter { it.columnType.nullable && !it.isDatabaseGenerated }
-            }
-            data.map { single ->
-                val valuesAndDefaults = super.valuesAndDefaults(single) as MutableMap
-                val nullableMap = (nullableColumns - valuesAndDefaults.keys).associateWith { null }
-                valuesAndDefaults.putAll(nullableMap)
-                valuesAndDefaults.toList().sortedBy { it.first }
-            }.apply { field = this }
-        }
+            val columnsToInsert = (allColumnsInDataSet() + clientDefaultColumns()).toSet()
 
-    override fun valuesAndDefaults(values: Map<Column<*>, Any?>) = arguments!!.first().toMap()
+            data
+                .map { valuesAndClientDefaults(it) as MutableMap }
+                .map { values ->
+                    columnsToInsert.map { column ->
+                        column to when {
+                            values.contains(column) -> values[column]
+                            column.dbDefaultValue != null || column.isDatabaseGenerated -> DefaultValueMarker
+                            else -> {
+                                require(column.columnType.nullable) {
+                                    "The value for the column ${column.name} was not provided. " +
+                                        "The value for non-nullable column without defaults must be specified."
+                                }
+                                null
+                            }
+                        }
+                    }
+                }.apply { field = this }
+        }
 
     override fun prepared(transaction: Transaction, sql: String): PreparedStatementApi {
         return if (!shouldReturnGeneratedValues) {

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/statements/BatchUpsertStatement.kt

@@ -3,6 +3,7 @@ package org.jetbrains.exposed.sql.statements
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.statements.api.PreparedStatementApi
 import org.jetbrains.exposed.sql.vendors.MysqlFunctionProvider
+import org.jetbrains.exposed.sql.vendors.OracleDialect
 import org.jetbrains.exposed.sql.vendors.currentDialect
 
 /**
@@ -54,8 +55,9 @@ open class BatchUpsertStatement(
         val keyColumns = if (functionProvider is MysqlFunctionProvider) keys.toList() else getKeyColumns(keys = keys)
         val insertValues = arguments!!.first()
         val insertValuesSql = insertValues.toSqlString(prepared)
+        val updateExcludeColumns = (onUpdateExclude ?: emptyList()) + if (dialect is OracleDialect) keyColumns else emptyList()
         val updateExpressions = updateValues.takeIf { it.isNotEmpty() }?.toList()
-            ?: getUpdateExpressions(insertValues.unzip().first, onUpdateExclude, keyColumns)
+            ?: getUpdateExpressions(insertValues.unzip().first, updateExcludeColumns, keyColumns)
         return functionProvider.upsert(table, insertValues, insertValuesSql, updateExpressions, keyColumns, where, transaction)
     }
 
@@ -74,9 +76,4 @@ open class BatchUpsertStatement(
 
         return super.prepared(transaction, sql)
     }
-
-    override fun isColumnValuePreferredFromResultSet(column: Column<*>, value: Any?): Boolean {
-        return isEntityIdClientSideGeneratedUUID(column) ||
-            super.isColumnValuePreferredFromResultSet(column, value)
-    }
 }