Change default key column size and make it configurable for MySQL and Oracle

core/src/main/java/com/scalar/db/common/error/CoreError.java

@@ -653,6 +653,12 @@ public enum CoreError implements ScalarDbError {
       "ScalarDB Transparent Data Encryption is not enabled. To use ScalarDB Transparent Data Encryption, you must enable it. Note that this feature is supported only in the ScalarDB Enterprise edition",
       "",
       ""),
+  INVALID_VARIABLE_KEY_COLUMN_SIZE(
+      Category.USER_ERROR,
+      "0144",
+      "The variable key column size must be greater than or equal to 64",
+      "",
+      ""),
 
   //
   // Errors for the concurrency error category

core/src/main/java/com/scalar/db/storage/jdbc/ConditionalMutator.java

@@ -46,14 +46,14 @@ public ConditionalMutator(
       Mutation mutation,
       TableMetadata tableMetadata,
       Connection connection,
-      QueryBuilder queryBuilder)
-      throws SQLException {
+      RdbEngineStrategy rdbEngine,
+      QueryBuilder queryBuilder) {
     assert mutation.getCondition().isPresent();
     this.mutation = mutation;
     this.tableMetadata = tableMetadata;
     this.connection = connection;
+    this.rdbEngine = rdbEngine;
     this.queryBuilder = queryBuilder;
-    rdbEngine = RdbEngineFactory.create(connection);
   }
 
   // For the SpotBugs warning CT_CONSTRUCTOR_THROW

core/src/main/java/com/scalar/db/storage/jdbc/JdbcConfig.java

@@ -47,6 +47,11 @@ public class JdbcConfig {
   public static final String ADMIN_CONNECTION_POOL_MAX_TOTAL =
       PREFIX + "admin.connection_pool.max_total";
 
+  public static final String MYSQL_VARIABLE_KEY_COLUMN_SIZE =
+      PREFIX + "mysql.variable_key_column_size";
+  public static final String ORACLE_VARIABLE_KEY_COLUMN_SIZE =
+      PREFIX + "oracle.variable_key_column_size";
+
   public static final int DEFAULT_CONNECTION_POOL_MIN_IDLE = 20;
   public static final int DEFAULT_CONNECTION_POOL_MAX_IDLE = 50;
   public static final int DEFAULT_CONNECTION_POOL_MAX_TOTAL = 200;
@@ -61,6 +66,22 @@ public class JdbcConfig {
   public static final int DEFAULT_ADMIN_CONNECTION_POOL_MAX_IDLE = 10;
   public static final int DEFAULT_ADMIN_CONNECTION_POOL_MAX_TOTAL = 25;
 
+  // MySQL and Oracle have limitations regarding the total size of key columns. Thus, we should set
+  // a small but enough key column size so that users can create multiple key columns without
+  // exceeding the limit and changing the default. Since we found the old default size of 64 bytes
+  // was small for some applications, we changed it based on the following specifications. See the
+  // official documents for details.
+  // 1) In MySQL, the maximum total size of key columns is 3072 bytes in the default, and thus,
+  // depending on the charset, it can be up to 768 characters long. It can further be reduced if the
+  // different settings are used.
+  // 2) In Oracle, the maximum total size of key columns is approximately 75% of the database block
+  // size minus some overhead. The default block size is 8KB, and it is typically 4kB or 8kB. Thus,
+  // the maximum size can be similar to the MySQL.
+  // See the official documents for details.
+  // https://dev.mysql.com/doc/refman/8.0/en/innodb-limits.html
+  // https://docs.oracle.com/en/database/oracle/oracle-database/23/refrn/logical-database-limits.html
+  public static final int DEFAULT_VARIABLE_KEY_COLUMN_SIZE = 128;
+
   private final String jdbcUrl;
   @Nullable private final String username;
   @Nullable private final String password;
@@ -82,6 +103,9 @@ public class JdbcConfig {
   private final int adminConnectionPoolMaxIdle;
   private final int adminConnectionPoolMaxTotal;
 
+  private final int mysqlVariableKeyColumnSize;
+  private final int oracleVariableKeyColumnSize;
+
   public JdbcConfig(DatabaseConfig databaseConfig) {
     String storage = databaseConfig.getStorage();
     String transactionManager = databaseConfig.getTransactionManager();
@@ -167,6 +191,22 @@ public JdbcConfig(DatabaseConfig databaseConfig) {
             ADMIN_CONNECTION_POOL_MAX_TOTAL,
             DEFAULT_ADMIN_CONNECTION_POOL_MAX_TOTAL);
 
+    mysqlVariableKeyColumnSize =
+        getInt(
+            databaseConfig.getProperties(),
+            MYSQL_VARIABLE_KEY_COLUMN_SIZE,
+            DEFAULT_VARIABLE_KEY_COLUMN_SIZE);
+
+    oracleVariableKeyColumnSize =
+        getInt(
+            databaseConfig.getProperties(),
+            ORACLE_VARIABLE_KEY_COLUMN_SIZE,
+            DEFAULT_VARIABLE_KEY_COLUMN_SIZE);
+
+    if (mysqlVariableKeyColumnSize < 64 || oracleVariableKeyColumnSize < 64) {
+      throw new IllegalArgumentException(CoreError.INVALID_VARIABLE_KEY_COLUMN_SIZE.buildMessage());
+    }
+
     if (databaseConfig.getProperties().containsKey(TABLE_METADATA_SCHEMA)) {
       logger.warn(
           "The configuration property \""
@@ -250,4 +290,12 @@ public int getAdminConnectionPoolMaxIdle() {
   public int getAdminConnectionPoolMaxTotal() {
     return adminConnectionPoolMaxTotal;
   }
+
+  public int getMysqlVariableKeyColumnSize() {
+    return mysqlVariableKeyColumnSize;
+  }
+
+  public int getOracleVariableKeyColumnSize() {
+    return oracleVariableKeyColumnSize;
+  }
 }

core/src/main/java/com/scalar/db/storage/jdbc/JdbcDatabase.java

@@ -18,7 +18,6 @@
 import com.scalar.db.exception.storage.ExecutionException;
 import com.scalar.db.exception.storage.NoMutationException;
 import com.scalar.db.exception.storage.RetriableExecutionException;
-import com.scalar.db.storage.jdbc.query.QueryBuilder;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.List;
@@ -60,8 +59,7 @@ public JdbcDatabase(DatabaseConfig databaseConfig) {
             databaseConfig.getMetadataCacheExpirationTimeSecs());
 
     OperationChecker operationChecker = new OperationChecker(databaseConfig, tableMetadataManager);
-    QueryBuilder queryBuilder = new QueryBuilder(rdbEngine);
-    jdbcService = new JdbcService(tableMetadataManager, operationChecker, queryBuilder);
+    jdbcService = new JdbcService(tableMetadataManager, operationChecker, rdbEngine);
   }
 
   @VisibleForTesting

core/src/main/java/com/scalar/db/storage/jdbc/JdbcService.java

@@ -2,6 +2,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.scalar.db.api.Delete;
 import com.scalar.db.api.Get;
 import com.scalar.db.api.Mutation;
@@ -41,15 +42,26 @@ public class JdbcService {
 
   private final TableMetadataManager tableMetadataManager;
   private final OperationChecker operationChecker;
+  private final RdbEngineStrategy rdbEngine;
   private final QueryBuilder queryBuilder;
 
   @SuppressFBWarnings("EI_EXPOSE_REP2")
   public JdbcService(
       TableMetadataManager tableMetadataManager,
       OperationChecker operationChecker,
+      RdbEngineStrategy rdbEngine) {
+    this(tableMetadataManager, operationChecker, rdbEngine, new QueryBuilder(rdbEngine));
+  }
+
+  @VisibleForTesting
+  JdbcService(
+      TableMetadataManager tableMetadataManager,
+      OperationChecker operationChecker,
+      RdbEngineStrategy rdbEngine,
       QueryBuilder queryBuilder) {
     this.tableMetadataManager = Objects.requireNonNull(tableMetadataManager);
     this.operationChecker = Objects.requireNonNull(operationChecker);
+    this.rdbEngine = Objects.requireNonNull(rdbEngine);
     this.queryBuilder = Objects.requireNonNull(queryBuilder);
   }
 
@@ -173,7 +185,8 @@ private boolean putInternal(Put put, Connection connection)
         return true;
       }
     } else {
-      return new ConditionalMutator(put, tableMetadata, connection, queryBuilder).mutate();
+      return new ConditionalMutator(put, tableMetadata, connection, rdbEngine, queryBuilder)
+          .mutate();
     }
   }
 
@@ -199,7 +212,8 @@ private boolean deleteInternal(Delete delete, Connection connection)
         return true;
       }
     } else {
-      return new ConditionalMutator(delete, tableMetadata, connection, queryBuilder).mutate();
+      return new ConditionalMutator(delete, tableMetadata, connection, rdbEngine, queryBuilder)
+          .mutate();
     }
   }
 

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineFactory.java

@@ -1,8 +1,6 @@
 package com.scalar.db.storage.jdbc;
 
 import com.scalar.db.common.error.CoreError;
-import java.sql.Connection;
-import java.sql.SQLException;
 
 /** Factory class of subclasses of {@link RdbEngineStrategy} */
 public final class RdbEngineFactory {
@@ -11,21 +9,14 @@ private RdbEngineFactory() {
   }
 
   public static RdbEngineStrategy create(JdbcConfig config) {
-    return create(config.getJdbcUrl());
-  }
-
-  public static RdbEngineStrategy create(Connection connection) throws SQLException {
-    String jdbcUrl = connection.getMetaData().getURL();
-    return create(jdbcUrl);
-  }
+    String jdbcUrl = config.getJdbcUrl();
 
-  static RdbEngineStrategy create(String jdbcUrl) {
     if (jdbcUrl.startsWith("jdbc:mysql:")) {
-      return new RdbEngineMysql();
+      return new RdbEngineMysql(config);
     } else if (jdbcUrl.startsWith("jdbc:postgresql:")) {
       return new RdbEnginePostgresql();
     } else if (jdbcUrl.startsWith("jdbc:oracle:")) {
-      return new RdbEngineOracle();
+      return new RdbEngineOracle(config);
     } else if (jdbcUrl.startsWith("jdbc:sqlserver:")) {
       return new RdbEngineSqlServer();
     } else if (jdbcUrl.startsWith("jdbc:sqlite:")) {

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineMysql.java

@@ -1,5 +1,6 @@
 package com.scalar.db.storage.jdbc;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.scalar.db.api.LikeExpression;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.common.error.CoreError;
@@ -21,6 +22,16 @@
 
 class RdbEngineMysql implements RdbEngineStrategy {
   private static final Logger logger = LoggerFactory.getLogger(RdbEngineMysql.class);
+  private final String keyColumnSize;
+
+  RdbEngineMysql(JdbcConfig config) {
+    keyColumnSize = String.valueOf(config.getMysqlVariableKeyColumnSize());
+  }
+
+  @VisibleForTesting
+  RdbEngineMysql() {
+    keyColumnSize = String.valueOf(JdbcConfig.DEFAULT_VARIABLE_KEY_COLUMN_SIZE);
+  }
 
   @Override
   public String[] createSchemaSqls(String fullSchema) {
@@ -201,9 +212,9 @@ public String getDataTypeForEngine(DataType scalarDbDataType) {
   public String getDataTypeForKey(DataType dataType) {
     switch (dataType) {
       case TEXT:
-        return "VARCHAR(64)";
+        return "VARCHAR(" + keyColumnSize + ")";
       case BLOB:
-        return "VARBINARY(64)";
+        return "VARBINARY(" + keyColumnSize + ")";
       default:
         return null;
     }

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineOracle.java

@@ -2,6 +2,7 @@
 
 import static com.scalar.db.util.ScalarDbUtils.getFullTableName;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.scalar.db.api.LikeExpression;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.common.error.CoreError;
@@ -23,6 +24,16 @@
 
 class RdbEngineOracle implements RdbEngineStrategy {
   private static final Logger logger = LoggerFactory.getLogger(RdbEngineOracle.class);
+  private final String keyColumnSize;
+
+  RdbEngineOracle(JdbcConfig config) {
+    keyColumnSize = String.valueOf(config.getOracleVariableKeyColumnSize());
+  }
+
+  @VisibleForTesting
+  RdbEngineOracle() {
+    keyColumnSize = String.valueOf(JdbcConfig.DEFAULT_VARIABLE_KEY_COLUMN_SIZE);
+  }
 
   @Override
   public String[] createSchemaSqls(String fullSchema) {
@@ -208,9 +219,9 @@ public String getDataTypeForEngine(DataType scalarDbDataType) {
   public String getDataTypeForKey(DataType dataType) {
     switch (dataType) {
       case TEXT:
-        return "VARCHAR2(64)";
+        return "VARCHAR2(" + keyColumnSize + ")";
       case BLOB:
-        return "RAW(64)";
+        return "RAW(" + keyColumnSize + ")";
       default:
         return null;
     }

core/src/main/java/com/scalar/db/storage/jdbc/RdbEnginePostgresql.java

@@ -204,6 +204,8 @@ public String getDataTypeForEngine(DataType scalarDbDataType) {
 
   @Override
   public String getDataTypeForKey(DataType dataType) {
+    // The number 10485760 is due to the maximum length of the character column.
+    // https://www.postgresql.org/docs/15/datatype-character.html
     if (dataType == DataType.TEXT) {
       return "VARCHAR(10485760)";
     }

core/src/main/java/com/scalar/db/transaction/jdbc/JdbcTransactionManager.java

@@ -33,7 +33,6 @@
 import com.scalar.db.storage.jdbc.JdbcUtils;
 import com.scalar.db.storage.jdbc.RdbEngineFactory;
 import com.scalar.db.storage.jdbc.RdbEngineStrategy;
-import com.scalar.db.storage.jdbc.query.QueryBuilder;
 import com.scalar.db.util.ThrowableFunction;
 import java.sql.SQLException;
 import java.util.List;
@@ -68,8 +67,7 @@ public JdbcTransactionManager(DatabaseConfig databaseConfig) {
             databaseConfig.getMetadataCacheExpirationTimeSecs());
 
     OperationChecker operationChecker = new OperationChecker(databaseConfig, tableMetadataManager);
-    QueryBuilder queryBuilder = new QueryBuilder(rdbEngine);
-    jdbcService = new JdbcService(tableMetadataManager, operationChecker, queryBuilder);
+    jdbcService = new JdbcService(tableMetadataManager, operationChecker, rdbEngine);
   }
 
   @VisibleForTesting