From f2bdae14ed1a13c35ebc8df640de78492fd28f41 Mon Sep 17 00:00:00 2001 From: Alexander Soklakov Date: Thu, 20 May 2021 16:56:02 +0400 Subject: [PATCH] Fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED WITH CUSTOM COLLATION. Fix for Bug#100606 (31818423), UNECESARY CALL TO "SET NAMES 'UTF8' COLLATE 'UTF8_GENERAL_CI'". --- CHANGES | 5 + .../java/com/mysql/cj/CharsetMapping.java | 344 ++++----- .../java/com/mysql/cj/CharsetSettings.java | 114 +++ .../core-api/java/com/mysql/cj/Constants.java | 4 +- .../mysql/cj/conf/PropertyDefinitions.java | 3 + .../java/com/mysql/cj/conf/PropertyKey.java | 1 + .../cj/protocol/AuthenticationProvider.java | 42 +- .../java/com/mysql/cj/protocol/Protocol.java | 4 +- .../mysql/cj/protocol/ServerCapabilities.java | 7 +- .../com/mysql/cj/protocol/ServerSession.java | 66 +- .../java/com/mysql/cj/result/Field.java | 10 +- .../mysql/cj/ClientPreparedQueryBindings.java | 5 +- .../java/com/mysql/cj/CoreSession.java | 4 +- .../com/mysql/cj/NativeCharsetSettings.java | 724 ++++++++++++++++++ .../java/com/mysql/cj/NativeSession.java | 421 +--------- .../com/mysql/cj/ServerPreparedQuery.java | 2 +- .../result/AbstractDateTimeValueFactory.java | 3 +- .../result/AbstractNumericValueFactory.java | 3 +- .../mysql/cj/result/BooleanValueFactory.java | 3 +- .../com/mysql/cj/result/ByteValueFactory.java | 3 +- .../cj/result/OffsetDateTimeValueFactory.java | 1 - .../cj/result/OffsetTimeValueFactory.java | 1 - .../cj/result/ZonedDateTimeValueFactory.java | 1 - .../cj/protocol/a/BinaryResultsetReader.java | 5 +- .../cj/protocol/a/ColumnDefinitionReader.java | 6 +- .../a/NativeAuthenticationProvider.java | 46 +- .../cj/protocol/a/NativeCapabilities.java | 61 +- .../mysql/cj/protocol/a/NativeProtocol.java | 100 +-- .../cj/protocol/a/NativeServerSession.java | 218 +----- .../cj/protocol/a/TextResultsetReader.java | 5 +- .../CachingSha2PasswordPlugin.java | 8 +- .../MysqlClearPasswordPlugin.java | 4 +- .../MysqlNativePasswordPlugin.java | 6 +- .../MysqlOldPasswordPlugin.java | 6 +- .../authentication/Sha256PasswordPlugin.java | 7 +- .../com/mysql/cj/protocol/x/FieldFactory.java | 6 +- .../protocol/x/XAuthenticationProvider.java | 13 +- .../com/mysql/cj/protocol/x/XProtocol.java | 22 +- .../cj/protocol/x/XServerCapabilities.java | 21 +- .../mysql/cj/protocol/x/XServerSession.java | 105 +-- .../cj/LocalizedErrorMessages.properties | 19 +- .../com/mysql/cj/jdbc/CallableStatement.java | 6 +- .../com/mysql/cj/jdbc/ConnectionImpl.java | 31 +- .../mysql/cj/jdbc/ParameterBindingsImpl.java | 4 +- .../java/com/mysql/cj/jdbc/StatementImpl.java | 15 +- .../mysql/cj/jdbc/result/ResultSetImpl.java | 2 +- .../cj/jdbc/result/ResultSetMetaData.java | 23 +- .../java/com/mysql/cj/xdevapi/ColumnImpl.java | 6 +- .../com/mysql/cj/CharsetMappingWrapper.java | 70 ++ src/test/java/com/mysql/cj/MessagesTest.java | 7 +- .../regression/CharsetRegressionTest.java | 256 ++++++- .../regression/ConnectionRegressionTest.java | 85 +- .../regression/MetaDataRegressionTest.java | 16 +- .../regression/StatementRegressionTest.java | 6 +- .../regression/SyntaxRegressionTest.java | 4 +- .../java/testsuite/simple/CharsetTest.java | 66 +- .../java/testsuite/simple/ConnectionTest.java | 4 +- .../java/testsuite/simple/ResultSetTest.java | 11 +- .../java/testsuite/simple/StatementsTest.java | 6 +- .../java/testsuite/x/devapi/SessionTest.java | 8 +- 60 files changed, 1606 insertions(+), 1449 deletions(-) create mode 100644 src/main/core-api/java/com/mysql/cj/CharsetSettings.java create mode 100644 src/main/core-impl/java/com/mysql/cj/NativeCharsetSettings.java create mode 100644 src/test/java/com/mysql/cj/CharsetMappingWrapper.java diff --git a/CHANGES b/CHANGES index 94f59acbf..8047d6bed 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,11 @@ Version 8.0.26 + - Fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED WITH CUSTOM COLLATION. + + - Fix for Bug#100606 (31818423), UNECESARY CALL TO "SET NAMES 'UTF8' COLLATE 'UTF8_GENERAL_CI'". + Thanks to Marc Fletcher for his contribution. + - Fix for Bug#102404 (32435618), CONTRIBUTION: ADD TRACK SESSION STATE CHANGE. Thanks to William Lee for his contribution. diff --git a/src/main/core-api/java/com/mysql/cj/CharsetMapping.java b/src/main/core-api/java/com/mysql/cj/CharsetMapping.java index 09c2c469d..8f3c7bba5 100644 --- a/src/main/core-api/java/com/mysql/cj/CharsetMapping.java +++ b/src/main/core-api/java/com/mysql/cj/CharsetMapping.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. + * Copyright (c) 2002, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -34,7 +34,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -47,69 +46,69 @@ */ public class CharsetMapping { - public static final int MAP_SIZE = 2048; // Size of static maps - public static final String[] COLLATION_INDEX_TO_COLLATION_NAME; - public static final MysqlCharset[] COLLATION_INDEX_TO_CHARSET; + public static final int MAP_SIZE = 1024; // Size of static maps + private static final String[] COLLATION_INDEX_TO_COLLATION_NAME; + private static final Map COLLATION_INDEX_TO_CHARSET; - public static final Map CHARSET_NAME_TO_CHARSET; - public static final Map CHARSET_NAME_TO_COLLATION_INDEX; + private static final Map CHARSET_NAME_TO_CHARSET; + private static final Map CHARSET_NAME_TO_COLLATION_INDEX; + private static final Map COLLATION_NAME_TO_COLLATION_INDEX; private static final Map> JAVA_ENCODING_UC_TO_MYSQL_CHARSET; private static final Set MULTIBYTE_ENCODINGS; - public static final Set UTF8MB4_INDEXES; - - private static final String MYSQL_CHARSET_NAME_armscii8 = "armscii8"; - private static final String MYSQL_CHARSET_NAME_ascii = "ascii"; - private static final String MYSQL_CHARSET_NAME_big5 = "big5"; - private static final String MYSQL_CHARSET_NAME_binary = "binary"; - private static final String MYSQL_CHARSET_NAME_cp1250 = "cp1250"; - private static final String MYSQL_CHARSET_NAME_cp1251 = "cp1251"; - private static final String MYSQL_CHARSET_NAME_cp1256 = "cp1256"; - private static final String MYSQL_CHARSET_NAME_cp1257 = "cp1257"; - private static final String MYSQL_CHARSET_NAME_cp850 = "cp850"; - private static final String MYSQL_CHARSET_NAME_cp852 = "cp852"; - private static final String MYSQL_CHARSET_NAME_cp866 = "cp866"; - private static final String MYSQL_CHARSET_NAME_cp932 = "cp932"; - private static final String MYSQL_CHARSET_NAME_dec8 = "dec8"; - private static final String MYSQL_CHARSET_NAME_eucjpms = "eucjpms"; - private static final String MYSQL_CHARSET_NAME_euckr = "euckr"; - private static final String MYSQL_CHARSET_NAME_gb18030 = "gb18030"; - private static final String MYSQL_CHARSET_NAME_gb2312 = "gb2312"; - private static final String MYSQL_CHARSET_NAME_gbk = "gbk"; - private static final String MYSQL_CHARSET_NAME_geostd8 = "geostd8"; - private static final String MYSQL_CHARSET_NAME_greek = "greek"; - private static final String MYSQL_CHARSET_NAME_hebrew = "hebrew"; - private static final String MYSQL_CHARSET_NAME_hp8 = "hp8"; - private static final String MYSQL_CHARSET_NAME_keybcs2 = "keybcs2"; - private static final String MYSQL_CHARSET_NAME_koi8r = "koi8r"; - private static final String MYSQL_CHARSET_NAME_koi8u = "koi8u"; - private static final String MYSQL_CHARSET_NAME_latin1 = "latin1"; - private static final String MYSQL_CHARSET_NAME_latin2 = "latin2"; - private static final String MYSQL_CHARSET_NAME_latin5 = "latin5"; - private static final String MYSQL_CHARSET_NAME_latin7 = "latin7"; - private static final String MYSQL_CHARSET_NAME_macce = "macce"; - private static final String MYSQL_CHARSET_NAME_macroman = "macroman"; - private static final String MYSQL_CHARSET_NAME_sjis = "sjis"; - private static final String MYSQL_CHARSET_NAME_swe7 = "swe7"; - private static final String MYSQL_CHARSET_NAME_tis620 = "tis620"; - private static final String MYSQL_CHARSET_NAME_ucs2 = "ucs2"; - private static final String MYSQL_CHARSET_NAME_ujis = "ujis"; - private static final String MYSQL_CHARSET_NAME_utf16 = "utf16"; - private static final String MYSQL_CHARSET_NAME_utf16le = "utf16le"; - private static final String MYSQL_CHARSET_NAME_utf32 = "utf32"; - private static final String MYSQL_CHARSET_NAME_utf8 = "utf8"; - private static final String MYSQL_CHARSET_NAME_utf8mb4 = "utf8mb4"; - - public static final String NOT_USED = MYSQL_CHARSET_NAME_latin1; // punting for not-used character sets - public static final String COLLATION_NOT_DEFINED = "none"; - - public static final int MYSQL_COLLATION_INDEX_utf8 = 33; + /** + * Indexes of collations using ucs2, utf16, utf16le or utf32 character sets and that cannot be set to character_set_client system variable. + */ + private static final Set IMPERMISSIBLE_INDEXES; + + public static final String MYSQL_CHARSET_NAME_armscii8 = "armscii8"; + public static final String MYSQL_CHARSET_NAME_ascii = "ascii"; + public static final String MYSQL_CHARSET_NAME_big5 = "big5"; + public static final String MYSQL_CHARSET_NAME_binary = "binary"; + public static final String MYSQL_CHARSET_NAME_cp1250 = "cp1250"; + public static final String MYSQL_CHARSET_NAME_cp1251 = "cp1251"; + public static final String MYSQL_CHARSET_NAME_cp1256 = "cp1256"; + public static final String MYSQL_CHARSET_NAME_cp1257 = "cp1257"; + public static final String MYSQL_CHARSET_NAME_cp850 = "cp850"; + public static final String MYSQL_CHARSET_NAME_cp852 = "cp852"; + public static final String MYSQL_CHARSET_NAME_cp866 = "cp866"; + public static final String MYSQL_CHARSET_NAME_cp932 = "cp932"; + public static final String MYSQL_CHARSET_NAME_dec8 = "dec8"; + public static final String MYSQL_CHARSET_NAME_eucjpms = "eucjpms"; + public static final String MYSQL_CHARSET_NAME_euckr = "euckr"; + public static final String MYSQL_CHARSET_NAME_gb18030 = "gb18030"; + public static final String MYSQL_CHARSET_NAME_gb2312 = "gb2312"; + public static final String MYSQL_CHARSET_NAME_gbk = "gbk"; + public static final String MYSQL_CHARSET_NAME_geostd8 = "geostd8"; + public static final String MYSQL_CHARSET_NAME_greek = "greek"; + public static final String MYSQL_CHARSET_NAME_hebrew = "hebrew"; + public static final String MYSQL_CHARSET_NAME_hp8 = "hp8"; + public static final String MYSQL_CHARSET_NAME_keybcs2 = "keybcs2"; + public static final String MYSQL_CHARSET_NAME_koi8r = "koi8r"; + public static final String MYSQL_CHARSET_NAME_koi8u = "koi8u"; + public static final String MYSQL_CHARSET_NAME_latin1 = "latin1"; + public static final String MYSQL_CHARSET_NAME_latin2 = "latin2"; + public static final String MYSQL_CHARSET_NAME_latin5 = "latin5"; + public static final String MYSQL_CHARSET_NAME_latin7 = "latin7"; + public static final String MYSQL_CHARSET_NAME_macce = "macce"; + public static final String MYSQL_CHARSET_NAME_macroman = "macroman"; + public static final String MYSQL_CHARSET_NAME_sjis = "sjis"; + public static final String MYSQL_CHARSET_NAME_swe7 = "swe7"; + public static final String MYSQL_CHARSET_NAME_tis620 = "tis620"; + public static final String MYSQL_CHARSET_NAME_ucs2 = "ucs2"; + public static final String MYSQL_CHARSET_NAME_ujis = "ujis"; + public static final String MYSQL_CHARSET_NAME_utf16 = "utf16"; + public static final String MYSQL_CHARSET_NAME_utf16le = "utf16le"; + public static final String MYSQL_CHARSET_NAME_utf32 = "utf32"; + public static final String MYSQL_CHARSET_NAME_utf8 = "utf8"; + public static final String MYSQL_CHARSET_NAME_utf8mb4 = "utf8mb4"; + + public static final int MYSQL_COLLATION_INDEX_utf8mb4_general_ci = 45; + public static final int MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci = 255; public static final int MYSQL_COLLATION_INDEX_binary = 63; - private static int numberOfEncodingsConfigured = 0; - static { // complete list of mysql character sets and their corresponding java encoding names MysqlCharset[] charset = new MysqlCharset[] { new MysqlCharset(MYSQL_CHARSET_NAME_ascii, 1, 0, new String[] { "US-ASCII", "ASCII" }), @@ -117,36 +116,33 @@ public class CharsetMapping { new MysqlCharset(MYSQL_CHARSET_NAME_big5, 2, 0, new String[] { "Big5" }), new MysqlCharset(MYSQL_CHARSET_NAME_gbk, 2, 0, new String[] { "GBK" }), - new MysqlCharset(MYSQL_CHARSET_NAME_sjis, 2, 0, new String[] { "SHIFT_JIS", "Cp943", "WINDOWS-31J" }), // SJIS is alias for SHIFT_JIS, Cp943 is rather a cp932 but we map it to sjis for years - new MysqlCharset(MYSQL_CHARSET_NAME_cp932, 2, 1, new String[] { "WINDOWS-31J" }), // MS932 is alias for WINDOWS-31J + new MysqlCharset(MYSQL_CHARSET_NAME_sjis, 2, 0, new String[] { "SHIFT_JIS", "Cp943", "WINDOWS-31J" }), // SJIS is alias for SHIFT_JIS, Cp943 is rather a cp932 but we map it to sjis for years + new MysqlCharset(MYSQL_CHARSET_NAME_cp932, 2, 1, new String[] { "WINDOWS-31J" }), new MysqlCharset(MYSQL_CHARSET_NAME_gb2312, 2, 0, new String[] { "GB2312" }), new MysqlCharset(MYSQL_CHARSET_NAME_ujis, 3, 0, new String[] { "EUC_JP" }), - new MysqlCharset(MYSQL_CHARSET_NAME_eucjpms, 3, 0, new String[] { "EUC_JP_Solaris" }, new ServerVersion(5, 0, 3)), // "EUC_JP_Solaris = >5.0.3 eucjpms," + new MysqlCharset(MYSQL_CHARSET_NAME_eucjpms, 3, 0, new String[] { "EUC_JP_Solaris" }, new ServerVersion(5, 0, 3)), new MysqlCharset(MYSQL_CHARSET_NAME_gb18030, 4, 0, new String[] { "GB18030" }, new ServerVersion(5, 7, 4)), new MysqlCharset(MYSQL_CHARSET_NAME_euckr, 2, 0, new String[] { "EUC-KR" }), new MysqlCharset(MYSQL_CHARSET_NAME_latin1, 1, 1, new String[] { "Cp1252", "ISO8859_1" }), - new MysqlCharset(MYSQL_CHARSET_NAME_swe7, 1, 0, new String[] { "Cp1252" }), // new mapping, Cp1252 ? - new MysqlCharset(MYSQL_CHARSET_NAME_hp8, 1, 0, new String[] { "Cp1252" }), // new mapping, Cp1252 ? - new MysqlCharset(MYSQL_CHARSET_NAME_dec8, 1, 0, new String[] { "Cp1252" }), // new mapping, Cp1252 ? - new MysqlCharset(MYSQL_CHARSET_NAME_armscii8, 1, 0, new String[] { "Cp1252" }), // new mapping, Cp1252 ? - new MysqlCharset(MYSQL_CHARSET_NAME_geostd8, 1, 0, new String[] { "Cp1252" }), // new mapping, Cp1252 ? - - new MysqlCharset(MYSQL_CHARSET_NAME_latin2, 1, 0, new String[] { "ISO8859_2" }), // latin2 is an alias + new MysqlCharset(MYSQL_CHARSET_NAME_swe7, 1, 0, new String[] { "Cp1252" }), + new MysqlCharset(MYSQL_CHARSET_NAME_hp8, 1, 0, new String[] { "Cp1252" }), + new MysqlCharset(MYSQL_CHARSET_NAME_dec8, 1, 0, new String[] { "Cp1252" }), + new MysqlCharset(MYSQL_CHARSET_NAME_armscii8, 1, 0, new String[] { "Cp1252" }), + new MysqlCharset(MYSQL_CHARSET_NAME_geostd8, 1, 0, new String[] { "Cp1252" }), + new MysqlCharset(MYSQL_CHARSET_NAME_latin2, 1, 0, new String[] { "ISO8859_2" }), new MysqlCharset(MYSQL_CHARSET_NAME_greek, 1, 0, new String[] { "ISO8859_7", "greek" }), - new MysqlCharset(MYSQL_CHARSET_NAME_latin7, 1, 0, new String[] { "ISO-8859-13" }), // was ISO8859_7, that's incorrect; also + "LATIN7 = latin7," is wrong java encoding name - - new MysqlCharset(MYSQL_CHARSET_NAME_hebrew, 1, 0, new String[] { "ISO8859_8" }), // hebrew is an alias - new MysqlCharset(MYSQL_CHARSET_NAME_latin5, 1, 0, new String[] { "ISO8859_9" }), // LATIN5 is an alias - + new MysqlCharset(MYSQL_CHARSET_NAME_latin7, 1, 0, new String[] { "ISO-8859-13" }), + new MysqlCharset(MYSQL_CHARSET_NAME_hebrew, 1, 0, new String[] { "ISO8859_8" }), + new MysqlCharset(MYSQL_CHARSET_NAME_latin5, 1, 0, new String[] { "ISO8859_9" }), new MysqlCharset(MYSQL_CHARSET_NAME_cp850, 1, 0, new String[] { "Cp850", "Cp437" }), new MysqlCharset(MYSQL_CHARSET_NAME_cp852, 1, 0, new String[] { "Cp852" }), - new MysqlCharset(MYSQL_CHARSET_NAME_keybcs2, 1, 0, new String[] { "Cp852" }), // new, Kamenicky encoding usually known as Cp895 but there is no official cp895 specification; close to Cp852, see http://ftp.muni.cz/pub/localization/charsets/cs-encodings-faq + new MysqlCharset(MYSQL_CHARSET_NAME_keybcs2, 1, 0, new String[] { "Cp852" }), // Kamenicky encoding usually known as Cp895 but there is no official cp895 specification; close to Cp852, see http://ftp.muni.cz/pub/localization/charsets/cs-encodings-faq new MysqlCharset(MYSQL_CHARSET_NAME_cp866, 1, 0, new String[] { "Cp866" }), @@ -154,23 +150,19 @@ public class CharsetMapping { new MysqlCharset(MYSQL_CHARSET_NAME_koi8u, 1, 0, new String[] { "KOI8_R" }), new MysqlCharset(MYSQL_CHARSET_NAME_tis620, 1, 0, new String[] { "TIS620" }), - new MysqlCharset(MYSQL_CHARSET_NAME_cp1250, 1, 0, new String[] { "Cp1250" }), - new MysqlCharset(MYSQL_CHARSET_NAME_cp1251, 1, 1, new String[] { "Cp1251" }), - new MysqlCharset(MYSQL_CHARSET_NAME_cp1256, 1, 0, new String[] { "Cp1256" }), new MysqlCharset(MYSQL_CHARSET_NAME_cp1257, 1, 0, new String[] { "Cp1257" }), new MysqlCharset(MYSQL_CHARSET_NAME_macroman, 1, 0, new String[] { "MacRoman" }), new MysqlCharset(MYSQL_CHARSET_NAME_macce, 1, 0, new String[] { "MacCentralEurope" }), - new MysqlCharset(MYSQL_CHARSET_NAME_utf8, 3, 1, new String[] { "UTF-8" }), - new MysqlCharset(MYSQL_CHARSET_NAME_utf8mb4, 4, 0, new String[] { "UTF-8" }), // "UTF-8 = *> 5.5.2 utf8mb4," - - new MysqlCharset(MYSQL_CHARSET_NAME_ucs2, 2, 0, new String[] { "UnicodeBig" }), + new MysqlCharset(MYSQL_CHARSET_NAME_utf8, 3, 0, new String[] { "UTF-8" }), + new MysqlCharset(MYSQL_CHARSET_NAME_utf8mb4, 4, 1, new String[] { "UTF-8" }), // "UTF-8 = *> 5.5.2 utf8mb4" - new MysqlCharset(MYSQL_CHARSET_NAME_binary, 1, 1, new String[] { "ISO8859_1" }), // US-ASCII ? + new MysqlCharset(MYSQL_CHARSET_NAME_binary, 1, 1, new String[] { "ISO8859_1" }), + new MysqlCharset(MYSQL_CHARSET_NAME_ucs2, 2, 0, new String[] { "UnicodeBig" }), new MysqlCharset(MYSQL_CHARSET_NAME_utf16, 4, 0, new String[] { "UTF-16" }), new MysqlCharset(MYSQL_CHARSET_NAME_utf16le, 4, 0, new String[] { "UTF-16LE" }), new MysqlCharset(MYSQL_CHARSET_NAME_utf32, 4, 0, new String[] { "UTF-32" }) @@ -178,17 +170,13 @@ public class CharsetMapping { }; HashMap charsetNameToMysqlCharsetMap = new HashMap<>(); HashMap> javaUcToMysqlCharsetMap = new HashMap<>(); - Set tempMultibyteEncodings = new HashSet<>(); // Character sets that we can't convert ourselves. + Set tempMultibyteEncodings = new HashSet<>(); + for (int i = 0; i < charset.length; i++) { String charsetName = charset[i].charsetName; - charsetNameToMysqlCharsetMap.put(charsetName, charset[i]); - numberOfEncodingsConfigured += charset[i].javaEncodingsUc.size(); - for (String encUC : charset[i].javaEncodingsUc) { - - // fill javaUcToMysqlCharsetMap List charsets = javaUcToMysqlCharsetMap.get(encUC); if (charsets == null) { charsets = new ArrayList<>(); @@ -196,13 +184,10 @@ public class CharsetMapping { } charsets.add(charset[i]); - // fill multi-byte charsets if (charset[i].mblen > 1) { tempMultibyteEncodings.add(encUC); } - } - } CHARSET_NAME_TO_CHARSET = Collections.unmodifiableMap(charsetNameToMysqlCharsetMap); JAVA_ENCODING_UC_TO_MYSQL_CHARSET = Collections.unmodifiableMap(javaUcToMysqlCharsetMap); @@ -463,7 +448,6 @@ public class CharsetMapping { collation[275] = new Collation(275, "utf8mb4_hr_0900_ai_ci", 0, MYSQL_CHARSET_NAME_utf8mb4); collation[277] = new Collation(277, "utf8mb4_vi_0900_ai_ci", 0, MYSQL_CHARSET_NAME_utf8mb4); - collation[278] = new Collation(278, "utf8mb4_0900_as_cs", 0, MYSQL_CHARSET_NAME_utf8mb4); collation[279] = new Collation(279, "utf8mb4_de_pb_0900_as_cs", 0, MYSQL_CHARSET_NAME_utf8mb4); collation[280] = new Collation(280, "utf8mb4_is_0900_as_cs", 0, MYSQL_CHARSET_NAME_utf8mb4); @@ -496,93 +480,68 @@ public class CharsetMapping { collation[308] = new Collation(308, "utf8mb4_zh_0900_as_cs", 0, MYSQL_CHARSET_NAME_utf8mb4); collation[309] = new Collation(309, "utf8mb4_0900_bin", 0, MYSQL_CHARSET_NAME_utf8mb4); - collation[326] = new Collation(326, "utf8mb4_test_ci", 0, MYSQL_CHARSET_NAME_utf8mb4); - collation[327] = new Collation(327, "utf16_test_ci", 0, MYSQL_CHARSET_NAME_utf16); - collation[328] = new Collation(328, "utf8mb4_test_400_ci", 0, MYSQL_CHARSET_NAME_utf8mb4); - - collation[336] = new Collation(336, "utf8_bengali_standard_ci", 0, MYSQL_CHARSET_NAME_utf8); - collation[337] = new Collation(337, "utf8_bengali_traditional_ci", 0, MYSQL_CHARSET_NAME_utf8); - - collation[352] = new Collation(352, "utf8_phone_ci", 0, MYSQL_CHARSET_NAME_utf8); - collation[353] = new Collation(353, "utf8_test_ci", 0, MYSQL_CHARSET_NAME_utf8); - collation[354] = new Collation(354, "utf8_5624_1", 0, MYSQL_CHARSET_NAME_utf8); - collation[355] = new Collation(355, "utf8_5624_2", 0, MYSQL_CHARSET_NAME_utf8); - collation[356] = new Collation(356, "utf8_5624_3", 0, MYSQL_CHARSET_NAME_utf8); - collation[357] = new Collation(357, "utf8_5624_4", 0, MYSQL_CHARSET_NAME_utf8); - collation[358] = new Collation(358, "ucs2_test_ci", 0, MYSQL_CHARSET_NAME_ucs2); - collation[359] = new Collation(359, "ucs2_vn_ci", 0, MYSQL_CHARSET_NAME_ucs2); - collation[360] = new Collation(360, "ucs2_5624_1", 0, MYSQL_CHARSET_NAME_ucs2); - - collation[368] = new Collation(368, "utf8_5624_5", 0, MYSQL_CHARSET_NAME_utf8); - collation[391] = new Collation(391, "utf32_test_ci", 0, MYSQL_CHARSET_NAME_utf32); - collation[2047] = new Collation(2047, "utf8_maxuserid_ci", 0, MYSQL_CHARSET_NAME_utf8); - COLLATION_INDEX_TO_COLLATION_NAME = new String[MAP_SIZE]; - COLLATION_INDEX_TO_CHARSET = new MysqlCharset[MAP_SIZE]; + Map collationIndexToCharset = new TreeMap<>(); Map charsetNameToCollationIndexMap = new TreeMap<>(); Map charsetNameToCollationPriorityMap = new TreeMap<>(); - Set tempUTF8MB4Indexes = new HashSet<>(); + Map collationNameToCollationIndexMap = new TreeMap<>(); + Set impermissibleIndexes = new HashSet<>(); - Collation notUsedCollation = new Collation(0, COLLATION_NOT_DEFINED, 0, NOT_USED); for (int i = 1; i < MAP_SIZE; i++) { - Collation coll = collation[i] != null ? collation[i] : notUsedCollation; - COLLATION_INDEX_TO_COLLATION_NAME[i] = coll.collationName; - COLLATION_INDEX_TO_CHARSET[i] = coll.mysqlCharset; - String charsetName = coll.mysqlCharset.charsetName; - - if (!charsetNameToCollationIndexMap.containsKey(charsetName) || charsetNameToCollationPriorityMap.get(charsetName) < coll.priority) { - charsetNameToCollationIndexMap.put(charsetName, i); - charsetNameToCollationPriorityMap.put(charsetName, coll.priority); - } + Collation coll = collation[i]; + if (coll != null) { + COLLATION_INDEX_TO_COLLATION_NAME[i] = coll.collationName; + collationIndexToCharset.put(i, coll.mysqlCharset); + collationNameToCollationIndexMap.put(coll.collationName, i); + String charsetName = coll.mysqlCharset.charsetName; + + if (!charsetNameToCollationIndexMap.containsKey(charsetName) || charsetNameToCollationPriorityMap.get(charsetName) < coll.priority) { + charsetNameToCollationIndexMap.put(charsetName, i); + charsetNameToCollationPriorityMap.put(charsetName, coll.priority); + } - // Filling indexes of utf8mb4 collations - if (charsetName.equals(MYSQL_CHARSET_NAME_utf8mb4)) { - tempUTF8MB4Indexes.add(i); + // Filling indexes of impermissible client character sets ucs2, utf16, utf16le, utf32 + if (charsetName.equals(MYSQL_CHARSET_NAME_ucs2) || charsetName.equals(MYSQL_CHARSET_NAME_utf16) + || charsetName.equals(MYSQL_CHARSET_NAME_utf16le) || charsetName.equals(MYSQL_CHARSET_NAME_utf32)) { + impermissibleIndexes.add(i); + } } } + COLLATION_INDEX_TO_CHARSET = Collections.unmodifiableMap(collationIndexToCharset); CHARSET_NAME_TO_COLLATION_INDEX = Collections.unmodifiableMap(charsetNameToCollationIndexMap); - UTF8MB4_INDEXES = Collections.unmodifiableSet(tempUTF8MB4Indexes); + COLLATION_NAME_TO_COLLATION_INDEX = Collections.unmodifiableMap(collationNameToCollationIndexMap); + IMPERMISSIBLE_INDEXES = Collections.unmodifiableSet(impermissibleIndexes); collation = null; } - public final static String getMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version) { - + protected static String getStaticMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version) { List mysqlCharsets = CharsetMapping.JAVA_ENCODING_UC_TO_MYSQL_CHARSET.get(javaEncoding.toUpperCase(Locale.ENGLISH)); - if (mysqlCharsets != null) { - Iterator iter = mysqlCharsets.iterator(); - + if (version == null) { + return mysqlCharsets.get(0).charsetName; // Take the first one we get + } MysqlCharset currentChoice = null; - - while (iter.hasNext()) { - MysqlCharset charset = iter.next(); - - if (version == null) { - // Take the first one we get - - return charset.charsetName; - } - - if (currentChoice == null || currentChoice.minimumVersion.compareTo(charset.minimumVersion) < 0 - || currentChoice.priority < charset.priority && currentChoice.minimumVersion.compareTo(charset.minimumVersion) == 0) { - if (charset.isOkayForVersion(version)) { - currentChoice = charset; - } + for (MysqlCharset charset : mysqlCharsets) { + if (charset.isOkayForVersion(version) && (currentChoice == null || currentChoice.minimumVersion.compareTo(charset.minimumVersion) < 0 + || currentChoice.priority < charset.priority && currentChoice.minimumVersion.compareTo(charset.minimumVersion) == 0)) { + currentChoice = charset; } } - if (currentChoice != null) { return currentChoice.charsetName; } } - return null; } - public static int getCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version) { - String charsetName = getMysqlCharsetForJavaEncoding(javaEncoding, version); + protected static int getStaticCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version) { + String charsetName = getStaticMysqlCharsetForJavaEncoding(javaEncoding, version); + return getStaticCollationIndexForMysqlCharsetName(charsetName); + } + + protected static int getStaticCollationIndexForMysqlCharsetName(String charsetName) { if (charsetName != null) { Integer ci = CHARSET_NAME_TO_COLLATION_INDEX.get(charsetName); if (ci != null) { @@ -592,13 +551,27 @@ public static int getCollationIndexForJavaEncoding(String javaEncoding, ServerVe return 0; } - public static String getMysqlCharsetNameForCollationIndex(Integer collationIndex) { + // TODO turn it to protected when com.mysql.cj.xdevapi.ColumnImpl can use dynamic maps + public static String getStaticMysqlCharsetNameForCollationIndex(Integer collationIndex) { + MysqlCharset charset = null; + if (collationIndex != null) { + charset = COLLATION_INDEX_TO_CHARSET.get(collationIndex); + } + return charset != null ? charset.charsetName : null; + } + + // TODO turn it to protected when com.mysql.cj.xdevapi.ColumnImpl can use dynamic maps + public static String getStaticCollationNameForCollationIndex(Integer collationIndex) { if (collationIndex != null && collationIndex > 0 && collationIndex < MAP_SIZE) { - return COLLATION_INDEX_TO_CHARSET[collationIndex].charsetName; + return COLLATION_INDEX_TO_COLLATION_NAME[collationIndex]; } return null; } + protected static Integer getStaticCollationIndexForCollationName(String collationName) { + return CharsetMapping.COLLATION_NAME_TO_COLLATION_INDEX.get(collationName); + } + /** * MySQL charset could map to several Java encodings. * So here we choose the one according to next rules: @@ -614,37 +587,34 @@ public static String getMysqlCharsetNameForCollationIndex(Integer collationIndex * * @param mysqlCharsetName * MySQL charset name - * @param javaEncoding + * @param fallbackJavaEncoding * fall-back java encoding name * @return java encoding name */ - public static String getJavaEncodingForMysqlCharset(String mysqlCharsetName, String javaEncoding) { - String res = javaEncoding; - MysqlCharset cs = CHARSET_NAME_TO_CHARSET.get(mysqlCharsetName); - if (cs != null) { - res = cs.getMatchingJavaEncoding(javaEncoding); - } - return res; + protected static String getStaticJavaEncodingForMysqlCharset(String mysqlCharsetName, String fallbackJavaEncoding) { + MysqlCharset cs = getStaticMysqlCharsetByName(mysqlCharsetName); + return cs != null ? cs.getMatchingJavaEncoding(fallbackJavaEncoding) : fallbackJavaEncoding; } - public static String getJavaEncodingForMysqlCharset(String mysqlCharsetName) { - return getJavaEncodingForMysqlCharset(mysqlCharsetName, null); + protected static MysqlCharset getStaticMysqlCharsetByName(String mysqlCharsetName) { + return CHARSET_NAME_TO_CHARSET.get(mysqlCharsetName); } - public static String getJavaEncodingForCollationIndex(Integer collationIndex, String javaEncoding) { - if (collationIndex != null && collationIndex > 0 && collationIndex < MAP_SIZE) { - MysqlCharset cs = COLLATION_INDEX_TO_CHARSET[collationIndex]; - return cs.getMatchingJavaEncoding(javaEncoding); - } - return null; + protected static String getStaticJavaEncodingForMysqlCharset(String mysqlCharsetName) { + return getStaticJavaEncodingForMysqlCharset(mysqlCharsetName, null); } - public static String getJavaEncodingForCollationIndex(Integer collationIndex) { - return getJavaEncodingForCollationIndex(collationIndex, null); + protected static String getStaticJavaEncodingForCollationIndex(Integer collationIndex, String fallbackJavaEncoding) { + MysqlCharset charset = null; + if (collationIndex != null) { + charset = COLLATION_INDEX_TO_CHARSET.get(collationIndex); + } + return charset != null ? charset.getMatchingJavaEncoding(fallbackJavaEncoding) : fallbackJavaEncoding; } - public final static int getNumberOfCharsetsConfigured() { - return numberOfEncodingsConfigured; + // TODO turn it to protected when com.mysql.cj.protocol.x.FieldFactory can use dynamic maps + public static String getStaticJavaEncodingForCollationIndex(Integer collationIndex) { + return getStaticJavaEncodingForCollationIndex(collationIndex, null); } /** @@ -654,19 +624,24 @@ public final static int getNumberOfCharsetsConfigured() { * java encoding name * @return true if the character set contains multi-byte encoded characters. */ - final public static boolean isMultibyteCharset(String javaEncodingName) { + protected static boolean isStaticMultibyteCharset(String javaEncodingName) { return MULTIBYTE_ENCODINGS.contains(javaEncodingName.toUpperCase(Locale.ENGLISH)); } - public static int getMblen(String charsetName) { + protected static int getStaticMblen(String charsetName) { if (charsetName != null) { - MysqlCharset cs = CHARSET_NAME_TO_CHARSET.get(charsetName); + MysqlCharset cs = getStaticMysqlCharsetByName(charsetName); if (cs != null) { return cs.mblen; } } return 0; } + + protected static boolean isStaticImpermissibleCollation(int collationIndex) { + return CharsetMapping.IMPERMISSIBLE_INDEXES.contains(collationIndex); + } + } class MysqlCharset { @@ -711,12 +686,7 @@ public MysqlCharset(String charsetName, int mblen, int priority, String[] javaEn try { Charset cs = Charset.forName(encoding); addEncodingMapping(cs.name()); - - Set als = cs.aliases(); - Iterator ali = als.iterator(); - while (ali.hasNext()) { - addEncodingMapping(ali.next()); - } + cs.aliases().forEach(this::addEncodingMapping); } catch (Exception e) { // if there is no support of this charset in JVM it's still possible to use our converter for 1-byte charsets if (mblen == 1) { @@ -726,11 +696,7 @@ public MysqlCharset(String charsetName, int mblen, int priority, String[] javaEn } if (this.javaEncodingsUc.size() == 0) { - if (mblen > 1) { - addEncodingMapping("UTF-8"); - } else { - addEncodingMapping("Cp1252"); - } + addEncodingMapping(mblen > 1 ? "UTF-8" : "Cp1252"); } this.minimumVersion = minimumVersion; @@ -780,7 +746,7 @@ public Collation(int index, String collationName, int priority, String charsetNa this.index = index; this.collationName = collationName; this.priority = priority; - this.mysqlCharset = CharsetMapping.CHARSET_NAME_TO_CHARSET.get(charsetName); + this.mysqlCharset = CharsetMapping.getStaticMysqlCharsetByName(charsetName); } @Override diff --git a/src/main/core-api/java/com/mysql/cj/CharsetSettings.java b/src/main/core-api/java/com/mysql/cj/CharsetSettings.java new file mode 100644 index 000000000..e6cd54a8c --- /dev/null +++ b/src/main/core-api/java/com/mysql/cj/CharsetSettings.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj; + +public interface CharsetSettings { + + public static final String CHARACTER_SET_CLIENT = "character_set_client"; + public static final String CHARACTER_SET_CONNECTION = "character_set_connection"; + public static final String CHARACTER_SET_RESULTS = "character_set_results"; + public static final String COLLATION_CONNECTION = "collation_connection"; + + /** + *

+ * Choose the MySQL collation index for the handshake packet and the corresponding Java encodings for the password and error messages. + *

+ *

+ * This index will be sent with HandshakeResponse setting server variables 'character_set_connection', 'collation_connection', 'character_set_client' + * and 'character_set_results' which will be used by the server for decoding passwords during the authentication phase and later on, if + * no SET NAMES are issued by {@link #configurePostHandshake(boolean)}. + *

+ *

+ * It also means that collation index should be set according to: + *

    + *
  1. 'passwordCharacterEncoding' if it's present, or + *
  2. 'connectionCollation' if it's present, or + *
  3. 'characterEncoding' if it's present + *
+ * otherwise it will be set to utf8mb4_general_ci or utf8mb4_0900_ai_ci depending on server version. + *

+ * Since Protocol::HandshakeV10 and Protocol::HandshakeResponse41 has only one byte for the collation it's not possible to use indexes > 255 during the + * handshake. + * Also, ucs2, utf16, utf16le and utf32 character sets are impermissible here. Connector/J will try to use utf8mb4 instead. + *

+ * + * @param reset + * reset the charsets configuration; needed for changeUser call. + * + * @return MySQL collation index to be used during the handshake. + */ + int configurePreHandshake(boolean reset); + + /** + * Sets up client character set. This must be done before any further communication with the server! + * + * The 'collation_connection', 'character_set_client', 'character_set_connection' and 'character_set_results' server variables are set + * according to the collation index selected by {@link #configurePreHandshake(boolean)} and sent in the Protocol::HandshakeV10 packet. + * Here Connector/J alters these server variables if needed. + * + * @param dontCheckServerMatch + * if true then send the SET NAMES query even if server charset already matches the new value; needed for changeUser call. + */ + void configurePostHandshake(boolean dontCheckServerMatch); + + public boolean doesPlatformDbCharsetMatches(); + + String getPasswordCharacterEncoding(); + + String getErrorMessageEncoding(); + + String getMetadataEncoding(); + + int getMetadataCollationIndex(); + + boolean getRequiresEscapingEncoder(); + + String getJavaEncodingForCollationIndex(int collationIndex); + + int getMaxBytesPerChar(String javaCharsetName); + + int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName); + + Integer getCollationIndexForCollationName(String collationName); + + String getCollationNameForCollationIndex(Integer collationIndex); + + String getMysqlCharsetNameForCollationIndex(Integer collationIndex); + + int getCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version); + + int getCollationIndexForMysqlCharsetName(String charsetName); + + String getJavaEncodingForMysqlCharset(String mysqlCharsetName); + + String getMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version); + + boolean isMultibyteCharset(String javaEncodingName); +} diff --git a/src/main/core-api/java/com/mysql/cj/Constants.java b/src/main/core-api/java/com/mysql/cj/Constants.java index d468c080c..cc918e677 100644 --- a/src/main/core-api/java/com/mysql/cj/Constants.java +++ b/src/main/core-api/java/com/mysql/cj/Constants.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. + * Copyright (c) 2002, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -96,6 +96,8 @@ public class Constants { public static final BigDecimal BIG_DECIMAL_MAX_FLOAT_VALUE = BigDecimal.valueOf(Float.MAX_VALUE); public static final BigDecimal BIG_DECIMAL_MAX_NEGATIVE_FLOAT_VALUE = BigDecimal.valueOf(-Float.MAX_VALUE); + public static final int UNSIGNED_BYTE_MAX_VALUE = 255; + /** * Prevents instantiation */ diff --git a/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java b/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java index a43d65177..67b7eb581 100644 --- a/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java +++ b/src/main/core-api/java/com/mysql/cj/conf/PropertyDefinitions.java @@ -259,6 +259,9 @@ public enum DatabaseTerm { new StringPropertyDefinition(PropertyKey.characterSetResults, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE, Messages.getString("ConnectionProperties.characterSetResults"), "3.0.13", CATEGORY_SESSION, Integer.MIN_VALUE), + new StringPropertyDefinition(PropertyKey.customCharsetMapping, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE, + Messages.getString("ConnectionProperties.customCharsetMapping"), "8.0.26", CATEGORY_SESSION, Integer.MIN_VALUE), + new StringPropertyDefinition(PropertyKey.connectionCollation, DEFAULT_VALUE_NULL_STRING, RUNTIME_MODIFIABLE, Messages.getString("ConnectionProperties.connectionCollation"), "3.0.13", CATEGORY_SESSION, Integer.MIN_VALUE), diff --git a/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java b/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java index fd15974f3..8bd8457dd 100644 --- a/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java +++ b/src/main/core-api/java/com/mysql/cj/conf/PropertyKey.java @@ -100,6 +100,7 @@ public enum PropertyKey { connectTimeout("connectTimeout", true), // continueBatchOnError("continueBatchOnError", true), // createDatabaseIfNotExist("createDatabaseIfNotExist", true), // + customCharsetMapping("customCharsetMapping", true), // databaseTerm("databaseTerm", true), // defaultAuthenticationPlugin("defaultAuthenticationPlugin", true), // defaultFetchSize("defaultFetchSize", true), // diff --git a/src/main/core-api/java/com/mysql/cj/protocol/AuthenticationProvider.java b/src/main/core-api/java/com/mysql/cj/protocol/AuthenticationProvider.java index 2320cba2f..b8414806e 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/AuthenticationProvider.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/AuthenticationProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -29,24 +29,18 @@ package com.mysql.cj.protocol; -import com.mysql.cj.CharsetMapping; -import com.mysql.cj.Messages; -import com.mysql.cj.ServerVersion; import com.mysql.cj.conf.PropertySet; -import com.mysql.cj.exceptions.ExceptionFactory; import com.mysql.cj.exceptions.ExceptionInterceptor; public interface AuthenticationProvider { void init(Protocol prot, PropertySet propertySet, ExceptionInterceptor exceptionInterceptor); - void connect(ServerSession serverSession, String username, String password, String database); + void connect(String username, String password, String database); /** * Re-authenticates as the given user and password * - * @param serverSession - * {@link ServerSession} object * @param username * user name * @param password @@ -54,35 +48,5 @@ public interface AuthenticationProvider { * @param database * db name */ - void changeUser(ServerSession serverSession, String username, String password, String database); - - String getEncodingForHandshake(); - - /** - * Get the MySQL collation index for the handshake packet. A single byte will be added to the packet corresponding to the collation index - * found for the requested Java encoding name. - * - * If the index is > 255 which may be valid at some point in the future, an exception will be thrown. At the time of this implementation - * the index cannot be > 255 and only the COM_CHANGE_USER rpc, not the handshake response, can handle a value > 255. - * - * @param enc - * The Java encoding name used to lookup the collation index - * @param sv - * server version - * @return collation index - */ - static byte getCharsetForHandshake(String enc, ServerVersion sv) { - int charsetIndex = 0; - if (enc != null) { - charsetIndex = CharsetMapping.getCollationIndexForJavaEncoding(enc, sv); - } - if (charsetIndex == 0) { - charsetIndex = CharsetMapping.MYSQL_COLLATION_INDEX_utf8; - } - if (charsetIndex > 255) { - throw ExceptionFactory.createException(Messages.getString("MysqlIO.113", new Object[] { enc })); - } - return (byte) charsetIndex; - } - + void changeUser(String username, String password, String database); } diff --git a/src/main/core-api/java/com/mysql/cj/protocol/Protocol.java b/src/main/core-api/java/com/mysql/cj/protocol/Protocol.java index 49d046530..601979942 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/Protocol.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/Protocol.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -127,8 +127,6 @@ public interface Protocol { */ void changeUser(String user, String password, String database); - String getPasswordCharacterEncoding(); - boolean versionMeetsMinimum(int major, int minor, int subminor); /** diff --git a/src/main/core-api/java/com/mysql/cj/protocol/ServerCapabilities.java b/src/main/core-api/java/com/mysql/cj/protocol/ServerCapabilities.java index 0400f4abc..756ec5546 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/ServerCapabilities.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/ServerCapabilities.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -42,8 +42,11 @@ public interface ServerCapabilities { ServerVersion getServerVersion(); - void setServerVersion(ServerVersion serverVersion); + long getThreadId(); + + void setThreadId(long threadId); boolean serverSupportsFracSecs(); + int getServerDefaultCollationIndex(); } diff --git a/src/main/core-api/java/com/mysql/cj/protocol/ServerSession.java b/src/main/core-api/java/com/mysql/cj/protocol/ServerSession.java index e3c91f8bc..1fdea1840 100644 --- a/src/main/core-api/java/com/mysql/cj/protocol/ServerSession.java +++ b/src/main/core-api/java/com/mysql/cj/protocol/ServerSession.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.TimeZone; +import com.mysql.cj.CharsetSettings; import com.mysql.cj.ServerVersion; import com.mysql.cj.exceptions.CJOperationNotSupportedException; import com.mysql.cj.exceptions.ExceptionFactory; @@ -62,8 +63,6 @@ public interface ServerSession { */ public static int TRANSACTION_COMPLETED = 3; - public static final String LOCAL_CHARACTER_SET_RESULTS = "local.character_set_results"; - ServerCapabilities getCapabilities(); void setCapabilities(ServerCapabilities capabilities); @@ -92,20 +91,6 @@ public interface ServerSession { void setOldStatusFlags(int statusFlags); - /** - * - * @return Collation index which server provided in handshake greeting packet - */ - int getServerDefaultCollationIndex(); - - /** - * Stores collation index which server provided in handshake greeting packet. - * - * @param serverDefaultCollationIndex - * collation index - */ - void setServerDefaultCollationIndex(int serverDefaultCollationIndex); - /** * * @return TRANSACTION_NOT_STARTED, TRANSACTION_IN_PROGRESS, TRANSACTION_STARTED or TRANSACTION_COMPLETED @@ -153,8 +138,6 @@ public interface ServerSession { void setServerVariables(Map serverVariables); - boolean characterSetNamesMatches(String mysqlEncodingName); - /** * Get the version of the MySQL server we are talking to. * @@ -174,46 +157,6 @@ public interface ServerSession { */ boolean isVersion(ServerVersion version); - /** - * - * @return the server's default character set name according to collation index from server greeting, - * or value of 'character_set_server' variable if there is no mapping for that index - */ - String getServerDefaultCharset(); - - String getErrorMessageEncoding(); - - void setErrorMessageEncoding(String errorMessageEncoding); - - int getMaxBytesPerChar(String javaCharsetName); - - int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName); - - /** - * Returns the Java character encoding name for the given MySQL server - * collation index - * - * @param collationIndex - * collation index - * @return the Java character encoding name for the given MySQL server - * collation index - */ - String getEncodingForIndex(int collationIndex); - - void configureCharacterSets(); - - String getCharacterSetMetadata(); - - void setCharacterSetMetadata(String characterSetMetadata); - - int getMetadataCollationIndex(); - - void setMetadataCollationIndex(int metadataCollationIndex); - - String getCharacterSetResultsOnServer(); - - void setCharacterSetResultsOnServer(String characterSetResultsOnServer); - /** * Is the server configured to use lower-case table names only? * @@ -231,10 +174,6 @@ public interface ServerSession { public boolean isServerTruncatesFracSecs(); - long getThreadId(); - - public void setThreadId(long threadId); - boolean isAutoCommit(); void setAutoCommit(boolean autoCommit); @@ -255,4 +194,7 @@ default ServerSessionStateController getServerSessionStateController() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); } + CharsetSettings getCharsetSettings(); + + void setCharsetSettings(CharsetSettings charsetSettings); } diff --git a/src/main/core-api/java/com/mysql/cj/result/Field.java b/src/main/core-api/java/com/mysql/cj/result/Field.java index b45ab1690..ebfc9f7fa 100644 --- a/src/main/core-api/java/com/mysql/cj/result/Field.java +++ b/src/main/core-api/java/com/mysql/cj/result/Field.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. + * Copyright (c) 2002, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -29,9 +29,7 @@ package com.mysql.cj.result; -import com.mysql.cj.CharsetMapping; import com.mysql.cj.MysqlType; -import com.mysql.cj.ServerVersion; import com.mysql.cj.protocol.ProtocolEntity; import com.mysql.cj.util.LazyString; @@ -182,12 +180,6 @@ public String getEncoding() { return this.encoding; } - // TODO Remove this after DBMD isn't using ByteArrayRow results. - public void setEncoding(String javaEncodingName, ServerVersion version) { - this.encoding = javaEncodingName; - this.collationIndex = CharsetMapping.getCollationIndexForJavaEncoding(javaEncodingName, version); - } - public String getColumnLabel() { return getName(); } diff --git a/src/main/core-impl/java/com/mysql/cj/ClientPreparedQueryBindings.java b/src/main/core-impl/java/com/mysql/cj/ClientPreparedQueryBindings.java index 2698b94dd..d4cd71036 100644 --- a/src/main/core-impl/java/com/mysql/cj/ClientPreparedQueryBindings.java +++ b/src/main/core-impl/java/com/mysql/cj/ClientPreparedQueryBindings.java @@ -77,7 +77,7 @@ public class ClientPreparedQueryBindings extends AbstractQueryBindings collationIndexToCollationName = null; + public Map collationNameToCollationIndex = null; + public Map collationIndexToCharsetName = null; + public Map charsetNameToMblen = null; + public Map charsetNameToJavaEncoding = null; + public Map charsetNameToCollationIndex = null; + public Map javaEncodingUcToCharsetName = null; + public Set multibyteEncodings = null; + + private Integer sessionCollationIndex = null; + + /** + * What character set is the metadata returned in? + */ + private String metadataEncoding = null; + private int metadataCollationIndex; + + /** + * The (Java) encoding used to interpret error messages received from the server. + * We use character_set_results (since MySQL 5.5) if it is not null or UTF-8 otherwise. + */ + private String errorMessageEncoding = "Cp1252"; // to begin with, changes after we talk to the server + + protected RuntimeProperty characterEncoding; + protected RuntimeProperty passwordCharacterEncoding; + protected RuntimeProperty characterSetResults; + protected RuntimeProperty connectionCollation; + protected RuntimeProperty cacheServerConfiguration; + + /** + * If a CharsetEncoder is required for escaping. Needed for SJIS and related problems with \u00A5. + */ + private boolean requiresEscapingEncoder; + + private NativeMessageBuilder commandBuilder = null; + + private static final Map> customCollationIndexToCollationNameByUrl = new HashMap<>(); + private static final Map> customCollationNameToCollationIndexByUrl = new HashMap<>(); + + /** + * Actual collation index to mysql charset name map of user defined charsets for given server URLs. + */ + private static final Map> customCollationIndexToCharsetNameByUrl = new HashMap<>(); + + /** + * Actual mysql charset name to mblen map of user defined charsets for given server URLs. + */ + private static final Map> customCharsetNameToMblenByUrl = new HashMap<>(); + + private static final Map> customCharsetNameToJavaEncodingByUrl = new HashMap<>(); + private static final Map> customCharsetNameToCollationIndexByUrl = new HashMap<>(); + private static final Map> customJavaEncodingUcToCharsetNameByUrl = new HashMap<>(); + private static final Map> customMultibyteEncodingsByUrl = new HashMap<>(); + + /** + * We store the platform 'encoding' here, only used to avoid munging filenames for LOAD DATA LOCAL INFILE... + */ + private static Charset jvmPlatformCharset = null; + + /** + * Does the character set of this connection match the character set of the platform + */ + private boolean platformDbCharsetMatches = true; // changed once we've connected. + + static { + OutputStreamWriter outWriter = null; + // Use the I/O system to get the encoding (if possible), to avoid security restrictions on System.getProperty("file.encoding") in applets (why is that restricted?) + try { + outWriter = new OutputStreamWriter(new ByteArrayOutputStream()); + jvmPlatformCharset = Charset.forName(outWriter.getEncoding()); + } finally { + try { + if (outWriter != null) { + outWriter.close(); + } + } catch (IOException ioEx) { + // ignore + } + } + } + + private NativeMessageBuilder getCommandBuilder() { + if (this.commandBuilder == null) { + this.commandBuilder = new NativeMessageBuilder(this.serverSession.supportsQueryAttributes()); + } + return this.commandBuilder; + } + + /** + * Determines if the database charset is the same as the platform charset + */ + private void checkForCharsetMismatch() { + String characterEncodingValue = this.characterEncoding.getValue(); + if (characterEncodingValue != null) { + Charset characterEncodingCs = Charset.forName(characterEncodingValue); + Charset encodingToCheck = jvmPlatformCharset; + + if (encodingToCheck == null) { + encodingToCheck = Charset.forName(Constants.PLATFORM_ENCODING); + } + + this.platformDbCharsetMatches = encodingToCheck == null ? false : encodingToCheck.equals(characterEncodingCs); + } + } + + @Override + public boolean doesPlatformDbCharsetMatches() { + return this.platformDbCharsetMatches; + } + + public NativeCharsetSettings(NativeSession sess) { + this.session = sess; + this.serverSession = this.session.getServerSession(); + + this.characterEncoding = sess.getPropertySet().getStringProperty(PropertyKey.characterEncoding); + this.characterSetResults = this.session.getPropertySet().getProperty(PropertyKey.characterSetResults); + this.passwordCharacterEncoding = this.session.getPropertySet().getStringProperty(PropertyKey.passwordCharacterEncoding); + this.connectionCollation = this.session.getPropertySet().getStringProperty(PropertyKey.connectionCollation); + this.cacheServerConfiguration = sess.getPropertySet().getBooleanProperty(PropertyKey.cacheServerConfiguration); + + tryAndFixEncoding(this.characterEncoding, true); + tryAndFixEncoding(this.passwordCharacterEncoding, true); + if (!"null".equalsIgnoreCase(this.characterSetResults.getValue())) { // the "null" instead of an encoding name is allowed for characterSetResults + tryAndFixEncoding(this.characterSetResults, false); + } + } + + /** + * Attempt to use the encoding, and bail out if it can't be used. + * + * @param encodingProperty + * connection property containing the Java encoding to try + * @param replaceImpermissibleEncodings + * The character_set_client system variable cannot be set to ucs2, utf16, utf16le, utf32 charsets. If "true" the corresponding connection + * property value will be replaced with "UTF-8" + */ + private void tryAndFixEncoding(RuntimeProperty encodingProperty, boolean replaceImpermissibleEncodings) { + String oldEncoding = encodingProperty.getValue(); + if (oldEncoding != null) { + if (replaceImpermissibleEncodings && ("UnicodeBig".equalsIgnoreCase(oldEncoding) || "UTF-16".equalsIgnoreCase(oldEncoding) + || "UTF-16LE".equalsIgnoreCase(oldEncoding) || "UTF-32".equalsIgnoreCase(oldEncoding))) { + encodingProperty.setValue("UTF-8"); + } else { + try { + StringUtils.getBytes("abc", oldEncoding); + } catch (WrongArgumentException waEx) { + // Try the MySQL character set name, then.... + String newEncoding = getStaticJavaEncodingForMysqlCharset(oldEncoding); + if (newEncoding == null) { + throw ExceptionFactory.createException(WrongArgumentException.class, Messages.getString("StringUtils.0", new Object[] { oldEncoding }), + this.session.getExceptionInterceptor()); + } + StringUtils.getBytes("abc", newEncoding); + encodingProperty.setValue(newEncoding); + } + } + } + } + + @Override + public int configurePreHandshake(boolean reset) { + if (reset) { + this.sessionCollationIndex = null; + } + + // Avoid the second execution of this method + if (this.sessionCollationIndex != null) { + return this.sessionCollationIndex; + } + + ServerCapabilities capabilities = this.serverSession.getCapabilities(); + String encoding = this.passwordCharacterEncoding.getStringValue(); + if (encoding == null) { + String connectionColl = this.connectionCollation.getStringValue(); + if ((connectionColl == null || (this.sessionCollationIndex = getStaticCollationIndexForCollationName(connectionColl)) == null) + && (encoding = this.characterEncoding.getValue()) == null) { + // If none of "passwordCharacterEncoding", "connectionCollation" or "characterEncoding" is specified then use UTF-8. + // It would be better to use the server default collation here, to avoid unnecessary SET NAMES queries after the handshake if server + // default charset if not utf8, but we can not do it until server Bug#32729185 is fixed. Server cuts collation index to lower byte and, for example, + // if the server is started with character-set-server=utf8mb4 and collation-server=utf8mb4_is_0900_ai_ci (collation index 257) the Protocol::HandshakeV10 + // will contain character_set=1, "big5_chinese_ci". This is true not only for MySQL 8.0, where built-in collations with indexes > 255 were first introduced, + // but also other server series would be affected when configured with custom collations, for which the reserved collation id range is >= 1024. + this.sessionCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci; + } + } + + if (this.sessionCollationIndex == null) { + if ((this.sessionCollationIndex = getStaticCollationIndexForJavaEncoding(encoding, capabilities.getServerVersion())) == 0) { + throw ExceptionFactory.createException(WrongArgumentException.class, Messages.getString("StringUtils.0", new Object[] { encoding })); + } + } + + if (this.sessionCollationIndex > Constants.UNSIGNED_BYTE_MAX_VALUE // + || isStaticImpermissibleCollation(this.sessionCollationIndex)) { // At this point, impermissible charset can be set only with "connectionCollation". + this.sessionCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci; + } + + if (this.sessionCollationIndex == MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci + && !capabilities.getServerVersion().meetsMinimum(new ServerVersion(8, 0, 1))) { + this.sessionCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_general_ci; // use utf8mb4_general_ci instead of utf8mb4_0900_ai_ci for old servers + } + + // error messages are returned according to character_set_results which, at this point, is set from the response packet + this.errorMessageEncoding = getStaticJavaEncodingForCollationIndex(this.sessionCollationIndex); + this.serverSession.getServerVariables().put(CHARACTER_SET_RESULTS, getStaticMysqlCharsetNameForCollationIndex(this.sessionCollationIndex)); + + return this.sessionCollationIndex; + } + + @Override + public void configurePostHandshake(boolean dontCheckServerMatch) { + + buildCollationMapping(); + + /* + * Configuring characterEncoding. + */ + + String requiredCollation = this.connectionCollation.getStringValue(); + String requiredEncoding = this.characterEncoding.getValue(); + String passwordEncoding = this.passwordCharacterEncoding.getValue(); + Integer requiredCollationIndex; + String sessionCharsetName = getServerDefaultCharset(); + String sessionCollationClause = ""; + + try { + + // connectionCollation overrides the characterEncoding value + if (requiredCollation != null && (requiredCollationIndex = getCollationIndexForCollationName(requiredCollation)) != null) { + if (isImpermissibleCollation(requiredCollationIndex)) { + if (this.serverSession.getCapabilities().getServerVersion().meetsMinimum(new ServerVersion(8, 0, 1))) { + requiredCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci; + requiredCollation = "utf8mb4_0900_ai_ci"; + } else { + requiredCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_general_ci; + requiredCollation = "utf8mb4_general_ci"; + } + } + sessionCollationClause = " COLLATE " + requiredCollation; + sessionCharsetName = getMysqlCharsetNameForCollationIndex(requiredCollationIndex); + requiredEncoding = getJavaEncodingForCollationIndex(requiredCollationIndex, requiredEncoding); + this.sessionCollationIndex = requiredCollationIndex; + } + + if (requiredEncoding != null) { // If either connectionCollation or characterEncoding is defined. + if (sessionCollationClause.length() == 0) { // If no connectionCollation is defined. + sessionCharsetName = getMysqlCharsetForJavaEncoding(requiredEncoding.toUpperCase(Locale.ENGLISH), this.serverSession.getServerVersion()); + } + + } else { // Neither connectionCollation nor characterEncoding are defined. + // Collations with index > 255 don't fit into server greeting packet. + // Now we can set sessionCollationIndex according to "collation_server" value. + if (!StringUtils.isNullOrEmpty(passwordEncoding)) { + if (this.serverSession.getCapabilities().getServerVersion().meetsMinimum(new ServerVersion(8, 0, 1))) { + this.sessionCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_0900_ai_ci; // We can't do more, just trying to use utf8mb4_0900_ai_ci because the most of collations in that range are utf8mb4. + requiredCollation = "utf8mb4_0900_ai_ci"; + } else { + this.sessionCollationIndex = MYSQL_COLLATION_INDEX_utf8mb4_general_ci; + requiredCollation = "utf8mb4_general_ci"; + } + sessionCollationClause = " COLLATE " + getCollationNameForCollationIndex(this.sessionCollationIndex); + } + + if (((requiredEncoding = getJavaEncodingForCollationIndex(this.sessionCollationIndex, requiredEncoding)) == null)) { + // if there is no mapping for default collation index leave characterEncoding as specified by user + throw ExceptionFactory.createException(Messages.getString("Connection.5", new Object[] { this.sessionCollationIndex.toString() }), + this.session.getExceptionInterceptor()); + } + + sessionCharsetName = getMysqlCharsetNameForCollationIndex(this.sessionCollationIndex); + } + + } catch (ArrayIndexOutOfBoundsException outOfBoundsEx) { + throw ExceptionFactory.createException(Messages.getString("Connection.6", new Object[] { this.sessionCollationIndex }), + this.session.getExceptionInterceptor()); + } + + this.characterEncoding.setValue(requiredEncoding); + + if (sessionCharsetName != null) { + boolean isCharsetDifferent = !characterSetNamesMatches(sessionCharsetName); + boolean isCollationDifferent = sessionCollationClause.length() > 0 + && !requiredCollation.equalsIgnoreCase(this.serverSession.getServerVariable(COLLATION_CONNECTION)); + if (dontCheckServerMatch || isCharsetDifferent || isCollationDifferent) { + this.session.sendCommand(getCommandBuilder().buildComQuery(null, "SET NAMES " + sessionCharsetName + sessionCollationClause), false, 0); + this.serverSession.getServerVariables().put(CHARACTER_SET_CLIENT, sessionCharsetName); + this.serverSession.getServerVariables().put(CHARACTER_SET_CONNECTION, sessionCharsetName); + } + } + + /* + * Configuring characterSetResults. + * + * We know how to deal with any charset coming back from the database, so tell the server not to do conversion + * if the user hasn't 'forced' a result-set character set. + */ + + String sessionResultsCharset = this.serverSession.getServerVariable(CHARACTER_SET_RESULTS); + String characterSetResultsValue = this.characterSetResults.getValue(); + if (StringUtils.isNullOrEmpty(characterSetResultsValue) || "null".equalsIgnoreCase(characterSetResultsValue)) { + if (!StringUtils.isNullOrEmpty(sessionResultsCharset) && !"NULL".equalsIgnoreCase(sessionResultsCharset)) { + this.session.sendCommand(getCommandBuilder().buildComQuery(null, "SET character_set_results = NULL"), false, 0); + this.serverSession.getServerVariables().put(CHARACTER_SET_RESULTS, null); + } + + String defaultMetadataCharsetMysql = this.serverSession.getServerVariable("character_set_system"); + this.metadataEncoding = defaultMetadataCharsetMysql != null ? getJavaEncodingForMysqlCharset(defaultMetadataCharsetMysql) : "UTF-8"; + this.errorMessageEncoding = "UTF-8"; + + } else { + String resultsCharsetName = getMysqlCharsetForJavaEncoding(characterSetResultsValue.toUpperCase(Locale.ENGLISH), + this.serverSession.getServerVersion()); + + if (resultsCharsetName == null) { + throw ExceptionFactory.createException(WrongArgumentException.class, + Messages.getString("Connection.7", new Object[] { characterSetResultsValue }), this.session.getExceptionInterceptor()); + } + + if (!resultsCharsetName.equalsIgnoreCase(sessionResultsCharset)) { + this.session.sendCommand(getCommandBuilder().buildComQuery(null, "SET character_set_results = " + resultsCharsetName), false, 0); + this.serverSession.getServerVariables().put(CHARACTER_SET_RESULTS, resultsCharsetName); + } + + this.metadataEncoding = characterSetResultsValue; + this.errorMessageEncoding = characterSetResultsValue; + } + + this.metadataCollationIndex = getCollationIndexForJavaEncoding(this.metadataEncoding, this.serverSession.getServerVersion()); + + checkForCharsetMismatch(); + + /** + * Check if we need a CharsetEncoder for escaping codepoints that are + * transformed to backslash (0x5c) in the connection encoding. + */ + try { + CharsetEncoder enc = Charset.forName(this.characterEncoding.getValue()).newEncoder(); + CharBuffer cbuf = CharBuffer.allocate(1); + ByteBuffer bbuf = ByteBuffer.allocate(1); + + cbuf.put("\u00a5"); + cbuf.position(0); + enc.encode(cbuf, bbuf, true); + if (bbuf.get(0) == '\\') { + this.requiresEscapingEncoder = true; + } else { + cbuf.clear(); + bbuf.clear(); + + cbuf.put("\u20a9"); + cbuf.position(0); + enc.encode(cbuf, bbuf, true); + if (bbuf.get(0) == '\\') { + this.requiresEscapingEncoder = true; + } + } + } catch (java.nio.charset.UnsupportedCharsetException ucex) { + // fallback to String API + byte bbuf[] = StringUtils.getBytes("\u00a5", this.characterEncoding.getValue()); + if (bbuf[0] == '\\') { + this.requiresEscapingEncoder = true; + } else { + bbuf = StringUtils.getBytes("\u20a9", this.characterEncoding.getValue()); + if (bbuf[0] == '\\') { + this.requiresEscapingEncoder = true; + } + } + } + } + + private boolean characterSetNamesMatches(String mysqlEncodingName) { + // set names is equivalent to character_set_client ..._results and ..._connection, but we set _results later, so don't check it here. + return (mysqlEncodingName != null && mysqlEncodingName.equalsIgnoreCase(this.serverSession.getServerVariable(CHARACTER_SET_CLIENT)) + && mysqlEncodingName.equalsIgnoreCase(this.serverSession.getServerVariable(CHARACTER_SET_CONNECTION))); + } + + /** + * Get the server's default character set name according to collation index from server greeting, + * or value of 'character_set_server' variable if there is no mapping for that index + * + * @return MySQL charset name + */ + public String getServerDefaultCharset() { + String charset = getStaticMysqlCharsetNameForCollationIndex(this.sessionCollationIndex); + return charset != null ? charset : this.serverSession.getServerVariable("character_set_server"); + } + + @Override + public String getErrorMessageEncoding() { + return this.errorMessageEncoding; + } + + @Override + public String getMetadataEncoding() { + return this.metadataEncoding; + } + + @Override + public int getMetadataCollationIndex() { + return this.metadataCollationIndex; + } + + @Override + public boolean getRequiresEscapingEncoder() { + return this.requiresEscapingEncoder; + } + + @Override + public String getPasswordCharacterEncoding() { + return getStaticJavaEncodingForCollationIndex(this.sessionCollationIndex); + } + + /** + * Builds the map needed for 4.1.0 and newer servers that maps field-level + * charset/collation info to a java character encoding name. + */ + private void buildCollationMapping() { + + Map customCollationIndexToCollationName = null; + Map customCollationNameToCollationIndex = null; + Map customCollationIndexToCharsetName = null; + Map customCharsetNameToMblen = null; + Map customCharsetNameToJavaEncoding = new HashMap<>(); + Map customJavaEncodingUcToCharsetName = new HashMap<>(); + Map customCharsetNameToCollationIndex = new HashMap<>(); + Set customMultibyteEncodings = new HashSet<>(); + + String databaseURL = this.session.getHostInfo().getDatabaseUrl(); + + if (this.cacheServerConfiguration.getValue()) { + synchronized (customCollationIndexToCharsetNameByUrl) { + customCollationIndexToCollationName = customCollationIndexToCollationNameByUrl.get(databaseURL); + customCollationNameToCollationIndex = customCollationNameToCollationIndexByUrl.get(databaseURL); + customCollationIndexToCharsetName = customCollationIndexToCharsetNameByUrl.get(databaseURL); + customCharsetNameToMblen = customCharsetNameToMblenByUrl.get(databaseURL); + customCharsetNameToJavaEncoding = customCharsetNameToJavaEncodingByUrl.get(databaseURL); + customJavaEncodingUcToCharsetName = customJavaEncodingUcToCharsetNameByUrl.get(databaseURL); + customCharsetNameToCollationIndex = customCharsetNameToCollationIndexByUrl.get(databaseURL); + customMultibyteEncodings = customMultibyteEncodingsByUrl.get(databaseURL); + } + } + + if (customCollationIndexToCharsetName == null && this.session.getPropertySet().getBooleanProperty(PropertyKey.detectCustomCollations).getValue()) { + customCollationIndexToCollationName = new HashMap<>(); + customCollationNameToCollationIndex = new HashMap<>(); + customCollationIndexToCharsetName = new HashMap<>(); + customCharsetNameToMblen = new HashMap<>(); + + String customCharsetMapping = this.session.getPropertySet().getStringProperty(PropertyKey.customCharsetMapping).getValue(); + if (customCharsetMapping != null) { + String[] pairs = customCharsetMapping.split(","); + for (String pair : pairs) { + int keyEnd = pair.indexOf(":"); + if (keyEnd > 0 && (keyEnd + 1) < pair.length()) { + String charset = pair.substring(0, keyEnd); + String encoding = pair.substring(keyEnd + 1); + customCharsetNameToJavaEncoding.put(charset, encoding); + customJavaEncodingUcToCharsetName.put(encoding.toUpperCase(Locale.ENGLISH), charset); + } + } + } + + ValueFactory ivf = new IntegerValueFactory(this.session.getPropertySet()); + + try { + NativePacketPayload resultPacket = this.session.sendCommand(getCommandBuilder().buildComQuery(null, + "select c.COLLATION_NAME, c.CHARACTER_SET_NAME, c.ID, cs.MAXLEN, c.IS_DEFAULT='Yes' from INFORMATION_SCHEMA.COLLATIONS as c left join" + + " INFORMATION_SCHEMA.CHARACTER_SETS as cs on cs.CHARACTER_SET_NAME=c.CHARACTER_SET_NAME"), + false, 0); + Resultset rs = this.session.getProtocol().readAllResults(-1, false, resultPacket, false, null, new ResultsetFactory(Type.FORWARD_ONLY, null)); + ValueFactory svf = new StringValueFactory(this.session.getPropertySet()); + Row r; + while ((r = rs.getRows().next()) != null) { + String collationName = r.getValue(0, svf); + String charsetName = r.getValue(1, svf); + int collationIndex = ((Number) r.getValue(2, ivf)).intValue(); + int maxlen = ((Number) r.getValue(3, ivf)).intValue(); + boolean isDefault = ((Number) r.getValue(4, ivf)).intValue() > 0; + + if (collationIndex >= MAP_SIZE // + || !collationName.equals(getStaticCollationNameForCollationIndex(collationIndex)) + || !charsetName.equals(getStaticMysqlCharsetNameForCollationIndex(collationIndex))) { + customCollationIndexToCollationName.put(collationIndex, collationName); + customCollationNameToCollationIndex.put(collationName, collationIndex); + customCollationIndexToCharsetName.put(collationIndex, charsetName); + if (isDefault) { + customCharsetNameToCollationIndex.put(charsetName, collationIndex); + } else { + customCharsetNameToCollationIndex.putIfAbsent(charsetName, collationIndex); + } + + } + + // if no static map for charsetName adding to custom map + if (getStaticMysqlCharsetByName(charsetName) == null) { + customCharsetNameToMblen.put(charsetName, maxlen); + if (maxlen > 1) { + String enc = customCharsetNameToJavaEncoding.get(charsetName); + if (enc != null) { + customMultibyteEncodings.add(enc.toUpperCase(Locale.ENGLISH)); + } + } + } + + } + } catch (IOException e) { + throw ExceptionFactory.createException(e.getMessage(), e, this.session.getExceptionInterceptor()); + } + + if (this.cacheServerConfiguration.getValue()) { + synchronized (customCollationIndexToCharsetNameByUrl) { + customCollationIndexToCollationNameByUrl.put(databaseURL, Collections.unmodifiableMap(customCollationIndexToCollationName)); + customCollationNameToCollationIndexByUrl.put(databaseURL, Collections.unmodifiableMap(customCollationNameToCollationIndex)); + customCollationIndexToCharsetNameByUrl.put(databaseURL, Collections.unmodifiableMap(customCollationIndexToCharsetName)); + customCharsetNameToMblenByUrl.put(databaseURL, Collections.unmodifiableMap(customCharsetNameToMblen)); + customCharsetNameToJavaEncodingByUrl.put(databaseURL, Collections.unmodifiableMap(customCharsetNameToJavaEncoding)); + customJavaEncodingUcToCharsetNameByUrl.put(databaseURL, Collections.unmodifiableMap(customJavaEncodingUcToCharsetName)); + customCharsetNameToCollationIndexByUrl.put(databaseURL, Collections.unmodifiableMap(customCharsetNameToCollationIndex)); + customMultibyteEncodingsByUrl.put(databaseURL, Collections.unmodifiableSet(customMultibyteEncodings)); + } + } + } + + if (customCollationIndexToCharsetName != null) { + this.collationIndexToCollationName = customCollationIndexToCollationName; + this.collationNameToCollationIndex = customCollationNameToCollationIndex; + this.collationIndexToCharsetName = customCollationIndexToCharsetName; + this.charsetNameToMblen = customCharsetNameToMblen; + this.charsetNameToJavaEncoding = customCharsetNameToJavaEncoding; + this.javaEncodingUcToCharsetName = customJavaEncodingUcToCharsetName; + this.charsetNameToCollationIndex = customCharsetNameToCollationIndex; + this.multibyteEncodings = customMultibyteEncodings; + } + } + + @Override + public Integer getCollationIndexForCollationName(String collationName) { + Integer collationIndex = null; + if (this.collationNameToCollationIndex == null || (collationIndex = this.collationNameToCollationIndex.get(collationName)) == null) { + collationIndex = getStaticCollationIndexForCollationName(collationName); + } + return collationIndex; + } + + @Override + public String getCollationNameForCollationIndex(Integer collationIndex) { + String collationName = null; + if (collationIndex != null + && (this.collationIndexToCollationName == null || (collationName = this.collationIndexToCollationName.get(collationIndex)) == null)) { + collationName = getStaticCollationNameForCollationIndex(collationIndex); + } + return collationName; + } + + @Override + public String getMysqlCharsetNameForCollationIndex(Integer collationIndex) { + String charset = null; + if (this.collationIndexToCharsetName == null || (charset = this.collationIndexToCharsetName.get(collationIndex)) == null) { + charset = getStaticMysqlCharsetNameForCollationIndex(collationIndex); + } + return charset; + } + + @Override + public String getJavaEncodingForCollationIndex(int collationIndex) { + return getJavaEncodingForCollationIndex(collationIndex, this.characterEncoding.getValue()); + } + + public String getJavaEncodingForCollationIndex(Integer collationIndex, String fallBackJavaEncoding) { + String encoding = null; + String charset = null; + if (collationIndex != NativeConstants.NO_CHARSET_INFO) { + if (this.collationIndexToCharsetName != null && (charset = this.collationIndexToCharsetName.get(collationIndex)) != null) { + encoding = getJavaEncodingForMysqlCharset(charset, fallBackJavaEncoding); + } + if (encoding == null) { + encoding = getStaticJavaEncodingForCollationIndex(collationIndex, fallBackJavaEncoding); + } + } + return encoding != null ? encoding : fallBackJavaEncoding; + } + + public int getCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version) { + return getCollationIndexForMysqlCharsetName(getMysqlCharsetForJavaEncoding(javaEncoding, version)); + } + + public int getCollationIndexForMysqlCharsetName(String charsetName) { + Integer index = null; + if (this.charsetNameToCollationIndex == null || (index = this.charsetNameToCollationIndex.get(charsetName)) == null) { + index = getStaticCollationIndexForMysqlCharsetName(charsetName); + } + return index; + } + + public String getJavaEncodingForMysqlCharset(String mysqlCharsetName) { + String encoding = null; + if (this.charsetNameToJavaEncoding == null || (encoding = this.charsetNameToJavaEncoding.get(mysqlCharsetName)) == null) { + encoding = getStaticJavaEncodingForMysqlCharset(mysqlCharsetName); + } + return encoding; + } + + public String getJavaEncodingForMysqlCharset(String mysqlCharsetName, String javaEncoding) { + String encoding = null; + if (this.charsetNameToJavaEncoding == null || (encoding = this.charsetNameToJavaEncoding.get(mysqlCharsetName)) == null) { + encoding = getStaticJavaEncodingForMysqlCharset(mysqlCharsetName, javaEncoding); + } + return encoding; + } + + public String getMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version) { + String charset = null; + if (this.javaEncodingUcToCharsetName == null || (charset = this.javaEncodingUcToCharsetName.get(javaEncoding.toUpperCase(Locale.ENGLISH))) == null) { + charset = getStaticMysqlCharsetForJavaEncoding(javaEncoding, version); + } + return charset; + } + + public boolean isImpermissibleCollation(int collationIndex) { + String charsetName = null; + if (this.collationIndexToCharsetName != null && (charsetName = this.collationIndexToCharsetName.get(collationIndex)) != null) { + if (charsetName.equals(MYSQL_CHARSET_NAME_ucs2) || charsetName.equals(MYSQL_CHARSET_NAME_utf16) || charsetName.equals(MYSQL_CHARSET_NAME_utf16le) + || charsetName.equals(MYSQL_CHARSET_NAME_utf32)) { + return true; + } + } + return isStaticImpermissibleCollation(collationIndex); + } + + public boolean isMultibyteCharset(String javaEncodingName) { + if (this.multibyteEncodings != null && this.multibyteEncodings.contains(javaEncodingName.toUpperCase(Locale.ENGLISH))) { + return true; + } + return isStaticMultibyteCharset(javaEncodingName); + } + + @Override + public int getMaxBytesPerChar(String javaCharsetName) { + return getMaxBytesPerChar(null, javaCharsetName); + } + + @Override + public int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName) { + String charset = null; + if ((charset = getMysqlCharsetNameForCollationIndex(charsetIndex)) == null) { + // if we didn't find charset name by index + charset = getStaticMysqlCharsetForJavaEncoding(javaCharsetName, this.serverSession.getServerVersion()); + } + Integer mblen = null; + if (this.charsetNameToMblen == null || (mblen = this.charsetNameToMblen.get(charset)) == null) { + mblen = getStaticMblen(charset); + } + return mblen != null ? mblen.intValue() : 1; + } +} diff --git a/src/main/core-impl/java/com/mysql/cj/NativeSession.java b/src/main/core-impl/java/com/mysql/cj/NativeSession.java index bf4374c73..7f82da0e5 100644 --- a/src/main/core-impl/java/com/mysql/cj/NativeSession.java +++ b/src/main/core-impl/java/com/mysql/cj/NativeSession.java @@ -34,16 +34,10 @@ import java.io.Serializable; import java.lang.ref.WeakReference; import java.net.SocketAddress; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Timer; @@ -62,7 +56,6 @@ import com.mysql.cj.exceptions.ExceptionInterceptorChain; import com.mysql.cj.exceptions.MysqlErrorNumbers; import com.mysql.cj.exceptions.OperationCancelledException; -import com.mysql.cj.exceptions.WrongArgumentException; import com.mysql.cj.interceptors.QueryInterceptor; import com.mysql.cj.log.Log; import com.mysql.cj.protocol.ColumnDefinition; @@ -70,7 +63,6 @@ import com.mysql.cj.protocol.ProtocolEntityFactory; import com.mysql.cj.protocol.Resultset; import com.mysql.cj.protocol.Resultset.Type; -import com.mysql.cj.protocol.ServerSession; import com.mysql.cj.protocol.SocketConnection; import com.mysql.cj.protocol.SocketFactory; import com.mysql.cj.protocol.a.NativeMessageBuilder; @@ -80,7 +72,6 @@ import com.mysql.cj.protocol.a.NativeSocketConnection; import com.mysql.cj.protocol.a.ResultsetFactory; import com.mysql.cj.result.Field; -import com.mysql.cj.result.IntegerValueFactory; import com.mysql.cj.result.LongValueFactory; import com.mysql.cj.result.Row; import com.mysql.cj.result.StringValueFactory; @@ -93,22 +84,6 @@ public class NativeSession extends CoreSession implements Serializable { private CacheAdapter> serverConfigCache; - /** - * Actual collation index to mysql charset name map of user defined charsets for given server URLs. - */ - private static final Map> customIndexToCharsetMapByUrl = new HashMap<>(); - - /** - * Actual mysql charset name to mblen map of user defined charsets for given server URLs. - */ - private static final Map> customCharsetToMblenMapByUrl = new HashMap<>(); - - /** - * If a CharsetEncoder is required for escaping. Needed for SJIS and related - * problems with \u00A5. - */ - private boolean requiresEscapingEncoder; - /** When did the last query finish? */ private long lastQueryFinishedTime = 0; @@ -156,9 +131,6 @@ public void connect(HostInfo hi, String user, String password, String database, // protocol is responsible for building a session and authenticating (using AuthenticationProvider) internally this.protocol.connect(user, password, database); - // error messages are returned according to character_set_results which, at this point, is set from the response packet - this.protocol.getServerSession().setErrorMessageEncoding(this.protocol.getAuthenticationProvider().getEncodingForHandshake()); - this.isClosed = false; this.commandBuilder = new NativeMessageBuilder(this.getServerSession().supportsQueryAttributes()); @@ -272,13 +244,6 @@ public int getSocketTimeout() { return sto.getValue(); } - /** - * Determines if the database charset is the same as the platform charset - */ - public void checkForCharsetMismatch() { - ((NativeProtocol) this.protocol).checkForCharsetMismatch(); - } - /** * Returns the packet used for sending data (used by PreparedStatement) with position set to 0. * Guarded by external synchronization on a mutex. @@ -355,282 +320,6 @@ public void setLocalInfileInputStream(InputStream stream) { this.protocol.setLocalInfileInputStream(stream); } - /** - * Configures client-side properties for character set information. - */ - private void configureCharsetProperties() { - if (this.characterEncoding.getValue() != null) { - // Attempt to use the encoding, and bail out if it can't be used - try { - String testString = "abc"; - StringUtils.getBytes(testString, this.characterEncoding.getValue()); - } catch (WrongArgumentException waEx) { - // Try the MySQL character encoding, then.... - String oldEncoding = this.characterEncoding.getValue(); - - this.characterEncoding.setValue(CharsetMapping.getJavaEncodingForMysqlCharset(oldEncoding)); - - if (this.characterEncoding.getValue() == null) { - throw ExceptionFactory.createException(WrongArgumentException.class, Messages.getString("Connection.5", new Object[] { oldEncoding }), - getExceptionInterceptor()); - } - - String testString = "abc"; - StringUtils.getBytes(testString, this.characterEncoding.getValue()); - } - } - } - - /** - * Sets up client character set. This must be done before any further communication with the server! - * - * @param dontCheckServerMatch - * if true then send the SET NAMES query even if server charset already matches the new value - * @return true if this routine actually configured the client character - * set, or false if the driver needs to use 'older' methods to - * detect the character set, as it is connected to a MySQL server - * older than 4.1.0 - * @throws CJException - * if an exception happens while sending 'SET NAMES' to the - * server, or the server sends character set information that - * the client doesn't know about. - */ - public boolean configureClientCharacterSet(boolean dontCheckServerMatch) { - String realJavaEncoding = this.characterEncoding.getValue(); - RuntimeProperty characterSetResults = getPropertySet().getProperty(PropertyKey.characterSetResults); - boolean characterSetAlreadyConfigured = false; - - try { - characterSetAlreadyConfigured = true; - - configureCharsetProperties(); - realJavaEncoding = this.characterEncoding.getValue(); // we need to do this again to grab this for versions > 4.1.0 - - String connectionCollationSuffix = ""; - String connectionCollationCharset = null; - - String connectionCollation = getPropertySet().getStringProperty(PropertyKey.connectionCollation).getStringValue(); - if (connectionCollation != null) { - for (int i = 1; i < CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME.length; i++) { - if (CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i].equals(connectionCollation)) { - connectionCollationSuffix = " COLLATE " + CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i]; - connectionCollationCharset = CharsetMapping.COLLATION_INDEX_TO_CHARSET[i].charsetName; - realJavaEncoding = CharsetMapping.getJavaEncodingForCollationIndex(i); - } - } - } - - try { - String serverEncodingToSet = CharsetMapping.getJavaEncodingForCollationIndex(this.protocol.getServerSession().getServerDefaultCollationIndex()); - - if (serverEncodingToSet == null || serverEncodingToSet.length() == 0) { - if (realJavaEncoding != null) { - // user knows best, try it - this.characterEncoding.setValue(realJavaEncoding); - } else { - throw ExceptionFactory.createException( - Messages.getString("Connection.6", new Object[] { this.protocol.getServerSession().getServerDefaultCollationIndex() }), - getExceptionInterceptor()); - } - } - - // "latin1" on MySQL-4.1.0+ is actually CP1252, not ISO8859_1 - if ("ISO8859_1".equalsIgnoreCase(serverEncodingToSet)) { - serverEncodingToSet = "Cp1252"; - } - if ("UnicodeBig".equalsIgnoreCase(serverEncodingToSet) || "UTF-16".equalsIgnoreCase(serverEncodingToSet) - || "UTF-16LE".equalsIgnoreCase(serverEncodingToSet) || "UTF-32".equalsIgnoreCase(serverEncodingToSet)) { - serverEncodingToSet = "UTF-8"; - } - - this.characterEncoding.setValue(serverEncodingToSet); - - } catch (ArrayIndexOutOfBoundsException outOfBoundsEx) { - if (realJavaEncoding != null) { - // user knows best, try it - this.characterEncoding.setValue(realJavaEncoding); - } else { - throw ExceptionFactory.createException( - Messages.getString("Connection.6", new Object[] { this.protocol.getServerSession().getServerDefaultCollationIndex() }), - getExceptionInterceptor()); - } - } - - if (this.characterEncoding.getValue() == null) { - // punt? - this.characterEncoding.setValue("ISO8859_1"); - } - - if (realJavaEncoding != null) { - - // - // Now, inform the server what character set we will be using from now-on... - // - if (realJavaEncoding.equalsIgnoreCase("UTF-8") || realJavaEncoding.equalsIgnoreCase("UTF8")) { - // charset names are case-sensitive - String utf8CharsetName = connectionCollationSuffix.length() > 0 ? connectionCollationCharset : "utf8mb4"; - - if (dontCheckServerMatch || !this.protocol.getServerSession().characterSetNamesMatches("utf8") - || (!this.protocol.getServerSession().characterSetNamesMatches("utf8mb4")) || (connectionCollationSuffix.length() > 0 - && !connectionCollation.equalsIgnoreCase(this.protocol.getServerSession().getServerVariable("collation_server")))) { - - sendCommand(this.commandBuilder.buildComQuery(null, "SET NAMES " + utf8CharsetName + connectionCollationSuffix), false, 0); - - this.protocol.getServerSession().getServerVariables().put("character_set_client", utf8CharsetName); - this.protocol.getServerSession().getServerVariables().put("character_set_connection", utf8CharsetName); - } - - this.characterEncoding.setValue(realJavaEncoding); - } /* not utf-8 */else { - String mysqlCharsetName = connectionCollationSuffix.length() > 0 ? connectionCollationCharset - : CharsetMapping.getMysqlCharsetForJavaEncoding(realJavaEncoding.toUpperCase(Locale.ENGLISH), - getServerSession().getServerVersion()); - - if (mysqlCharsetName != null) { - - if (dontCheckServerMatch || !this.protocol.getServerSession().characterSetNamesMatches(mysqlCharsetName)) { - sendCommand(this.commandBuilder.buildComQuery(null, "SET NAMES " + mysqlCharsetName + connectionCollationSuffix), false, 0); - - this.protocol.getServerSession().getServerVariables().put("character_set_client", mysqlCharsetName); - this.protocol.getServerSession().getServerVariables().put("character_set_connection", mysqlCharsetName); - } - } - - // Switch driver's encoding now, since the server knows what we're sending... - // - this.characterEncoding.setValue(realJavaEncoding); - } - } else if (this.characterEncoding.getValue() != null) { - // Tell the server we'll use the server default charset to send our queries from now on.... - String mysqlCharsetName = connectionCollationSuffix.length() > 0 ? connectionCollationCharset : getServerSession().getServerDefaultCharset(); - - boolean ucs2 = false; - if ("ucs2".equalsIgnoreCase(mysqlCharsetName) || "utf16".equalsIgnoreCase(mysqlCharsetName) || "utf16le".equalsIgnoreCase(mysqlCharsetName) - || "utf32".equalsIgnoreCase(mysqlCharsetName)) { - mysqlCharsetName = "utf8"; - ucs2 = true; - if (characterSetResults.getValue() == null) { - characterSetResults.setValue("UTF-8"); - } - } - - if (dontCheckServerMatch || !this.protocol.getServerSession().characterSetNamesMatches(mysqlCharsetName) || ucs2) { - sendCommand(this.commandBuilder.buildComQuery(null, "SET NAMES " + mysqlCharsetName + connectionCollationSuffix), false, 0); - - this.protocol.getServerSession().getServerVariables().put("character_set_client", mysqlCharsetName); - this.protocol.getServerSession().getServerVariables().put("character_set_connection", mysqlCharsetName); - } - - realJavaEncoding = this.characterEncoding.getValue(); - } - - // - // We know how to deal with any charset coming back from the database, so tell the server not to do conversion if the user hasn't 'forced' a - // result-set character set - // - - String onServer = this.protocol.getServerSession().getServerVariable("character_set_results"); - if (characterSetResults.getValue() == null) { - - // - // Only send if needed, if we're caching server variables we -have- to send, because we don't know what it was before we cached them. - // - if (onServer != null && onServer.length() > 0 && !"NULL".equalsIgnoreCase(onServer)) { - sendCommand(this.commandBuilder.buildComQuery(null, "SET character_set_results = NULL"), false, 0); - this.protocol.getServerSession().getServerVariables().put(ServerSession.LOCAL_CHARACTER_SET_RESULTS, null); - } else { - this.protocol.getServerSession().getServerVariables().put(ServerSession.LOCAL_CHARACTER_SET_RESULTS, onServer); - } - } else { - - String charsetResults = characterSetResults.getValue(); - String mysqlEncodingName = null; - - if ("UTF-8".equalsIgnoreCase(charsetResults) || "UTF8".equalsIgnoreCase(charsetResults)) { - mysqlEncodingName = "utf8"; - } else if ("null".equalsIgnoreCase(charsetResults)) { - mysqlEncodingName = "NULL"; - } else { - mysqlEncodingName = CharsetMapping.getMysqlCharsetForJavaEncoding(charsetResults.toUpperCase(Locale.ENGLISH), - getServerSession().getServerVersion()); - } - - // - // Only change the value if needed - // - - if (mysqlEncodingName == null) { - throw ExceptionFactory.createException(WrongArgumentException.class, Messages.getString("Connection.7", new Object[] { charsetResults }), - getExceptionInterceptor()); - } - - if (!mysqlEncodingName.equalsIgnoreCase(this.protocol.getServerSession().getServerVariable("character_set_results"))) { - StringBuilder setBuf = new StringBuilder("SET character_set_results = ".length() + mysqlEncodingName.length()); - setBuf.append("SET character_set_results = ").append(mysqlEncodingName); - - sendCommand(this.commandBuilder.buildComQuery(null, setBuf.toString()), false, 0); - - this.protocol.getServerSession().getServerVariables().put(ServerSession.LOCAL_CHARACTER_SET_RESULTS, mysqlEncodingName); - - // We have to set errorMessageEncoding according to new value of charsetResults for server version 5.5 and higher - this.protocol.getServerSession().setErrorMessageEncoding(charsetResults); - - } else { - this.protocol.getServerSession().getServerVariables().put(ServerSession.LOCAL_CHARACTER_SET_RESULTS, onServer); - } - } - - } finally { - // Failsafe, make sure that the driver's notion of character encoding matches what the user has specified. - // - this.characterEncoding.setValue(realJavaEncoding); - } - - /** - * Check if we need a CharsetEncoder for escaping codepoints that are - * transformed to backslash (0x5c) in the connection encoding. - */ - try { - CharsetEncoder enc = Charset.forName(this.characterEncoding.getValue()).newEncoder(); - CharBuffer cbuf = CharBuffer.allocate(1); - ByteBuffer bbuf = ByteBuffer.allocate(1); - - cbuf.put("\u00a5"); - cbuf.position(0); - enc.encode(cbuf, bbuf, true); - if (bbuf.get(0) == '\\') { - this.requiresEscapingEncoder = true; - } else { - cbuf.clear(); - bbuf.clear(); - - cbuf.put("\u20a9"); - cbuf.position(0); - enc.encode(cbuf, bbuf, true); - if (bbuf.get(0) == '\\') { - this.requiresEscapingEncoder = true; - } - } - } catch (java.nio.charset.UnsupportedCharsetException ucex) { - // fallback to String API - byte bbuf[] = StringUtils.getBytes("\u00a5", this.characterEncoding.getValue()); - if (bbuf[0] == '\\') { - this.requiresEscapingEncoder = true; - } else { - bbuf = StringUtils.getBytes("\u20a9", this.characterEncoding.getValue()); - if (bbuf[0] == '\\') { - this.requiresEscapingEncoder = true; - } - } - } - - return characterSetAlreadyConfigured; - } - - public boolean getRequiresEscapingEncoder() { - return this.requiresEscapingEncoder; - } - private void createConfigCacheIfNeeded(Object syncMutex) { synchronized (syncMutex) { if (this.serverConfigCache != null) { @@ -772,7 +461,9 @@ public void loadServerVariables(Object syncMutex, String version) { Row r; if ((r = rs.getRows().next()) != null) { for (int i = 0; i < f.length; i++) { - this.protocol.getServerSession().getServerVariables().put(f[i].getColumnLabel(), r.getValue(i, vf)); + String value = r.getValue(i, vf); + this.protocol.getServerSession().getServerVariables().put(f[i].getColumnLabel(), + "utf8mb3".equalsIgnoreCase(value) ? "utf8" : value); // recent server versions return "utf8mb3" instead of "utf8" } } } @@ -823,112 +514,6 @@ public void setSessionVariables() { } } - /** - * Builds the map needed for 4.1.0 and newer servers that maps field-level - * charset/collation info to a java character encoding name. - */ - public void buildCollationMapping() { - - Map customCharset = null; - Map customMblen = null; - - String databaseURL = this.hostInfo.getDatabaseUrl(); - - if (this.cacheServerConfiguration.getValue()) { - synchronized (customIndexToCharsetMapByUrl) { - customCharset = customIndexToCharsetMapByUrl.get(databaseURL); - customMblen = customCharsetToMblenMapByUrl.get(databaseURL); - } - } - - if (customCharset == null && getPropertySet().getBooleanProperty(PropertyKey.detectCustomCollations).getValue()) { - customCharset = new HashMap<>(); - customMblen = new HashMap<>(); - - ValueFactory ivf = new IntegerValueFactory(getPropertySet()); - - try { - NativePacketPayload resultPacket = sendCommand(this.commandBuilder.buildComQuery(null, "SHOW COLLATION"), false, 0); - Resultset rs = ((NativeProtocol) this.protocol).readAllResults(-1, false, resultPacket, false, null, - new ResultsetFactory(Type.FORWARD_ONLY, null)); - ValueFactory svf = new StringValueFactory(this.propertySet); - Row r; - while ((r = rs.getRows().next()) != null) { - int collationIndex = ((Number) r.getValue(2, ivf)).intValue(); - String charsetName = r.getValue(1, svf); - - // if no static map for charsetIndex or server has a different mapping then our static map, adding it to custom map - if (collationIndex >= CharsetMapping.MAP_SIZE || !charsetName.equals(CharsetMapping.getMysqlCharsetNameForCollationIndex(collationIndex))) { - customCharset.put(collationIndex, charsetName); - } - - // if no static map for charsetName adding to custom map - if (!CharsetMapping.CHARSET_NAME_TO_CHARSET.containsKey(charsetName)) { - customMblen.put(charsetName, null); - } - } - } catch (IOException e) { - throw ExceptionFactory.createException(e.getMessage(), e, this.exceptionInterceptor); - } - - // if there is a number of custom charsets we should execute SHOW CHARACTER SET to know theirs mblen - if (customMblen.size() > 0) { - try { - NativePacketPayload resultPacket = sendCommand(this.commandBuilder.buildComQuery(null, "SHOW CHARACTER SET"), false, 0); - Resultset rs = ((NativeProtocol) this.protocol).readAllResults(-1, false, resultPacket, false, null, - new ResultsetFactory(Type.FORWARD_ONLY, null)); - - int charsetColumn = rs.getColumnDefinition().getColumnNameToIndex().get("Charset"); - int maxlenColumn = rs.getColumnDefinition().getColumnNameToIndex().get("Maxlen"); - - ValueFactory svf = new StringValueFactory(this.propertySet); - Row r; - while ((r = rs.getRows().next()) != null) { - String charsetName = r.getValue(charsetColumn, svf); - if (customMblen.containsKey(charsetName)) { - customMblen.put(charsetName, r.getValue(maxlenColumn, ivf)); - } - } - } catch (IOException e) { - throw ExceptionFactory.createException(e.getMessage(), e, this.exceptionInterceptor); - } - } - - if (this.cacheServerConfiguration.getValue()) { - synchronized (customIndexToCharsetMapByUrl) { - customIndexToCharsetMapByUrl.put(databaseURL, customCharset); - customCharsetToMblenMapByUrl.put(databaseURL, customMblen); - } - } - } - - // set charset maps - if (customCharset != null) { - ((NativeServerSession) this.protocol.getServerSession()).indexToCustomMysqlCharset = Collections.unmodifiableMap(customCharset); - } - if (customMblen != null) { - ((NativeServerSession) this.protocol.getServerSession()).mysqlCharsetToCustomMblen = Collections.unmodifiableMap(customMblen); - } - - // Trying to workaround server collations with index > 255. Such index doesn't fit into server greeting packet, 0 is sent instead. - // Now we could set io.serverCharsetIndex according to "collation_server" value. - if (this.protocol.getServerSession().getServerDefaultCollationIndex() == 0) { - String collationServer = this.protocol.getServerSession().getServerVariable("collation_server"); - if (collationServer != null) { - for (int i = 1; i < CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME.length; i++) { - if (CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i].equals(collationServer)) { - this.protocol.getServerSession().setServerDefaultCollationIndex(i); - break; - } - } - } else { - // We can't do more, just trying to use utf8mb4_general_ci because the most of collations in that range are utf8mb4. - this.protocol.getServerSession().setServerDefaultCollationIndex(45); - } - } - - } - public String getProcessHost() { try { long threadId = getThreadId(); diff --git a/src/main/core-impl/java/com/mysql/cj/ServerPreparedQuery.java b/src/main/core-impl/java/com/mysql/cj/ServerPreparedQuery.java index 46e31a8b4..a6d0e4114 100644 --- a/src/main/core-impl/java/com/mysql/cj/ServerPreparedQuery.java +++ b/src/main/core-impl/java/com/mysql/cj/ServerPreparedQuery.java @@ -633,7 +633,7 @@ public void storeReader(int parameterIndex, NativePacketPayload packet, Reader i if (clobEncoding != null) { if (!clobEncoding.equals("UTF-16")) { - maxBytesChar = this.session.getServerSession().getMaxBytesPerChar(clobEncoding); + maxBytesChar = this.session.getServerSession().getCharsetSettings().getMaxBytesPerChar(clobEncoding); if (maxBytesChar == 1) { maxBytesChar = 2; // for safety diff --git a/src/main/core-impl/java/com/mysql/cj/result/AbstractDateTimeValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/AbstractDateTimeValueFactory.java index 9e29da340..fa7a57aec 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/AbstractDateTimeValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/AbstractDateTimeValueFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. + * Copyright (c) 2019, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -124,7 +124,6 @@ public T createFromBytes(byte[] bytes, int offset, int length, Field f) { return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/AbstractNumericValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/AbstractNumericValueFactory.java index 4d609f88a..4391fa069 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/AbstractNumericValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/AbstractNumericValueFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. + * Copyright (c) 2019, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -48,7 +48,6 @@ public T createFromBytes(byte[] bytes, int offset, int length, Field f) { return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/BooleanValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/BooleanValueFactory.java index 03f53f022..e6e3b0ce3 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/BooleanValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/BooleanValueFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -94,7 +94,6 @@ public Boolean createFromBytes(byte[] bytes, int offset, int length, Field f) { return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/ByteValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/ByteValueFactory.java index 6e0ae88ea..07a966f22 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/ByteValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/ByteValueFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -106,7 +106,6 @@ public Byte createFromBytes(byte[] bytes, int offset, int length, Field f) { if (length == 0 && this.pset.getBooleanProperty(PropertyKey.emptyStringsConvertToZero).getValue()) { return (byte) 0; } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/OffsetDateTimeValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/OffsetDateTimeValueFactory.java index 9fcff9afc..af6f8c85a 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/OffsetDateTimeValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/OffsetDateTimeValueFactory.java @@ -112,7 +112,6 @@ public OffsetDateTime createFromBytes(byte[] bytes, int offset, int length, Fiel return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/OffsetTimeValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/OffsetTimeValueFactory.java index 65c705d9e..9e1ae72b7 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/OffsetTimeValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/OffsetTimeValueFactory.java @@ -101,7 +101,6 @@ public OffsetTime createFromBytes(byte[] bytes, int offset, int length, Field f) return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/core-impl/java/com/mysql/cj/result/ZonedDateTimeValueFactory.java b/src/main/core-impl/java/com/mysql/cj/result/ZonedDateTimeValueFactory.java index 8e37dff10..eebf84ac1 100644 --- a/src/main/core-impl/java/com/mysql/cj/result/ZonedDateTimeValueFactory.java +++ b/src/main/core-impl/java/com/mysql/cj/result/ZonedDateTimeValueFactory.java @@ -109,7 +109,6 @@ public ZonedDateTime createFromBytes(byte[] bytes, int offset, int length, Field return createFromLong(0); } - // TODO: Too expensive to convert from other charset to ASCII here? UTF-8 (e.g.) doesn't need any conversion before being sent to the decoder String s = StringUtils.toString(bytes, offset, length, f.getEncoding()); byte[] newBytes = s.getBytes(); diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/BinaryResultsetReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/BinaryResultsetReader.java index cf52ed98a..33d2bc4f7 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/BinaryResultsetReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/BinaryResultsetReader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. + * Copyright (c) 2016, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -111,7 +111,8 @@ public Resultset read(int maxRows, boolean streamResults, NativePacketPayload re // check for file request if (columnCount == NativePacketPayload.NULL_LENGTH) { String charEncoding = this.protocol.getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(); - String fileName = resultPacket.readString(StringSelfDataType.STRING_TERM, this.protocol.doesPlatformDbCharsetMatches() ? charEncoding : null); + String fileName = resultPacket.readString(StringSelfDataType.STRING_TERM, + this.protocol.getServerSession().getCharsetSettings().doesPlatformDbCharsetMatches() ? null : charEncoding); resultPacket = this.protocol.sendFileToServer(fileName); } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ColumnDefinitionReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ColumnDefinitionReader.java index fd9aa6610..f244a5e0c 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ColumnDefinitionReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/ColumnDefinitionReader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. + * Copyright (c) 2016, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -74,7 +74,7 @@ public ColumnDefinition read(ProtocolEntityFactory prot, PropertySet propSet, Except * Initialize communications with the MySQL server. Handles logging on, and * handling initial connection errors. * - * @param sessState - * The session state object. It's intended to be updated from the handshake * @param user * user name * @param pass @@ -131,7 +129,8 @@ public void init(Protocol prot, PropertySet propSet, Except * database name */ @Override - public void connect(ServerSession sessState, String user, String pass, String db) { + public void connect(String user, String pass, String db) { + ServerSession sessState = this.protocol.getServerSession(); this.username = user; this.password = pass; this.database = db; @@ -152,7 +151,6 @@ public void connect(ServerSession sessState, String user, String pass, String db throw ExceptionFactory.createException(UnableToConnectException.class, "CLIENT_PLUGIN_AUTH is required", getExceptionInterceptor()); } - sessState.setServerDefaultCollationIndex(capabilities.getServerDefaultCollationIndex()); // read character set (1 byte) sessState.setStatusFlags(capabilities.getStatusFlags()); // read status flags (2 bytes) int authPluginDataLength = capabilities.getAuthPluginDataLength(); @@ -208,7 +206,7 @@ public void connect(ServerSession sessState, String user, String pass, String db throw ExceptionFactory.createException(Messages.getString("AuthenticationProvider.UnexpectedAuthenticationApproval"), getExceptionInterceptor()); } - proceedHandshakeWithPluggableAuthentication(sessState, buf); + proceedHandshakeWithPluggableAuthentication(buf); this.password = null; } @@ -359,14 +357,14 @@ private void checkConfidentiality(AuthenticationPlugin plugin) { * * This method will use registered authentication plugins as requested by the server. * - * @param serverSession - * The current state of the session * @param challenge * the Auth Challenge Packet received from server if * this method is used during the initial connection. * Otherwise null. */ - private void proceedHandshakeWithPluggableAuthentication(ServerSession serverSession, final NativePacketPayload challenge) { + private void proceedHandshakeWithPluggableAuthentication(final NativePacketPayload challenge) { + ServerSession serverSession = this.protocol.getServerSession(); + if (this.authenticationPlugins == null) { loadAuthenticationPlugins(); } @@ -377,6 +375,8 @@ private void proceedHandshakeWithPluggableAuthentication(ServerSession serverSes forChangeUser = false; } + serverSession.getCharsetSettings().configurePreHandshake(forChangeUser); + /* * Select the initial plugin: * Choose the client-side default authentication plugin, if explicitely specified, otherwise choose the server-side default authentication plugin. @@ -556,20 +556,6 @@ private void appendConnectionAttributes(NativePacketPayload buf, String attribut buf.writeBytes(StringLengthDataType.STRING_FIXED, lb.getByteBuffer(), 0, lb.getPosition()); } - /** - * Get the Java encoding to be used for the handshake - * response. Defaults to UTF-8. - * - * @return encoding name - */ - public String getEncodingForHandshake() { - String enc = this.propertySet.getStringProperty(PropertyKey.characterEncoding).getValue(); - if (enc == null) { - enc = "UTF-8"; - } - return enc; - } - public ExceptionInterceptor getExceptionInterceptor() { return this.exceptionInterceptor; } @@ -577,8 +563,6 @@ public ExceptionInterceptor getExceptionInterceptor() { /** * Re-authenticates as the given user and password * - * @param serverSession - * current {@link ServerSession} * @param user * user name * @param pass @@ -587,18 +571,19 @@ public ExceptionInterceptor getExceptionInterceptor() { * database name */ @Override - public void changeUser(ServerSession serverSession, String user, String pass, String db) { + public void changeUser(String user, String pass, String db) { this.username = user; this.password = pass; this.database = db; - proceedHandshakeWithPluggableAuthentication(serverSession, null); + proceedHandshakeWithPluggableAuthentication(null); this.password = null; } private NativePacketPayload createHandshakeResponsePacket(ServerSession serverSession, String pluginName, ArrayList toServer) { long clientParam = serverSession.getClientParam(); - String enc = getEncodingForHandshake(); + int collationIndex = serverSession.getCharsetSettings().configurePreHandshake(false); + String enc = serverSession.getCharsetSettings().getPasswordCharacterEncoding(); int userLength = this.username == null ? 0 : this.username.length(); NativePacketPayload last_sent = new NativePacketPayload(AUTH_411_OVERHEAD + 7 // @@ -608,7 +593,7 @@ private NativePacketPayload createHandshakeResponsePacket(ServerSession serverSe ); last_sent.writeInteger(IntegerDataType.INT4, clientParam); last_sent.writeInteger(IntegerDataType.INT4, NativeConstants.MAX_PACKET_SIZE); - last_sent.writeInteger(IntegerDataType.INT1, AuthenticationProvider.getCharsetForHandshake(enc, serverSession.getCapabilities().getServerVersion())); + last_sent.writeInteger(IntegerDataType.INT1, collationIndex); last_sent.writeBytes(StringLengthDataType.STRING_FIXED, new byte[23]); // Set of bytes reserved for future use. // User/Password data @@ -638,7 +623,8 @@ private NativePacketPayload createHandshakeResponsePacket(ServerSession serverSe private NativePacketPayload createChangeUserPacket(ServerSession serverSession, String pluginName, ArrayList toServer) { // write Auth Response Packet long clientParam = serverSession.getClientParam(); - String enc = getEncodingForHandshake(); + int collationIndex = serverSession.getCharsetSettings().configurePreHandshake(false); + String enc = serverSession.getCharsetSettings().getPasswordCharacterEncoding(); NativePacketPayload last_sent = new NativePacketPayload(AUTH_411_OVERHEAD + 7 // + 48 // passwordLength @@ -665,7 +651,7 @@ private NativePacketPayload createChangeUserPacket(ServerSession serverSession, last_sent.writeInteger(IntegerDataType.INT1, 0); } - last_sent.writeInteger(IntegerDataType.INT1, AuthenticationProvider.getCharsetForHandshake(enc, serverSession.getCapabilities().getServerVersion())); + last_sent.writeInteger(IntegerDataType.INT1, collationIndex); last_sent.writeInteger(IntegerDataType.INT1, 0); // two (little-endian) bytes for charset in this packet // plugin name diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeCapabilities.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeCapabilities.java index 4bf751552..73d916133 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeCapabilities.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeCapabilities.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -52,27 +52,20 @@ public class NativeCapabilities implements ServerCapabilities { private int authPluginDataLength = 0; private boolean serverHasFracSecsSupport = true; - public NativeCapabilities() { - } - - public NativePacketPayload getInitialHandshakePacket() { - return this.initialHandshakePacket; - } - - public void setInitialHandshakePacket(NativePacketPayload initialHandshakePacket) { + public NativeCapabilities(NativePacketPayload initialHandshakePacket) { this.initialHandshakePacket = initialHandshakePacket; // Get the protocol version - setProtocolVersion((byte) initialHandshakePacket.readInteger(IntegerDataType.INT1)); + this.protocolVersion = (byte) initialHandshakePacket.readInteger(IntegerDataType.INT1); try { - setServerVersion(ServerVersion.parseVersion(initialHandshakePacket.readString(StringSelfDataType.STRING_TERM, "ASCII"))); + this.serverVersion = ServerVersion.parseVersion(initialHandshakePacket.readString(StringSelfDataType.STRING_TERM, "ASCII")); // read connection id - setThreadId(initialHandshakePacket.readInteger(IntegerDataType.INT4)); + this.threadId = initialHandshakePacket.readInteger(IntegerDataType.INT4); // read auth-plugin-data-part-1 (string[8]) - setSeed(initialHandshakePacket.readString(StringLengthDataType.STRING_FIXED, "ASCII", 8)); + this.seed = initialHandshakePacket.readString(StringLengthDataType.STRING_FIXED, "ASCII", 8); // read filler ([00]) initialHandshakePacket.readInteger(IntegerDataType.INT1); @@ -85,9 +78,9 @@ public void setInitialHandshakePacket(NativePacketPayload initialHandshakePacket } // read character set (1 byte) - setServerDefaultCollationIndex((int) initialHandshakePacket.readInteger(IntegerDataType.INT1)); + this.serverDefaultCollationIndex = (int) initialHandshakePacket.readInteger(IntegerDataType.INT1); // read status flags (2 bytes) - setStatusFlags((int) initialHandshakePacket.readInteger(IntegerDataType.INT2)); + this.statusFlags = (int) initialHandshakePacket.readInteger(IntegerDataType.INT2); // read capability flags (upper 2 bytes) flags |= (int) initialHandshakePacket.readInteger(IntegerDataType.INT2) << 16; @@ -117,6 +110,10 @@ public void setInitialHandshakePacket(NativePacketPayload initialHandshakePacket } } + public NativePacketPayload getInitialHandshakePacket() { + return this.initialHandshakePacket; + } + @Override public int getCapabilityFlags() { return this.capabilityFlags; @@ -127,22 +124,10 @@ public void setCapabilityFlags(int capabilityFlags) { this.capabilityFlags = capabilityFlags; } - public byte getProtocolVersion() { - return this.protocolVersion; - } - - public void setProtocolVersion(byte protocolVersion) { - this.protocolVersion = protocolVersion; - } - public ServerVersion getServerVersion() { return this.serverVersion; } - public void setServerVersion(ServerVersion serverVersion) { - this.serverVersion = serverVersion; - } - public long getThreadId() { return this.threadId; } @@ -155,10 +140,6 @@ public String getSeed() { return this.seed; } - public void setSeed(String seed) { - this.seed = seed; - } - /** * * @return Collation index which server provided in handshake greeting packet @@ -167,32 +148,14 @@ public int getServerDefaultCollationIndex() { return this.serverDefaultCollationIndex; } - /** - * Stores collation index which server provided in handshake greeting packet. - * - * @param serverDefaultCollationIndex - * server default collation index - */ - public void setServerDefaultCollationIndex(int serverDefaultCollationIndex) { - this.serverDefaultCollationIndex = serverDefaultCollationIndex; - } - public int getStatusFlags() { return this.statusFlags; } - public void setStatusFlags(int statusFlags) { - this.statusFlags = statusFlags; - } - public int getAuthPluginDataLength() { return this.authPluginDataLength; } - public void setAuthPluginDataLength(int authPluginDataLength) { - this.authPluginDataLength = authPluginDataLength; - } - @Override public boolean serverSupportsFracSecs() { return this.serverHasFracSecsSupport; diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java index 788e659b0..cee062867 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeProtocol.java @@ -30,11 +30,9 @@ package com.mysql.cj.protocol.a; import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStreamWriter; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; @@ -68,6 +66,8 @@ import com.mysql.cj.MessageBuilder; import com.mysql.cj.Messages; import com.mysql.cj.MysqlType; +import com.mysql.cj.NativeCharsetSettings; +import com.mysql.cj.NativeSession; import com.mysql.cj.Query; import com.mysql.cj.QueryAttributesBindValue; import com.mysql.cj.QueryAttributesBindings; @@ -99,7 +99,6 @@ import com.mysql.cj.log.ProfilerEvent; import com.mysql.cj.log.ProfilerEventHandler; import com.mysql.cj.protocol.AbstractProtocol; -import com.mysql.cj.protocol.AuthenticationProvider; import com.mysql.cj.protocol.ColumnDefinition; import com.mysql.cj.protocol.ExportControlled; import com.mysql.cj.protocol.FullReadInputStream; @@ -189,12 +188,6 @@ public class NativeProtocol extends AbstractProtocol implem protected Map, ProtocolEntityReader> PROTOCOL_ENTITY_CLASS_TO_TEXT_READER; protected Map, ProtocolEntityReader> PROTOCOL_ENTITY_CLASS_TO_BINARY_READER; - /** - * Does the character set of this connection match the character set of the - * platform - */ - protected boolean platformDbCharsetMatches = true; // changed once we've connected. - private int statementExecutionDepth = 0; private List queryInterceptors; @@ -211,34 +204,8 @@ public class NativeProtocol extends AbstractProtocol implem */ private String queryComment = null; - /** - * We store the platform 'encoding' here, only used to avoid munging filenames for LOAD DATA LOCAL INFILE... - */ - private static String jvmPlatformCharset = null; - private NativeMessageBuilder commandBuilder = null; - static { - OutputStreamWriter outWriter = null; - - // - // Use the I/O system to get the encoding (if possible), to avoid security restrictions on System.getProperty("file.encoding") in applets (why is that - // restricted?) - // - try { - outWriter = new OutputStreamWriter(new ByteArrayOutputStream()); - jvmPlatformCharset = outWriter.getEncoding(); - } finally { - try { - if (outWriter != null) { - outWriter.close(); - } - } catch (IOException ioEx) { - // ignore - } - } - } - public static NativeProtocol getInstance(Session session, SocketConnection socketConnection, PropertySet propertySet, Log log, TransactionEventHandler transactionManager) { NativeProtocol protocol = new NativeProtocol(log); @@ -332,8 +299,7 @@ public void negotiateSSLConnection() { NativePacketPayload packet = new NativePacketPayload(SSL_REQUEST_LENGTH); packet.writeInteger(IntegerDataType.INT4, clientParam); packet.writeInteger(IntegerDataType.INT4, NativeConstants.MAX_PACKET_SIZE); - packet.writeInteger(IntegerDataType.INT1, AuthenticationProvider.getCharsetForHandshake(this.authProvider.getEncodingForHandshake(), - this.serverSession.getCapabilities().getServerVersion())); + packet.writeInteger(IntegerDataType.INT1, this.serverSession.getCharsetSettings().configurePreHandshake(false)); packet.writeBytes(StringLengthDataType.STRING_FIXED, new byte[23]); // Set of bytes reserved for future use. send(packet, packet.getPosition()); @@ -390,6 +356,8 @@ public void beforeHandshake() { // Create session state this.serverSession = new NativeServerSession(this.propertySet); + this.serverSession.setCharsetSettings(new NativeCharsetSettings((NativeSession) this.session)); + // Read the first packet this.serverSession.setCapabilities(readServerCapabilities()); @@ -511,11 +479,7 @@ public NativeCapabilities readServerCapabilities() { rejectProtocol(buf); } - NativeCapabilities serverCapabilities = new NativeCapabilities(); - serverCapabilities.setInitialHandshakePacket(buf); - - return serverCapabilities; - + return new NativeCapabilities(buf); } @Override @@ -744,7 +708,7 @@ public void checkErrorMessage(NativePacketPayload resultPacket) { String xOpen = null; - serverErrorMessage = resultPacket.readString(StringSelfDataType.STRING_TERM, this.serverSession.getErrorMessageEncoding()); + serverErrorMessage = resultPacket.readString(StringSelfDataType.STRING_TERM, this.serverSession.getCharsetSettings().getErrorMessageEncoding()); if (serverErrorMessage.charAt(0) == '#') { @@ -943,7 +907,7 @@ public final T sendQueryString(Query callingQuery, String sendPacket.writeBytes(StringLengthDataType.STRING_FIXED, Constants.SPACE_STAR_SLASH_SPACE_AS_BYTES); } - if (!this.platformDbCharsetMatches && StringUtils.startsWithIgnoreCaseAndWs(query, "LOAD DATA")) { + if (!this.session.getServerSession().getCharsetSettings().doesPlatformDbCharsetMatches() && StringUtils.startsWithIgnoreCaseAndWs(query, "LOAD DATA")) { sendPacket.writeBytes(StringLengthDataType.STRING_FIXED, StringUtils.getBytes(query)); } else { sendPacket.writeBytes(StringLengthDataType.STRING_FIXED, StringUtils.getBytes(query, characterEncoding)); @@ -1340,27 +1304,7 @@ public void changeUser(String user, String password, String database) { this.packetSender = this.packetSender.undecorateAll(); this.packetReader = this.packetReader.undecorateAll(); - this.authProvider.changeUser(this.serverSession, user, password, database); - } - - /** - * Determines if the database charset is the same as the platform charset - */ - public void checkForCharsetMismatch() { - String characterEncoding = this.propertySet.getStringProperty(PropertyKey.characterEncoding).getValue(); - if (characterEncoding != null) { - String encodingToCheck = jvmPlatformCharset; - - if (encodingToCheck == null) { - encodingToCheck = Constants.PLATFORM_ENCODING; - } - - if (encodingToCheck == null) { - this.platformDbCharsetMatches = false; - } else { - this.platformDbCharsetMatches = encodingToCheck.equals(characterEncoding); - } - } + this.authProvider.changeUser(user, password, database); } protected boolean useNanosForElapsedTime() { @@ -1405,7 +1349,7 @@ public void connect(String user, String password, String database) { beforeHandshake(); - this.authProvider.connect(this.serverSession, user, password, database); + this.authProvider.connect(user, password, database); } protected boolean isDataAvailable() { @@ -1447,22 +1391,6 @@ public void dumpPacketRingBuffer() { } } - public boolean doesPlatformDbCharsetMatches() { - return this.platformDbCharsetMatches; - } - - public String getPasswordCharacterEncoding() { - String encoding; - if ((encoding = this.propertySet.getStringProperty(PropertyKey.passwordCharacterEncoding).getStringValue()) != null) { - return encoding; - } - if ((encoding = this.propertySet.getStringProperty(PropertyKey.characterEncoding).getValue()) != null) { - return encoding; - } - return "UTF-8"; - - } - public boolean versionMeetsMinimum(int major, int minor, int subminor) { return this.serverSession.getServerVersion().meetsMinimum(new ServerVersion(major, minor, subminor)); } @@ -1743,7 +1671,7 @@ public final T readServerStatusForResultSets(NativePacketPayload rowPacket, checkTransactionState(); } else { // read OK packet - OkPacket ok = OkPacket.parse(rowPacket, this.serverSession.getErrorMessageEncoding()); + OkPacket ok = OkPacket.parse(rowPacket, this.serverSession.getCharsetSettings().getErrorMessageEncoding()); result = (T) ok; this.serverSession.setStatusFlags(ok.getStatusFlags(), saveOldStatus); @@ -2272,8 +2200,8 @@ public void configureTimeZone() { public void initServerSession() { configureTimeZone(); - if (this.session.getServerSession().getServerVariables().containsKey("max_allowed_packet")) { - int serverMaxAllowedPacket = this.session.getServerSession().getServerVariable("max_allowed_packet", -1); + if (this.serverSession.getServerVariables().containsKey("max_allowed_packet")) { + int serverMaxAllowedPacket = this.serverSession.getServerVariable("max_allowed_packet", -1); // use server value if maxAllowedPacket hasn't been given, or max_allowed_packet is smaller if (serverMaxAllowedPacket != -1 && (!this.maxAllowedPacket.isExplicitlySet() || serverMaxAllowedPacket < this.maxAllowedPacket.getValue())) { @@ -2296,5 +2224,7 @@ public void initServerSession() { blobSendChunkSize.setValue(allowedBlobSendChunkSize); } } + + this.serverSession.getCharsetSettings().configurePostHandshake(false); } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeServerSession.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeServerSession.java index dfacc973e..fb5042d1e 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeServerSession.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/NativeServerSession.java @@ -33,8 +33,7 @@ import java.util.Map; import java.util.TimeZone; -import com.mysql.cj.CharsetMapping; -import com.mysql.cj.Messages; +import com.mysql.cj.CharsetSettings; import com.mysql.cj.ServerVersion; import com.mysql.cj.conf.PropertyKey; import com.mysql.cj.conf.PropertySet; @@ -44,7 +43,6 @@ import com.mysql.cj.protocol.ServerCapabilities; import com.mysql.cj.protocol.ServerSession; import com.mysql.cj.protocol.ServerSessionStateController; -import com.mysql.cj.util.StringUtils; import com.mysql.cj.util.TimeUtil; public class NativeServerSession implements ServerSession { @@ -85,34 +83,13 @@ public class NativeServerSession implements ServerSession { private NativeCapabilities capabilities; private int oldStatusFlags = 0; private int statusFlags = 0; - private int serverDefaultCollationIndex; private long clientParam = 0; private NativeServerSessionStateController serverSessionStateController; /** The map of server variables that we retrieve at connection init. */ private Map serverVariables = new HashMap<>(); - public Map indexToCustomMysqlCharset = null; - - public Map mysqlCharsetToCustomMblen = null; - - /** - * What character set is the metadata returned in? - */ - private String characterSetMetadata = null; - private int metadataCollationIndex; - - /** - * The character set we want results and result metadata returned in (null == - * results in any charset, metadata in UTF-8). - */ - private String characterSetResultsOnServer = null; - - /** - * The (Java) encoding used to interpret error messages received from the server. - * We use character_set_results (since MySQL 5.5) if it is not null or UTF-8 otherwise. - */ - private String errorMessageEncoding = "Cp1252"; // to begin with, changes after we talk to the server + private CharsetSettings charsetSettings; /** Are we in autoCommit mode? */ private boolean autoCommit = true; @@ -128,9 +105,6 @@ public NativeServerSession(PropertySet propertySet) { this.propertySet = propertySet; this.cacheDefaultTimeZone = this.propertySet.getBooleanProperty(PropertyKey.cacheDefaultTimeZone); this.serverSessionStateController = new NativeServerSessionStateController(); - - // preconfigure some server variables which are consulted before their initialization from server - this.serverVariables.put("character_set_server", "utf8"); } @Override @@ -255,16 +229,6 @@ public boolean supportsQueryAttributes() { return (this.clientParam & CLIENT_QUERY_ATTRIBUTES) != 0; } - @Override - public int getServerDefaultCollationIndex() { - return this.serverDefaultCollationIndex; - } - - @Override - public void setServerDefaultCollationIndex(int serverDefaultCollationIndex) { - this.serverDefaultCollationIndex = serverDefaultCollationIndex; - } - @Override public Map getServerVariables() { return this.serverVariables; @@ -291,12 +255,6 @@ public void setServerVariables(Map serverVariables) { this.serverVariables = serverVariables; } - public boolean characterSetNamesMatches(String mysqlEncodingName) { - // set names is equivalent to character_set_client ..._results and ..._connection, but we set _results later, so don't check it here. - return (mysqlEncodingName != null && mysqlEncodingName.equalsIgnoreCase(getServerVariable("character_set_client")) - && mysqlEncodingName.equalsIgnoreCase(getServerVariable("character_set_connection"))); - } - public final ServerVersion getServerVersion() { return this.capabilities.getServerVersion(); } @@ -333,159 +291,6 @@ public boolean isSetNeededForAutoCommitMode(boolean autoCommitFlag, boolean elid return true; } - @Override - public String getErrorMessageEncoding() { - return this.errorMessageEncoding; - } - - @Override - public void setErrorMessageEncoding(String errorMessageEncoding) { - this.errorMessageEncoding = errorMessageEncoding; - } - - public String getServerDefaultCharset() { - String charset = null; - if (this.indexToCustomMysqlCharset != null) { - charset = this.indexToCustomMysqlCharset.get(getServerDefaultCollationIndex()); - } - if (charset == null) { - charset = CharsetMapping.getMysqlCharsetNameForCollationIndex(getServerDefaultCollationIndex()); - } - return charset != null ? charset : getServerVariable("character_set_server"); - } - - public int getMaxBytesPerChar(String javaCharsetName) { - return getMaxBytesPerChar(null, javaCharsetName); - } - - public int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName) { - - String charset = null; - int res = 1; - - // if we can get it by charsetIndex just doing it - - // getting charset name from dynamic maps in connection; we do it before checking against static maps because custom charset on server can be mapped - // to index from our static map key's diapason - if (this.indexToCustomMysqlCharset != null) { - charset = this.indexToCustomMysqlCharset.get(charsetIndex); - } - // checking against static maps if no custom charset found - if (charset == null) { - charset = CharsetMapping.getMysqlCharsetNameForCollationIndex(charsetIndex); - } - - // if we didn't find charset name by index - if (charset == null) { - charset = CharsetMapping.getMysqlCharsetForJavaEncoding(javaCharsetName, getServerVersion()); - } - - // checking against dynamic maps in connection - Integer mblen = null; - if (this.mysqlCharsetToCustomMblen != null) { - mblen = this.mysqlCharsetToCustomMblen.get(charset); - } - - // checking against static maps - if (mblen == null) { - mblen = CharsetMapping.getMblen(charset); - } - - if (mblen != null) { - res = mblen.intValue(); - } - - return res; // we don't know - } - - public String getEncodingForIndex(int charsetIndex) { - String javaEncoding = null; - - String characterEncoding = this.propertySet.getStringProperty(PropertyKey.characterEncoding).getValue(); - - if (charsetIndex != NativeConstants.NO_CHARSET_INFO) { - try { - // getting charset name from dynamic maps in connection; we do it before checking against static maps because custom charset on server can be mapped - // to index from our static map key's diapason - if (this.indexToCustomMysqlCharset != null) { - String cs = this.indexToCustomMysqlCharset.get(charsetIndex); - if (cs != null) { - javaEncoding = CharsetMapping.getJavaEncodingForMysqlCharset(cs, characterEncoding); - } - } - // checking against static maps if no custom charset found - if (javaEncoding == null) { - javaEncoding = CharsetMapping.getJavaEncodingForCollationIndex(charsetIndex, characterEncoding); - } - - } catch (ArrayIndexOutOfBoundsException outOfBoundsEx) { - throw ExceptionFactory.createException(WrongArgumentException.class, Messages.getString("Connection.11", new Object[] { charsetIndex })); - } - - // Punt - if (javaEncoding == null) { - javaEncoding = characterEncoding; - } - } else { - javaEncoding = characterEncoding; - } - - return javaEncoding; - } - - public void configureCharacterSets() { - // - // We need to figure out what character set metadata and error messages will be returned in, and then map them to Java encoding names - // - // We've already set it, and it might be different than what was originally on the server, which is why we use the "special" key to retrieve it - String characterSetResultsOnServerMysql = getServerVariable(LOCAL_CHARACTER_SET_RESULTS); - - if (characterSetResultsOnServerMysql == null || StringUtils.startsWithIgnoreCaseAndWs(characterSetResultsOnServerMysql, "NULL") - || characterSetResultsOnServerMysql.length() == 0) { - String defaultMetadataCharsetMysql = getServerVariable("character_set_system"); - String defaultMetadataCharset = null; - - if (defaultMetadataCharsetMysql != null) { - defaultMetadataCharset = CharsetMapping.getJavaEncodingForMysqlCharset(defaultMetadataCharsetMysql); - } else { - defaultMetadataCharset = "UTF-8"; - } - - this.characterSetMetadata = defaultMetadataCharset; - setErrorMessageEncoding("UTF-8"); - } else { - this.characterSetResultsOnServer = CharsetMapping.getJavaEncodingForMysqlCharset(characterSetResultsOnServerMysql); - this.characterSetMetadata = this.characterSetResultsOnServer; - setErrorMessageEncoding(this.characterSetResultsOnServer); - } - - this.metadataCollationIndex = CharsetMapping.getCollationIndexForJavaEncoding(this.characterSetMetadata, getServerVersion()); - } - - public String getCharacterSetMetadata() { - return this.characterSetMetadata; - } - - public void setCharacterSetMetadata(String characterSetMetadata) { - this.characterSetMetadata = characterSetMetadata; - } - - public int getMetadataCollationIndex() { - return this.metadataCollationIndex; - } - - public void setMetadataCollationIndex(int metadataCollationIndex) { - this.metadataCollationIndex = metadataCollationIndex; - } - - public String getCharacterSetResultsOnServer() { - return this.characterSetResultsOnServer; - } - - public void setCharacterSetResultsOnServer(String characterSetResultsOnServer) { - this.characterSetResultsOnServer = characterSetResultsOnServer; - } - public void preserveOldTransactionState() { this.statusFlags |= this.oldStatusFlags & SERVER_STATUS_IN_TRANS; } @@ -527,16 +332,6 @@ public boolean isServerTruncatesFracSecs() { return sqlModeAsString != null && sqlModeAsString.indexOf("TIME_TRUNCATE_FRACTIONAL") != -1; } - @Override - public long getThreadId() { - return this.capabilities.getThreadId(); - } - - @Override - public void setThreadId(long threadId) { - this.capabilities.setThreadId(threadId); - } - public boolean isAutoCommit() { return this.autoCommit; } @@ -579,4 +374,13 @@ public ServerSessionStateController getServerSessionStateController() { return this.serverSessionStateController; } + @Override + public CharsetSettings getCharsetSettings() { + return this.charsetSettings; + } + + @Override + public void setCharsetSettings(CharsetSettings charsetSettings) { + this.charsetSettings = charsetSettings; + } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TextResultsetReader.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TextResultsetReader.java index 43e447b7c..d71512747 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TextResultsetReader.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/TextResultsetReader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. + * Copyright (c) 2016, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -103,7 +103,8 @@ public Resultset read(int maxRows, boolean streamResults, NativePacketPayload re // check for file request if (columnCount == NativePacketPayload.NULL_LENGTH) { String charEncoding = this.protocol.getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(); - String fileName = resultPacket.readString(StringSelfDataType.STRING_TERM, this.protocol.doesPlatformDbCharsetMatches() ? charEncoding : null); + String fileName = resultPacket.readString(StringSelfDataType.STRING_TERM, + this.protocol.getServerSession().getCharsetSettings().doesPlatformDbCharsetMatches() ? charEncoding : null); resultPacket = this.protocol.sendFileToServer(fileName); } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/CachingSha2PasswordPlugin.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/CachingSha2PasswordPlugin.java index 088ecc9b3..ba94b5759 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/CachingSha2PasswordPlugin.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/CachingSha2PasswordPlugin.java @@ -91,8 +91,9 @@ public boolean nextAuthenticationStep(NativePacketPayload fromServer, List toServer) { toServer.clear(); - String encoding = this.protocol.versionMeetsMinimum(5, 7, 6) ? this.protocol.getPasswordCharacterEncoding() : "UTF-8"; + String encoding = this.protocol.getServerSession().getCharsetSettings().getPasswordCharacterEncoding(); NativePacketPayload bresp = new NativePacketPayload(StringUtils.getBytes(this.password != null ? this.password : "", encoding)); bresp.setPosition(bresp.getPayloadLength()); diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/MysqlNativePasswordPlugin.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/MysqlNativePasswordPlugin.java index 5786a9da1..ee2da4db8 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/MysqlNativePasswordPlugin.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/a/authentication/MysqlNativePasswordPlugin.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2020, Oracle and/or its affiliates. + * Copyright (c) 2012, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -90,8 +90,8 @@ public boolean nextAuthenticationStep(NativePacketPayload fromServer, List prot, PropertySet propertySet, ExceptionInte } @Override - public void connect(ServerSession serverSession, String userName, String password, String database) { - changeUser(serverSession, userName, password, database); + public void connect(String userName, String password, String database) { + changeUser(userName, password, database); } @Override - public void changeUser(ServerSession serverSession, String userName, String password, String database) { + public void changeUser(String userName, String password, String database) { boolean overTLS = ((XServerCapabilities) this.protocol.getServerSession().getCapabilities()).getTls(); RuntimeProperty authMechProp = this.protocol.getPropertySet().getEnumProperty(PropertyKey.xdevapiAuth); List tryAuthMech; @@ -139,8 +138,4 @@ public void changeUser(ServerSession serverSession, String userName, String pass this.protocol.afterHandshake(); } - @Override - public String getEncodingForHandshake() { - return null; // TODO - } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XProtocol.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XProtocol.java index 7c319f09e..d805e0f1e 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XProtocol.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XProtocol.java @@ -49,7 +49,6 @@ import java.util.function.Consumer; import com.google.protobuf.GeneratedMessageV3; -import com.mysql.cj.CharsetMapping; import com.mysql.cj.Constants; import com.mysql.cj.Messages; import com.mysql.cj.QueryResult; @@ -524,7 +523,7 @@ public void connect(String user, String password, String database) { this.currDatabase = database; beforeHandshake(); - this.authProvider.connect(null, user, password, database); + this.authProvider.connect(user, password, database); } public void changeUser(String user, String password, String database) { @@ -532,7 +531,7 @@ public void changeUser(String user, String password, String database) { this.currPassword = password; this.currDatabase = database; - this.authProvider.changeUser(null, user, password, database); + this.authProvider.changeUser(user, password, database); } public void afterHandshake() { @@ -582,7 +581,7 @@ public void readAuthenticateOk() { if (notice instanceof XSessionStateChanged) { switch (((XSessionStateChanged) notice).getParamType()) { case Notice.SessionStateChanged_CLIENT_ID_ASSIGNED: - this.getServerSession().setThreadId(((XSessionStateChanged) notice).getValue().getVUnsignedInt()); + this.getServerSession().getCapabilities().setThreadId(((XSessionStateChanged) notice).getValue().getVUnsignedInt()); break; case Notice.SessionStateChanged_ACCOUNT_EXPIRED: // TODO @@ -688,15 +687,6 @@ public void drainRows() { } } - // TODO: put this in CharsetMapping.. - public static Map COLLATION_NAME_TO_COLLATION_INDEX = new java.util.HashMap<>(); - - static { - for (int i = 0; i < CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME.length; ++i) { - COLLATION_NAME_TO_COLLATION_INDEX.put(CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i], i); - } - } - public ColumnDefinition readMetadata() { return readMetadata(null); } @@ -1012,7 +1002,7 @@ public void reset() { } } - this.authProvider.changeUser(null, this.currUser, this.currPassword, this.currDatabase); + this.authProvider.changeUser(this.currUser, this.currPassword, this.currDatabase); } // No prepared statements survived to Mysqlx.Session.Reset. Reset all related control structures. @@ -1034,10 +1024,6 @@ public void changeDatabase(String database) { // TODO: Figure out how this is relevant for X Protocol client Session } - public String getPasswordCharacterEncoding() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - public boolean versionMeetsMinimum(int major, int minor, int subminor) { //TODO: expose this via ServerVersion so calls look like x.getServerVersion().meetsMinimum(major, minor, subminor) throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerCapabilities.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerCapabilities.java index ede5e68dd..c8730c72e 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerCapabilities.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerCapabilities.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Oracle and/or its affiliates. + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -55,6 +55,9 @@ public class XServerCapabilities implements ServerCapabilities { static String SUBKEY_COMPRESSION_SERVER_COMBINE_MIXED_MESSAGES = "server_combine_mixed_messages"; static String SUBKEY_COMPRESSION_SERVER_MAX_COMBINE_MESSAGES = "server_max_combine_messages"; + /** Server-assigned client-id. */ + private long clientId = -1; + public XServerCapabilities(Map capabilities) { this.capabilities = capabilities; } @@ -118,13 +121,23 @@ public ServerVersion getServerVersion() { } @Override - public void setServerVersion(ServerVersion serverVersion) { + public boolean serverSupportsFracSecs() { + return true; + } + + @Override + public int getServerDefaultCollationIndex() { // TODO Auto-generated method stub + return 0; + } + @Override + public long getThreadId() { + return this.clientId; } @Override - public boolean serverSupportsFracSecs() { - return true; + public void setThreadId(long threadId) { + this.clientId = threadId; } } diff --git a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerSession.java b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerSession.java index 43a82b1f6..61461b63a 100644 --- a/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerSession.java +++ b/src/main/protocol-impl/java/com/mysql/cj/protocol/x/XServerSession.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.TimeZone; +import com.mysql.cj.CharsetSettings; import com.mysql.cj.ServerVersion; import com.mysql.cj.exceptions.CJOperationNotSupportedException; import com.mysql.cj.exceptions.ExceptionFactory; @@ -41,8 +42,6 @@ public class XServerSession implements ServerSession { XServerCapabilities serverCapabilities = null; - /** Server-assigned client-id. */ - private long clientId = -1; private TimeZone defaultTimeZone = TimeZone.getDefault(); @@ -81,16 +80,6 @@ public void setOldStatusFlags(int statusFlags) { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); } - @Override - public int getServerDefaultCollationIndex() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void setServerDefaultCollationIndex(int serverDefaultCollationIndex) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - @Override public int getTransactionState() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); @@ -186,11 +175,6 @@ public void setServerVariables(Map serverVariables) { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); } - @Override - public boolean characterSetNamesMatches(String mysqlEncodingName) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - @Override public ServerVersion getServerVersion() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); @@ -201,71 +185,6 @@ public boolean isVersion(ServerVersion version) { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); } - @Override - public String getServerDefaultCharset() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public String getErrorMessageEncoding() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void setErrorMessageEncoding(String errorMessageEncoding) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public int getMaxBytesPerChar(String javaCharsetName) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public int getMaxBytesPerChar(Integer charsetIndex, String javaCharsetName) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public String getEncodingForIndex(int collationIndex) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void configureCharacterSets() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public String getCharacterSetMetadata() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void setCharacterSetMetadata(String characterSetMetadata) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public int getMetadataCollationIndex() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void setMetadataCollationIndex(int metadataCollationIndex) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public String getCharacterSetResultsOnServer() { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - - @Override - public void setCharacterSetResultsOnServer(String characterSetResultsOnServer) { - throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); - } - @Override public boolean isLowerCaseTableNames() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); @@ -296,16 +215,6 @@ public boolean isServerTruncatesFracSecs() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); } - @Override - public long getThreadId() { - return this.clientId; - } - - @Override - public void setThreadId(long threadId) { - this.clientId = threadId; - } - @Override public boolean isAutoCommit() { throw ExceptionFactory.createException(CJOperationNotSupportedException.class, "Not supported"); @@ -327,4 +236,16 @@ public void setSessionTimeZone(TimeZone sessionTimeZone) { public TimeZone getDefaultTimeZone() { return this.defaultTimeZone; } + + @Override + public CharsetSettings getCharsetSettings() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setCharsetSettings(CharsetSettings charsetSettings) { + // TODO Auto-generated method stub + + } } diff --git a/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties b/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties index 494bef02f..5a28bd4a1 100644 --- a/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties +++ b/src/main/resources/com/mysql/cj/LocalizedErrorMessages.properties @@ -105,13 +105,12 @@ Connection.1=Cannot connect to MySQL server on {0}:{1}.\n\nMake sure that there Connection.2=No operations allowed after connection closed. Connection.3=Can''t call commit when autocommit=true Connection.4=Communications link failure during commit(). Transaction resolution unknown. -Connection.5=Java does not support the MySQL character encoding ''{0}''. -Connection.6=Unknown initial character set index ''{0}'' received from server. Initial client character set can be forced via the ''characterEncoding'' property. +Connection.5=Unknown Java encoding for the character set with index ''{0}''. Use the ''customCharsetMapping'' property to force it. +Connection.6=Unknown character set index ''{0}'' received from server. The appropriate client character set can be forced via the ''characterEncoding'' property. Connection.7=Can''t map {0} given for characterSetResults to a supported MySQL encoding. Connection.8=Unable to use encoding: {0} Connection.9=No timezone mapping entry for ''{0}'' Connection.10=Illegal connection port value ''{0}'' -Connection.11=Unknown character set index ''{0}'' was received from server. Connection.12=Could not map transaction isolation ''{0}'' to a valid JDBC level. Connection.13=Could not retrieve transaction isolation level from server Connection.15=Connection setting too low for ''maxAllowedPacket''. When ''useServerPrepStmts=true'', ''maxAllowedPacket'' must be higher than {0}. Check also ''max_allowed_packet'' in MySQL configuration files. @@ -295,15 +294,13 @@ MysqlIO.105=Negative skip length not allowed MysqlIO.106=Value ''0000-00-00'' can not be represented as java.sql.Date MysqlIO.107=Value ''0000-00-00'' can not be represented as java.sql.Timestamp MysqlIO.111=Could not allocate packet of {0} bytes required for "LOAD DATA LOCAL INFILE" operation. Try increasing max heap allocation for JVM or decreasing server variable ''max_allowed_packet'' -MysqlIO.113=Invalid character set index for encoding: {0} +MysqlIO.113=Invalid character set index {0} for handshake, only values 1-255 are allowed. MysqlIO.EOF=Can not read response from server. Expected to read {0} bytes, read {1} bytes before connection was unexpectedly lost. MysqlIO.NoInnoDBStatusFound=No InnoDB status output returned by server. MysqlIO.InnoDBStatusFailed=Couldn''t retrieve InnoDB status due to underlying exception: MysqlIO.LoadDataLocalNotAllowed=Server asked for stream in response to "LOAD DATA LOCAL INFILE" but functionality is not enabled at client by setting "allowLoadLocalInfile=true" or specifying a path with ''allowLoadLocalInfileInPath''. MysqlIo.BadQueryInterceptor=Unable to load query interceptor ''{0}''. -MysqlNativePasswordPlugin.1=Unsupported character encoding ''{0}'' for ''passwordCharacterEncoding'' or ''characterEncoding''. - MysqlParameterMetadata.0=Parameter metadata not available for the given statement MysqlParameterMetadata.1=Parameter index of ''{0}'' is invalid. MysqlParameterMetadata.2=Parameter index of ''{0}'' is greater than number of parameters, which is ''{1}''. @@ -592,7 +589,6 @@ Session.Create.Failover.0=Unable to connect to any of the target hosts. Sha256PasswordPlugin.0=Unable to read public key {0} Sha256PasswordPlugin.1=Unable to close public key file Sha256PasswordPlugin.2=Public Key Retrieval is not allowed -Sha256PasswordPlugin.3=Unsupported character encoding ''{0}'' for ''passwordCharacterEncoding'' or ''characterEncoding''. Schema.CreateCollection=The server doesn't support the requested operation. Please update the MySQL Server and or Client library @@ -812,17 +808,18 @@ ConnectionProperties.cachePrepStmts=Should the driver cache the parsing stage of ConnectionProperties.cacheRSMetadata=Should the driver cache ResultSetMetaData for Statements and PreparedStatements? (Req. JDK-1.4+, true/false, default ''false'') ConnectionProperties.cacheServerConfiguration=Should the driver cache the results of ''SHOW VARIABLES'' and ''SHOW COLLATION'' on a per-URL basis? ConnectionProperties.callableStmtCacheSize=If ''cacheCallableStmts'' is enabled, how many callable statements should be cached? -ConnectionProperties.characterEncoding=What character encoding should the driver use when dealing with strings? (defaults is to ''autodetect'') -ConnectionProperties.characterSetResults=Character set to tell the server to return results as. +ConnectionProperties.characterEncoding=Instructs the server to set session system variables ''character_set_client'' and ''character_set_connection'' to the default character set for the specified Java encoding and set ''collation_connection'' to the default collation for this character set. If neither this property nor the property ''connectionCollation'' is set:[CR]For Connector/J 8.0.25 and earlier, the driver will try to use the server default character set;[CR]For Connector/J 8.0.26 and later, the driver will use "utf8mb4". +ConnectionProperties.characterSetResults=Instructs the server to return the data encoded with the default character set for the specified Java encoding. If not set or set to "null", the server will send data in its original character set and the driver will decode it according to the result metadata. ConnectionProperties.clientInfoProvider=The name of a class that implements the com.mysql.cj.jdbc.ClientInfoProvider interface in order to support JDBC-4.0''s Connection.get/setClientInfo() methods ConnectionProperties.clobberStreamingResults=This will cause a ''streaming'' ResultSet to be automatically closed, and any outstanding data still streaming from the server to be discarded if another query is executed before all the data has been read from the server. ConnectionProperties.clobCharacterEncoding=The character encoding to use for sending and retrieving TEXT, MEDIUMTEXT and LONGTEXT values instead of the configured connection characterEncoding ConnectionProperties.compensateOnDuplicateKeyUpdateCounts=Should the driver compensate for the update counts of "ON DUPLICATE KEY" INSERT statements (2 = 1, 0 = 1) when using prepared statements? -ConnectionProperties.connectionCollation=If set, tells the server to use this collation in SET NAMES charset COLLATE connectionCollation. Also overrides the characterEncoding with those corresponding to the character set of this collation. +ConnectionProperties.connectionCollation=Instructs the server to set session system variable ''collation_connection'' to the specified collation name and set ''character_set_client'' and ''character_set_connection'' to the corresponding character set. This property overrides the value of ''characterEncoding'' with the character set this collation belongs to. If neither this property nor the property ''characterEncoding'' is set:[CR]For Connector/J 8.0.25 and earlier, the driver will try to use the server default character set;[CR]For Connector/J 8.0.26 and later, the driver will use "utf8mb4" default collation. ConnectionProperties.connectionLifecycleInterceptors=A comma-delimited list of classes that implement "com.mysql.cj.jdbc.interceptors.ConnectionLifecycleInterceptor" that should notified of connection lifecycle events (creation, destruction, commit, rollback, setting the current database and changing the autocommit mode) and potentially alter the execution of these commands. ConnectionLifecycleInterceptors are "stackable", more than one interceptor may be specified via the configuration property as a comma-delimited list, with the interceptors executed in order from left to right. ConnectionProperties.connectTimeout=Timeout for socket connect (in milliseconds), with 0 being no timeout. Only works on JDK-1.4 or newer. Defaults to ''0''. ConnectionProperties.continueBatchOnError=Should the driver continue processing batch commands if one statement fails. The JDBC spec allows either way (defaults to ''true''). ConnectionProperties.createDatabaseIfNotExist=Creates the database given in the URL if it doesn''t yet exist. Assumes the configured user has permissions to create databases. +ConnectionProperties.customCharsetMapping=A comma-delimited list of custom "charset:java encoding" pairs.[CR]In case the MySQL server is configured with custom character sets and ''detectCustomCollations=true'', Connector/J needs to know which Java character encoding to use for the data represented by these character sets. Example usage: ''customCharsetMapping=charset1:UTF-8,charset2:Cp1252''. ConnectionProperties.defaultFetchSize=The driver will call setFetchSize(n) with this value on all newly-created Statements ConnectionProperties.useServerPrepStmts=Use server-side prepared statements if the server supports them? ConnectionProperties.dontTrackOpenResources=The JDBC specification requires the driver to automatically track and close resources, however if your application doesn''t do a good job of explicitly calling close() on statements or result sets, this can cause memory leakage. Setting this property to true relaxes this constraint, and can be more memory efficient for some applications. Also the automatic closing of the Statement and current ResultSet in Statement.closeOnCompletion() and Statement.getMoreResults ([Statement.CLOSE_CURRENT_RESULT | Statement.CLOSE_ALL_RESULTS]), respectively, ceases to happen. This property automatically sets holdResultsOpenOverStatementClose=true. @@ -961,7 +958,7 @@ ConnectionProperties.sendFractionalSeconds=If set to "false", the fractional sec ConnectionProperties.sendFractionalSecondsForTime=If set to "false", the fractional seconds of java.sql.Time will be ignored as required by JDBC specification. If set to "true", it's value is rendered with fractional seconds allowing to store milliseconds into MySQL TIME column. This option applies only to prepared statements, callable statements or updatable result sets. It has no effect if sendFractionalSeconds=false. ConnectionProperties.useColumnNamesInFindColumn=Prior to JDBC-4.0, the JDBC specification had a bug related to what could be given as a "column name" to ResultSet methods like findColumn(), or getters that took a String property. JDBC-4.0 clarified "column name" to mean the label, as given in an "AS" clause and returned by ResultSetMetaData.getColumnLabel(), and if no AS clause, the column name. Setting this property to "true" will give behavior that is congruent to JDBC-3.0 and earlier versions of the JDBC specification, but which because of the specification bug could give unexpected results. This property is preferred over "useOldAliasMetadataBehavior" unless you need the specific behavior that it provides with respect to ResultSetMetadata. ConnectionProperties.useAffectedRows=Don''t set the CLIENT_FOUND_ROWS flag when connecting to the server (not JDBC-compliant, will break most applications that rely on "found" rows vs. "affected rows" for DML statements), but does cause "correct" update counts from "INSERT ... ON DUPLICATE KEY UPDATE" statements to be returned by the server. -ConnectionProperties.passwordCharacterEncoding=What character encoding is used for passwords? Leaving this set to the default value (null), uses the value set in "characterEncoding" if there is one, otherwise uses UTF-8 as default encoding. If the password contains non-ASCII characters, the password encoding must match what server encoding was set to when the password was created. For passwords in other character encodings, the encoding will have to be specified with this property (or with "characterEncoding"), as it''s not possible for the driver to auto-detect this. +ConnectionProperties.passwordCharacterEncoding=Instructs the server to use the default character set for the specified Java encoding during the authentication phase. If this property is not set, Connector/J falls back to the collation name specified in the property ''connectionCollation'' or to the Java encoding specified in the property ''characterEncoding'', in that order of priority. The "utf8mb4" default collation is used if none of the properties is set. ConnectionProperties.exceptionInterceptors=Comma-delimited list of classes that implement com.mysql.cj.exceptions.ExceptionInterceptor. These classes will be instantiated one per Connection instance, and all SQLExceptions thrown by the driver will be allowed to be intercepted by these interceptors, in a chained fashion, with the first class listed as the head of the chain. ConnectionProperties.maxAllowedPacket=Maximum allowed packet size to send to server. If not set, the value of system variable ''max_allowed_packet'' will be used to initialize this upon connecting. This value will not take effect if set larger than the value of ''max_allowed_packet''. Also, due to an internal dependency with the property "blobSendChunkSize", this setting has a minimum value of "8203" if "useServerPrepStmts" is set to "true". ConnectionProperties.queryTimeoutKillsConnection=If the timeout given in Statement.setQueryTimeout() expires, should the driver forcibly abort the Connection instead of attempting to abort the query? diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/CallableStatement.java b/src/main/user-impl/java/com/mysql/cj/jdbc/CallableStatement.java index 9b4fce5eb..8a5933398 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/CallableStatement.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/CallableStatement.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. + * Copyright (c) 2002, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -705,8 +705,8 @@ public void clearParameters() throws SQLException { */ private void fakeParameterTypes(boolean isReallyProcedure) throws SQLException { synchronized (checkClosed().getConnectionMutex()) { - String encoding = this.connection.getSession().getServerSession().getCharacterSetMetadata(); - int collationIndex = this.connection.getSession().getServerSession().getMetadataCollationIndex(); + String encoding = this.connection.getSession().getServerSession().getCharsetSettings().getMetadataEncoding(); + int collationIndex = this.connection.getSession().getServerSession().getCharsetSettings().getMetadataCollationIndex(); Field[] fields = new Field[13]; fields[0] = new Field("", "PROCEDURE_CAT", collationIndex, encoding, MysqlType.CHAR, 0); diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/ConnectionImpl.java b/src/main/user-impl/java/com/mysql/cj/jdbc/ConnectionImpl.java index fcac83b45..1b9d7d8d0 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/ConnectionImpl.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/ConnectionImpl.java @@ -202,12 +202,6 @@ public int hashCode() { } } - /** - * The mapping between MySQL charset names and Java charset names. - * Initialized by loadCharacterSetMapping() - */ - public static Map charsetMap; - /** Default logger class name */ protected static final String DEFAULT_LOGGER_CLASS = StandardLogger.class.getName(); @@ -561,7 +555,7 @@ public void changeUser(String userName, String newPassword) throws SQLException this.user = userName; this.password = newPassword; - this.session.configureClientCharacterSet(true); + this.session.getServerSession().getCharsetSettings().configurePostHandshake(true); this.session.setSessionVariables(); @@ -1134,7 +1128,7 @@ public String getCatalog() throws SQLException { @Override public String getCharacterSetMetadata() { synchronized (getConnectionMutex()) { - return this.session.getServerSession().getCharacterSetMetadata(); + return this.session.getServerSession().getCharsetSettings().getMetadataEncoding(); } } @@ -1177,8 +1171,8 @@ private java.sql.DatabaseMetaData getMetaData(boolean checkClosed, boolean check this.nullStatementResultSetFactory); if (getSession() != null && getSession().getProtocol() != null) { - dbmeta.setMetadataEncoding(getSession().getServerSession().getCharacterSetMetadata()); - dbmeta.setMetadataCollationIndex(getSession().getServerSession().getMetadataCollationIndex()); + dbmeta.setMetadataEncoding(getSession().getServerSession().getCharsetSettings().getMetadataEncoding()); + dbmeta.setMetadataCollationIndex(getSession().getServerSession().getCharsetSettings().getMetadataCollationIndex()); } return dbmeta; @@ -1305,8 +1299,6 @@ private void initializePropsFromServer() throws SQLException { this.autoIncrementIncrement = this.session.getServerSession().getServerVariable("auto_increment_increment", 1); - this.session.buildCollationMapping(); - try { LicenseConfiguration.checkLicenseType(this.session.getServerSession().getServerVariables()); } catch (CJException e) { @@ -1317,20 +1309,11 @@ private void initializePropsFromServer() throws SQLException { checkTransactionIsolationLevel(); - this.session.checkForCharsetMismatch(); - - this.session.configureClientCharacterSet(false); - handleAutoCommitDefaults(); - // - // We need to figure out what character set metadata and error messages will be returned in, and then map them to Java encoding names - // - // We've already set it, and it might be different than what was originally on the server, which is why we use the "special" key to retrieve it - this.session.getServerSession().configureCharacterSets(); - - ((com.mysql.cj.jdbc.DatabaseMetaData) this.dbmd).setMetadataEncoding(getSession().getServerSession().getCharacterSetMetadata()); - ((com.mysql.cj.jdbc.DatabaseMetaData) this.dbmd).setMetadataCollationIndex(getSession().getServerSession().getMetadataCollationIndex()); + ((com.mysql.cj.jdbc.DatabaseMetaData) this.dbmd).setMetadataEncoding(this.session.getServerSession().getCharsetSettings().getMetadataEncoding()); + ((com.mysql.cj.jdbc.DatabaseMetaData) this.dbmd) + .setMetadataCollationIndex(this.session.getServerSession().getCharsetSettings().getMetadataCollationIndex()); // // Server can do this more efficiently for us diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/ParameterBindingsImpl.java b/src/main/user-impl/java/com/mysql/cj/jdbc/ParameterBindingsImpl.java index 489c8a625..b601339c0 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/ParameterBindingsImpl.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/ParameterBindingsImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Oracle and/or its affiliates. + * Copyright (c) 2019, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -107,7 +107,7 @@ public class ParameterBindingsImpl implements ParameterBindings { break; default: try { - charsetIndex = CharsetMapping.getCollationIndexForJavaEncoding( + charsetIndex = session.getServerSession().getCharsetSettings().getCollationIndexForJavaEncoding( this.propertySet.getStringProperty(PropertyKey.characterEncoding).getValue(), session.getServerSession().getServerVersion()); } catch (RuntimeException ex) { throw SQLError.createSQLException(ex.toString(), MysqlErrorNumbers.SQL_STATE_ILLEGAL_ARGUMENT, ex, null); diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/StatementImpl.java b/src/main/user-impl/java/com/mysql/cj/jdbc/StatementImpl.java index db17be8da..06e3a1a5b 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/StatementImpl.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/StatementImpl.java @@ -43,7 +43,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.mysql.cj.CancelQueryTask; -import com.mysql.cj.CharsetMapping; import com.mysql.cj.Messages; import com.mysql.cj.MysqlType; import com.mysql.cj.NativeSession; @@ -981,7 +980,7 @@ private long[] executeBatchUsingMultiQueries(boolean multiQueriesEnabled, int nb String connectionEncoding = locallyScopedConn.getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(); int numberOfBytesPerChar = StringUtils.startsWithIgnoreCase(connectionEncoding, "utf") ? 3 - : (CharsetMapping.isMultibyteCharset(connectionEncoding) ? 2 : 1); + : (this.session.getServerSession().getCharsetSettings().isMultibyteCharset(connectionEncoding) ? 2 : 1); int escapeAdjust = 1; @@ -1224,8 +1223,8 @@ protected void doPingInstead() throws SQLException { protected ResultSetInternalMethods generatePingResultSet() throws SQLException { synchronized (checkClosed().getConnectionMutex()) { - String encoding = this.session.getServerSession().getCharacterSetMetadata(); - int collationIndex = this.session.getServerSession().getMetadataCollationIndex(); + String encoding = this.session.getServerSession().getCharsetSettings().getMetadataEncoding(); + int collationIndex = this.session.getServerSession().getCharsetSettings().getMetadataCollationIndex(); Field[] fields = { new Field(null, "1", collationIndex, encoding, MysqlType.BIGINT, 1) }; ArrayList rows = new ArrayList<>(); byte[] colVal = new byte[] { (byte) '1' }; @@ -1387,8 +1386,8 @@ public java.sql.ResultSet getGeneratedKeys() throws SQLException { return this.generatedKeysResults = getGeneratedKeysInternal(); } - String encoding = this.session.getServerSession().getCharacterSetMetadata(); - int collationIndex = this.session.getServerSession().getMetadataCollationIndex(); + String encoding = this.session.getServerSession().getCharsetSettings().getMetadataEncoding(); + int collationIndex = this.session.getServerSession().getCharsetSettings().getMetadataCollationIndex(); Field[] fields = new Field[1]; fields[0] = new Field("", "GENERATED_KEY", collationIndex, encoding, MysqlType.BIGINT_UNSIGNED, 20); @@ -1411,8 +1410,8 @@ protected ResultSetInternalMethods getGeneratedKeysInternal() throws SQLExceptio protected ResultSetInternalMethods getGeneratedKeysInternal(long numKeys) throws SQLException { synchronized (checkClosed().getConnectionMutex()) { - String encoding = this.session.getServerSession().getCharacterSetMetadata(); - int collationIndex = this.session.getServerSession().getMetadataCollationIndex(); + String encoding = this.session.getServerSession().getCharsetSettings().getMetadataEncoding(); + int collationIndex = this.session.getServerSession().getCharsetSettings().getMetadataCollationIndex(); Field[] fields = new Field[1]; fields[0] = new Field("", "GENERATED_KEY", collationIndex, encoding, MysqlType.BIGINT_UNSIGNED, 20); diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetImpl.java b/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetImpl.java index 10606d9cd..b3e05ca7d 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetImpl.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetImpl.java @@ -875,7 +875,7 @@ public String getString(int columnIndex) throws SQLException { String stringVal = this.thisRow.getValue(columnIndex - 1, vf); if (this.padCharsWithSpace && stringVal != null && f.getMysqlTypeId() == MysqlType.FIELD_TYPE_STRING) { - int maxBytesPerChar = this.session.getServerSession().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); + int maxBytesPerChar = this.session.getServerSession().getCharsetSettings().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); int fieldLength = (int) f.getLength() /* safe, bytes in a CHAR <= 1024 */ / maxBytesPerChar; /* safe, this will never be 0 */ return StringUtils.padString(stringVal, fieldLength); } diff --git a/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetMetaData.java b/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetMetaData.java index b89172e32..155e98d2a 100644 --- a/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetMetaData.java +++ b/src/main/user-impl/java/com/mysql/cj/jdbc/result/ResultSetMetaData.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2020, Oracle and/or its affiliates. + * Copyright (c) 2002, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -31,10 +31,8 @@ import java.sql.SQLException; -import com.mysql.cj.CharsetMapping; import com.mysql.cj.Messages; import com.mysql.cj.MysqlType; -import com.mysql.cj.NativeSession; import com.mysql.cj.Session; import com.mysql.cj.conf.PropertyDefinitions.DatabaseTerm; import com.mysql.cj.conf.PropertyKey; @@ -127,18 +125,7 @@ public String getColumnCharacterEncoding(int column) throws SQLException { * if an invalid column index is given. */ public String getColumnCharacterSet(int column) throws SQLException { - int index = getField(column).getCollationIndex(); - - String charsetName = null; - - if (((NativeSession) this.session).getProtocol().getServerSession().indexToCustomMysqlCharset != null) { - charsetName = ((NativeSession) this.session).getProtocol().getServerSession().indexToCustomMysqlCharset.get(index); - } - if (charsetName == null) { - charsetName = CharsetMapping.getMysqlCharsetNameForCollationIndex(index); - } - - return charsetName; + return this.session.getServerSession().getCharsetSettings().getMysqlCharsetNameForCollationIndex(getField(column).getCollationIndex()); } @Override @@ -169,7 +156,7 @@ public int getColumnDisplaySize(int column) throws SQLException { int lengthInBytes = clampedGetLength(f); - return lengthInBytes / this.session.getServerSession().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); + return lengthInBytes / this.session.getServerSession().getCharsetSettings().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); } @Override @@ -241,7 +228,7 @@ public int getPrecision(int column) throws SQLException { default: return f.getMysqlType().isDecimal() ? clampedGetLength(f) - : clampedGetLength(f) / this.session.getServerSession().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); + : clampedGetLength(f) / this.session.getServerSession().getCharsetSettings().getMaxBytesPerChar(f.getCollationIndex(), f.getEncoding()); } } @@ -315,7 +302,7 @@ public boolean isCaseSensitive(int column) throws java.sql.SQLException { case JSON: case ENUM: case SET: - String collationName = CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[field.getCollationIndex()]; + String collationName = this.session.getServerSession().getCharsetSettings().getCollationNameForCollationIndex(field.getCollationIndex()); return ((collationName != null) && !collationName.endsWith("_ci")); default: diff --git a/src/main/user-impl/java/com/mysql/cj/xdevapi/ColumnImpl.java b/src/main/user-impl/java/com/mysql/cj/xdevapi/ColumnImpl.java index 902017707..8cccf54c6 100644 --- a/src/main/user-impl/java/com/mysql/cj/xdevapi/ColumnImpl.java +++ b/src/main/user-impl/java/com/mysql/cj/xdevapi/ColumnImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2020, Oracle and/or its affiliates. + * Copyright (c) 2016, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -151,11 +151,11 @@ public boolean isNumberSigned() { } public String getCollationName() { - return CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[this.field.getCollationIndex()]; + return CharsetMapping.getStaticCollationNameForCollationIndex(this.field.getCollationIndex()); // TODO use CharsetSettings method } public String getCharacterSetName() { - return CharsetMapping.getMysqlCharsetNameForCollationIndex(this.field.getCollationIndex()); + return CharsetMapping.getStaticMysqlCharsetNameForCollationIndex(this.field.getCollationIndex()); // TODO use CharsetSettings method } public boolean isPadded() { diff --git a/src/test/java/com/mysql/cj/CharsetMappingWrapper.java b/src/test/java/com/mysql/cj/CharsetMappingWrapper.java new file mode 100644 index 000000000..074ab2c2a --- /dev/null +++ b/src/test/java/com/mysql/cj/CharsetMappingWrapper.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2021, Oracle and/or its affiliates. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License, version 2.0, as published by the + * Free Software Foundation. + * + * This program is also distributed with certain software (including but not + * limited to OpenSSL) that is licensed under separate terms, as designated in a + * particular file or component or in included license documentation. The + * authors of MySQL hereby grant you an additional permission to link the + * program and your derivative works with the separately licensed software that + * they have included with MySQL. + * + * Without limiting anything contained in the foregoing, this file, which is + * part of MySQL Connector/J, is also subject to the Universal FOSS Exception, + * version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, + * for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +package com.mysql.cj; + +/** + * Exposes protected {@link CharsetMapping} methods for use in tests. + */ +public class CharsetMappingWrapper { + + public static boolean isStaticMultibyteCharset(String javaEncodingName) { + return CharsetMapping.isStaticMultibyteCharset(javaEncodingName); + } + + public static int getStaticCollationIndexForJavaEncoding(String javaEncoding, ServerVersion version) { + return CharsetMapping.getStaticCollationIndexForJavaEncoding(javaEncoding, version); + } + + public static String getStaticCollationNameForCollationIndex(Integer collationIndex) { + return CharsetMapping.getStaticCollationNameForCollationIndex(collationIndex); + } + + public static String getStaticJavaEncodingForCollationIndex(Integer collationIndex) { + return CharsetMapping.getStaticJavaEncodingForCollationIndex(collationIndex); + } + + public static String getStaticJavaEncodingForMysqlCharset(String mysqlCharsetName) { + return CharsetMapping.getStaticJavaEncodingForMysqlCharset(mysqlCharsetName); + } + + public static String getStaticMysqlCharsetByName(String mysqlCharsetName) { + Object cs = CharsetMapping.getStaticMysqlCharsetByName(mysqlCharsetName); + return cs == null ? null : cs.toString(); + } + + public static String getStaticMysqlCharsetForJavaEncoding(String javaEncoding, ServerVersion version) { + return CharsetMapping.getStaticMysqlCharsetForJavaEncoding(javaEncoding, version); + } + + public static String getStaticMysqlCharsetNameForCollationIndex(Integer collationIndex) { + return CharsetMapping.getStaticMysqlCharsetNameForCollationIndex(collationIndex); + } + +} diff --git a/src/test/java/com/mysql/cj/MessagesTest.java b/src/test/java/com/mysql/cj/MessagesTest.java index 183fb21fe..745aca9cb 100644 --- a/src/test/java/com/mysql/cj/MessagesTest.java +++ b/src/test/java/com/mysql/cj/MessagesTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -104,9 +104,10 @@ public void testLocalizedErrorMessages() throws Exception { assertEquals("Illegal starting position for search, '10'", Messages.getString("Clob.8", new Object[] { 10 })); - assertEquals("Java does not support the MySQL character encoding 'Test'.", Messages.getString("Connection.5", new Object[] { "Test" })); + assertEquals("Unknown Java encoding for the character set with index '1234'. Use the 'customCharsetMapping' property to force it.", + Messages.getString("Connection.5", new Object[] { "1234" })); assertEquals( - "Unknown initial character set index 'Test' received from server. Initial client character set can be forced via the 'characterEncoding' property.", + "Unknown character set index 'Test' received from server. The appropriate client character set can be forced via the 'characterEncoding' property.", Messages.getString("Connection.6", new Object[] { "Test" })); assertEquals("Can't map Test given for characterSetResults to a supported MySQL encoding.", Messages.getString("Connection.7", new Object[] { "Test" })); diff --git a/src/test/java/testsuite/regression/CharsetRegressionTest.java b/src/test/java/testsuite/regression/CharsetRegressionTest.java index d9ea76423..d426d1d8e 100644 --- a/src/test/java/testsuite/regression/CharsetRegressionTest.java +++ b/src/test/java/testsuite/regression/CharsetRegressionTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2020, Oracle and/or its affiliates. + * Copyright (c) 2014, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -32,6 +32,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.sql.Connection; import java.sql.ResultSet; @@ -43,12 +45,15 @@ import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.MysqlConnection; import com.mysql.cj.Query; import com.mysql.cj.conf.PropertyKey; -import com.mysql.cj.exceptions.ExceptionFactory; +import com.mysql.cj.interceptors.QueryInterceptor; +import com.mysql.cj.log.Log; +import com.mysql.cj.protocol.Message; import com.mysql.cj.protocol.Resultset; +import com.mysql.cj.util.StringUtils; import testsuite.BaseQueryInterceptor; import testsuite.BaseTestCase; @@ -68,29 +73,37 @@ public void testBug73663() throws Exception { this.rs.next(); String collation = this.rs.getString(2); - if (collation != null && collation.startsWith("utf8mb4") - && "utf8mb4".equals(((MysqlConnection) this.conn).getSession().getServerSession().getServerVariable("character_set_server"))) { - Properties p = new Properties(); - p.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); - p.setProperty(PropertyKey.queryInterceptors.getKeyName(), Bug73663QueryInterceptor.class.getName()); + assumeTrue( + collation != null && collation.startsWith("utf8mb4") + && "utf8mb4".equals(((MysqlConnection) this.conn).getSession().getServerSession().getServerVariable("character_set_server")), + "This test requires server configured with character_set_server=utf8mb4 and collation-server set to one of utf8mb4 collations."); - getConnectionWithProps(p); - // exception will be thrown from the statement interceptor if any "SET NAMES utf8" statement is issued instead of "SET NAMES utf8mb4" - } else { - System.out.println( - "testBug73663 was skipped: This test is only run when character_set_server=utf8mb4 and collation-server set to one of utf8mb4 collations."); - } + Properties p = new Properties(); + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); + p.setProperty(PropertyKey.queryInterceptors.getKeyName(), Bug73663QueryInterceptor.class.getName()); + + getConnectionWithProps(p); + // failure will be thrown from the statement interceptor if any "SET NAMES utf8" statement is issued instead of "SET NAMES utf8mb4" } /** * Statement interceptor used to implement preceding test. */ public static class Bug73663QueryInterceptor extends BaseQueryInterceptor { + @Override + public M preProcess(M queryPacket) { + String sql = StringUtils.toString(queryPacket.getByteBuffer(), 1, (queryPacket.getPosition() - 1)); + if (sql.contains("SET NAMES utf8") && !sql.contains("utf8mb4")) { + fail("Character set statement issued: " + sql); + } + return null; + } + @Override public T preProcess(Supplier str, Query interceptedQuery) { String sql = str.get(); if (sql.contains("SET NAMES utf8") && !sql.contains("utf8mb4")) { - throw ExceptionFactory.createException("Character set statement issued: " + sql); + fail("Character set statement issued: " + sql); } return null; } @@ -152,7 +165,7 @@ public Void call() throws Exception { public void testBug25504578() throws Exception { Properties p = new Properties(); - String cjCharset = CharsetMapping.getJavaEncodingForMysqlCharset("latin7"); + String cjCharset = CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset("latin7"); p.setProperty(PropertyKey.characterEncoding.getKeyName(), cjCharset); getConnectionWithProps(p); @@ -176,7 +189,7 @@ public void testBug81196() throws Exception { Properties p = new Properties(); /* With a single-byte encoding */ - p.setProperty(PropertyKey.characterEncoding.getKeyName(), CharsetMapping.getJavaEncodingForMysqlCharset("latin1")); + p.setProperty(PropertyKey.characterEncoding.getKeyName(), CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset("latin1")); Connection conn1 = getConnectionWithProps(p); Statement st1 = conn1.createStatement(); st1.executeUpdate("INSERT INTO testBug81196(name) VALUES ('" + fourBytesValue + "')"); @@ -220,4 +233,213 @@ public Void call() throws Exception { }); } } + + /** + * Tests fix for Bug#100606 (31818423), UNECESARY CALL TO "SET NAMES 'UTF8' COLLATE 'UTF8_GENERAL_CI'". + * + * @throws Exception + */ + @Test + public void testBug100606() throws Exception { + this.rs = this.stmt.executeQuery("show variables like 'collation_server'"); + this.rs.next(); + String collation = this.rs.getString(2); + + assumeTrue( + collation != null && collation.startsWith("utf8_general_ci") + && ((MysqlConnection) this.conn).getSession().getServerSession().getServerVariable("character_set_server").startsWith("utf8"), + "This test requires server configured with character_set_server=utf8 and collation-server=utf8_general_ci."); + + String fallbackCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; + + Properties p = new Properties(); + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); + p.setProperty(PropertyKey.queryInterceptors.getKeyName(), TestSetNamesQueryInterceptor.class.getName()); + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + p.setProperty(PropertyKey.connectionCollation.getKeyName(), "utf8_general_ci"); + checkCollationConnection(p, "SET NAMES", false, "utf8_general_ci"); + } + + /** + * Tests fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED WITH CUSTOM COLLATION. + * + * This test requires a special server configuration with: + *
    + *
  • character-set-server = custom + *
  • collation-server = custom_bin + *
+ * where 'custom_bin' is not a primary collation for 'custom' character set and has an index == 1024 on MySQL 8.0+ or index == 253 for older servers. + * + * @throws Exception + */ + @Test + public void testBug25554464_1() throws Exception { + this.rs = this.stmt.executeQuery("show variables like 'collation_server'"); + this.rs.next(); + String collation = this.rs.getString(2); + this.rs = this.stmt.executeQuery("select ID from INFORMATION_SCHEMA.COLLATIONS where COLLATION_NAME='custom_bin'"); + + assumeTrue( + collation != null && collation.startsWith("custom_bin") && this.rs.next() && this.rs.getInt(1) == (versionMeetsMinimum(8, 0, 1) ? 1024 : 253), + "This test requires server configured with custom character set and custom_bin collation."); + + String fallbackCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; + + Properties p = new Properties(); + p.setProperty(PropertyKey.queryInterceptors.getKeyName(), TestSetNamesQueryInterceptor.class.getName()); + + // With no specific properties c/J is using a predefined fallback collation 'utf8mb4_0900_ai_ci' or 'utf8mb4_general_ci' depending on server version. + // Post-handshake does not issue a SET NAMES because the predefined collation should still be used. + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + // 'detectCustomCollations' itself doesn't change the expected collation, the predefined one should still be used. + p.setProperty(PropertyKey.detectCustomCollations.getKeyName(), "true"); + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + // The predefined collation should still be used + p.setProperty(PropertyKey.customCharsetMapping.getKeyName(), "custom:Cp1252"); + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + // Handshake collation is still 'utf8mb4_0900_ai_ci' because 'custom_bin' index > 255, then c/J sends the SET NAMES for the requested collation in a post-handshake. + p.setProperty(PropertyKey.connectionCollation.getKeyName(), "custom_bin"); + checkCollationConnection(p, "SET NAMES custom COLLATE custom_bin", true, "custom_bin"); + + p.setProperty(PropertyKey.connectionCollation.getKeyName(), "custom_general_ci"); + checkCollationConnection(p, "SET NAMES custom COLLATE custom_general_ci", true, "custom_general_ci"); + + // Handshake collation is still 'utf8mb4_0900_ai_ci' because 'Cp1252' is mapped to 'custom' charset via the 'customCharsetMapping' property. + // C/J sends the SET NAMES for the requested collation in a post-handshake. + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "Cp1252"); + p.remove(PropertyKey.connectionCollation.getKeyName()); + checkCollationConnection(p, "SET NAMES custom", true, "custom_general_ci"); + } + + /** + * Tests fix for Bug#25554464, CONNECT FAILS WITH NPE WHEN THE SERVER STARTED WITH CUSTOM COLLATION. + * + * This test requires a special server configuration with: + *
    + *
  • character-set-server = custom + *
  • collation-server = custom_general_ci + *
+ * where 'custom_general_ci' is a primary collation for 'custom' character set and has an index == 1025 on MySQL 8.0+ or index == 254 for older servers. + * + * @throws Exception + */ + @Test + public void testBug25554464_2() throws Exception { + this.rs = this.stmt.executeQuery("show variables like 'collation_server'"); + this.rs.next(); + String collation = this.rs.getString(2); + + this.rs = this.stmt.executeQuery("select ID from INFORMATION_SCHEMA.COLLATIONS where COLLATION_NAME='custom_general_ci'"); + + assumeTrue( + collation != null && collation.startsWith("custom_general_ci") && this.rs.next() + && this.rs.getInt(1) == (versionMeetsMinimum(8, 0, 1) ? 1025 : 254), + "This test requires server configured with custom character set and custom_general_ci collation."); + + String fallbackCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; + + Properties p = new Properties(); + p.setProperty(PropertyKey.queryInterceptors.getKeyName(), TestSetNamesQueryInterceptor.class.getName()); + + // With no specific properties c/J is using a predefined fallback collation 'utf8mb4_0900_ai_ci' or 'utf8mb4_general_ci' depending on server version. + // Post-handshake does not issue a SET NAMES because the predefined collation should still be used. + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + // The predefined one should still be used. + p.setProperty(PropertyKey.detectCustomCollations.getKeyName(), "true"); + p.setProperty(PropertyKey.customCharsetMapping.getKeyName(), "custom:Cp1252"); + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + // Sets the predefined collation via the handshake response, but, in a post-handshake, issues the SET NAMES setting the required 'connectionCollation'. + p.setProperty(PropertyKey.connectionCollation.getKeyName(), "custom_general_ci"); + checkCollationConnection(p, "SET NAMES custom COLLATE custom_general_ci", true, "custom_general_ci"); + + // Sets the predefined collation via the handshake response, but, in a post-handshake, issues the SET NAMES setting the required 'characterEncoding'. + // The chosen collation then is 'custom_general_ci' because it's a primary one for 'custom' character set. + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "Cp1252"); + p.remove(PropertyKey.connectionCollation.getKeyName()); + checkCollationConnection(p, "SET NAMES custom", true, "custom_general_ci"); + } + + public static class TestSetNamesQueryInterceptor extends BaseQueryInterceptor { + public static String query = ""; + public static boolean usedSetNames = false; + + @Override + public QueryInterceptor init(MysqlConnection conn, Properties props, Log log) { + usedSetNames = false; + return super.init(conn, props, log); + } + + @Override + public M preProcess(M queryPacket) { + String sql = StringUtils.toString(queryPacket.getByteBuffer(), 1, (queryPacket.getPosition() - 1)); + if (sql.contains(query)) { + usedSetNames = true; + } + return null; + } + + @Override + public T preProcess(Supplier str, Query interceptedQuery) { + String sql = str.get(); + if (sql.contains(query)) { + usedSetNames = true; + } + return null; + } + } + + @Test + public void testPasswordCharacterEncoding() throws Exception { + this.rs = this.stmt.executeQuery("show variables like 'collation_server'"); + this.rs.next(); + String collation = this.rs.getString(2); + assumeTrue(collation != null && collation.startsWith("latin1"), "This test requires a server configured with latin1 character set."); + + Properties p = new Properties(); + p.setProperty(PropertyKey.queryInterceptors.getKeyName(), TestSetNamesQueryInterceptor.class.getName()); + + String requestedCollation = "latin1_swedish_ci"; + String fallbackCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; + + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + p.setProperty(PropertyKey.passwordCharacterEncoding.getKeyName(), "UTF-8"); + checkCollationConnection(p, "SET NAMES", false, fallbackCollation); + + p.setProperty(PropertyKey.connectionCollation.getKeyName(), requestedCollation); + checkCollationConnection(p, "SET NAMES latin1 COLLATE " + requestedCollation, true, requestedCollation); + + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "Cp1252"); + p.remove(PropertyKey.connectionCollation.getKeyName()); + checkCollationConnection(p, "SET NAMES latin1", true, requestedCollation); + + requestedCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; + + p.setProperty(PropertyKey.connectionCollation.getKeyName(), requestedCollation); + checkCollationConnection(p, "SET NAMES", false, requestedCollation); + + p.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); + p.remove(PropertyKey.connectionCollation.getKeyName()); + checkCollationConnection(p, "SET NAMES", false, requestedCollation); + } + + private void checkCollationConnection(Properties props, String query, boolean expectQueryIsIssued, String expectedCollation) throws Exception { + TestSetNamesQueryInterceptor.query = query; + Connection c = getConnectionWithProps(props); + if (expectQueryIsIssued) { + assertTrue(TestSetNamesQueryInterceptor.usedSetNames); + } else { + assertFalse(TestSetNamesQueryInterceptor.usedSetNames); + } + this.rs = c.createStatement().executeQuery("show variables like 'collation_connection'"); + this.rs.next(); + assertEquals(expectedCollation, this.rs.getString(2)); + c.close(); + } } diff --git a/src/test/java/testsuite/regression/ConnectionRegressionTest.java b/src/test/java/testsuite/regression/ConnectionRegressionTest.java index 337724cee..271f8f258 100644 --- a/src/test/java/testsuite/regression/ConnectionRegressionTest.java +++ b/src/test/java/testsuite/regression/ConnectionRegressionTest.java @@ -122,7 +122,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; +import com.mysql.cj.CharsetSettings; import com.mysql.cj.Constants; import com.mysql.cj.Messages; import com.mysql.cj.MysqlConnection; @@ -4207,7 +4208,7 @@ public void testBug37931() throws Exception { _conn = getConnectionWithProps(props); assertTrue(false, "This point should not be reached."); } catch (Exception e) { - assertEquals("Can't map ISO88591 given for characterSetResults to a supported MySQL encoding.", e.getMessage()); + assertEquals("Unsupported character encoding 'ISO88591'", e.getMessage()); } finally { if (_conn != null) { _conn.close(); @@ -4554,7 +4555,7 @@ public void testBug11237() throws Exception { Statement stmt1 = conn1.createStatement(); int updateCount = stmt1.executeUpdate("LOAD DATA LOCAL INFILE '" + fileNameBuf.toString() + "' INTO TABLE testBug11237 CHARACTER SET " - + CharsetMapping.getMysqlCharsetForJavaEncoding( + + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding( ((MysqlConnection) this.conn).getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), ((JdbcConnection) conn1).getServerVersion())); @@ -5367,6 +5368,7 @@ public void testBug17251955() throws Exception { } catch (SQLException e) { assertFalse(e.getCause() instanceof java.lang.ArrayIndexOutOfBoundsException, "e.getCause() instanceof java.lang.ArrayIndexOutOfBoundsException"); + props.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); props.setProperty(PropertyKey.USER.getKeyName(), "\u30C6\u30B9\u30C8\u30C6\u30B9\u30C8"); c2 = DriverManager.getConnection(url + "/\u30C6\u30B9\u30C8\u30C6\u30B9\u30C8", props); this.rs = c2.createStatement().executeQuery("select 1"); @@ -5819,7 +5821,7 @@ public void testBug71038() throws Exception { JdbcConnection c = (JdbcConnection) getConnectionWithProps(p); Bug71038QueryInterceptor si = (Bug71038QueryInterceptor) c.getQueryInterceptorsInstances().get(0); - assertTrue(si.cnt == 0, "SHOW COLLATION was issued when detectCustomCollations=false"); + assertTrue(si.cnt == 0, "SELECT from INFORMATION_SCHEMA.COLLATIONS was issued when detectCustomCollations=false"); c.close(); p.setProperty(PropertyKey.detectCustomCollations.getKeyName(), "true"); @@ -5827,7 +5829,7 @@ public void testBug71038() throws Exception { c = (JdbcConnection) getConnectionWithProps(p); si = (Bug71038QueryInterceptor) c.getQueryInterceptorsInstances().get(0); - assertTrue(si.cnt > 0, "SHOW COLLATION wasn't issued when detectCustomCollations=true"); + assertTrue(si.cnt > 0, "SELECT from INFORMATION_SCHEMA.COLLATIONS wasn't issued when detectCustomCollations=true"); c.close(); } @@ -5840,7 +5842,7 @@ public static class Bug71038QueryInterceptor extends BaseQueryInterceptor { @Override public M preProcess(M queryPacket) { String sql = StringUtils.toString(queryPacket.getByteBuffer(), 1, (queryPacket.getPosition() - 1)); - if (sql.contains("SHOW COLLATION")) { + if (sql.contains("from INFORMATION_SCHEMA.COLLATIONS")) { this.cnt++; } return null; @@ -6077,7 +6079,8 @@ public static class Bug72712QueryInterceptor extends BaseQueryInterceptor { @Override public T preProcess(Supplier str, Query interceptedQuery) { String sql = str.get(); - if (sql.contains("SET NAMES") || sql.contains("character_set_results") && !(sql.contains("SHOW VARIABLES") || sql.contains("SELECT @@"))) { + if (sql.contains("SET NAMES") + || sql.contains(CharsetSettings.CHARACTER_SET_RESULTS) && !(sql.contains("SHOW VARIABLES") || sql.contains("SELECT @@"))) { throw ExceptionFactory.createException("Wrongt statement issued: " + sql); } return null; @@ -6969,7 +6972,8 @@ public void testBug75592() throws Exception { Map serverVariables = new HashMap<>(); this.rs = con.createStatement().executeQuery("SHOW VARIABLES"); while (this.rs.next()) { - serverVariables.put(this.rs.getString(1), this.rs.getString(2)); + String val = this.rs.getString(2); + serverVariables.put(this.rs.getString(1), "utf8mb3".equals(val) ? "utf8" : val); } // fix the renaming of "tx_isolation" to "transaction_isolation" that is made in NativeSession.loadServerVariables(). @@ -6980,15 +6984,17 @@ public void testBug75592() throws Exception { // check values from "select @@var..." assertEquals(serverVariables.get("auto_increment_increment"), session.getServerSession().getServerVariable("auto_increment_increment")); - assertEquals(serverVariables.get("character_set_client"), session.getServerSession().getServerVariable("character_set_client")); - assertEquals(serverVariables.get("character_set_connection"), session.getServerSession().getServerVariable("character_set_connection")); + assertEquals(serverVariables.get(CharsetSettings.CHARACTER_SET_CLIENT), + session.getServerSession().getServerVariable(CharsetSettings.CHARACTER_SET_CLIENT)); + assertEquals(serverVariables.get(CharsetSettings.CHARACTER_SET_CONNECTION), + session.getServerSession().getServerVariable(CharsetSettings.CHARACTER_SET_CONNECTION)); // we override character_set_results sometimes when configuring client charsets, thus need to check against actual value - if (session.getServerSession().getServerVariable(ServerSession.LOCAL_CHARACTER_SET_RESULTS) == null) { - assertEquals("", serverVariables.get("character_set_results")); + if (session.getServerSession().getServerVariable(CharsetSettings.CHARACTER_SET_RESULTS) == null) { + assertEquals("", serverVariables.get(CharsetSettings.CHARACTER_SET_RESULTS)); } else { - assertEquals(serverVariables.get("character_set_results"), - session.getServerSession().getServerVariable(ServerSession.LOCAL_CHARACTER_SET_RESULTS)); + assertEquals(serverVariables.get(CharsetSettings.CHARACTER_SET_RESULTS), + session.getServerSession().getServerVariable(CharsetSettings.CHARACTER_SET_RESULTS)); } assertEquals(serverVariables.get("character_set_server"), session.getServerSession().getServerVariable("character_set_server")); @@ -7161,13 +7167,13 @@ public void testBug20825727() throws Exception { String testStep = ""; try { testStep = "create user"; - testBug20825727CreateUser(testDbUrl, "testBug20825727", simplePwd, encoding, pluginName, pwdHashingMethod); + testBug20825727CreateUser(testDbUrl, "testBug20825727", simplePwd, pluginName, pwdHashingMethod); testStep = "login with simple password"; testBug20825727TestLogin(testDbUrl, testConn.getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), sslEnabled, rsaEnabled, "testBug20825727", simplePwd, encoding, pluginName); testStep = "change password"; - testBug20825727ChangePassword(testDbUrl, "testBug20825727", complexPwd, encoding, pluginName, pwdHashingMethod); + testBug20825727ChangePassword(testDbUrl, "testBug20825727", complexPwd, pluginName, pwdHashingMethod); testStep = "login with complex password"; testBug20825727TestLogin(testDbUrl, testConn.getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), sslEnabled, rsaEnabled, "testBug20825727", complexPwd, encoding, pluginName); @@ -7197,15 +7203,12 @@ public void testBug20825727() throws Exception { } } - private void testBug20825727CreateUser(String testDbUrl, String user, String password, String encoding, String pluginName, int pwdHashingMethod) - throws SQLException { + private void testBug20825727CreateUser(String testDbUrl, String user, String password, String pluginName, int pwdHashingMethod) throws SQLException { JdbcConnection testConn = null; try { Properties props = new Properties(); props.setProperty(PropertyKey.allowPublicKeyRetrieval.getKeyName(), "true"); - if (encoding.length() > 0) { - props.setProperty(PropertyKey.characterEncoding.getKeyName(), encoding); - } + props.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); testConn = (JdbcConnection) getConnectionWithProps(testDbUrl, props); Statement testStmt = testConn.createStatement(); @@ -7234,15 +7237,12 @@ private void testBug20825727CreateUser(String testDbUrl, String user, String pas } } - private void testBug20825727ChangePassword(String testDbUrl, String user, String password, String encoding, String pluginName, int pwdHashingMethod) - throws SQLException { + private void testBug20825727ChangePassword(String testDbUrl, String user, String password, String pluginName, int pwdHashingMethod) throws SQLException { JdbcConnection testConn = null; try { Properties props = new Properties(); props.setProperty(PropertyKey.allowPublicKeyRetrieval.getKeyName(), "true"); - if (encoding.length() > 0) { - props.setProperty(PropertyKey.characterEncoding.getKeyName(), encoding); - } + props.setProperty(PropertyKey.characterEncoding.getKeyName(), "UTF-8"); testConn = (JdbcConnection) getConnectionWithProps(testDbUrl, props); Statement testStmt = testConn.createStatement(); @@ -7348,13 +7348,8 @@ private void testBug20825727TestLogin(final String testDbUrl, String defaultServ boolean testShouldPass = true; if (pwdIsComplex) { - // if no encoding is specifically defined then our default password encoding ('UTF-8') and server's encoding must coincide - testShouldPass = encoding.length() > 0 || defaultServerEncoding.equalsIgnoreCase("UTF-8"); - - if (!testBaseConn.getSession().versionMeetsMinimum(5, 7, 6) && pluginName.equals("cleartext_plugin_server")) { - // 'cleartext_plugin_server' from servers below version 5.7.6 requires UTF-8 encoding - testShouldPass = encoding.equals("UTF-8") || (encoding.length() == 0 && defaultServerEncoding.equals("UTF-8")); - } + // if no encoding is specifically defined then our default password encoding is set to server's encoding + testShouldPass = encoding.equals("UTF-8") || (encoding.length() == 0 && defaultServerEncoding.equals("UTF-8")); } System.out.printf("%-25s : %-25s : %s : %-25s : %-18s : %-18s [%s]%n", testCaseMsg, pluginName, pwdIsComplex ? "cplx" : "smpl", encProp, @@ -10910,13 +10905,20 @@ private void testBug89948Check(String testCase, int expectedCount, int idOffset) public void testBug91317() throws Exception { Map defaultCollations = new HashMap<>(); + Properties p = new Properties(); + p.setProperty(PropertyKey.detectCustomCollations.getKeyName(), "true"); + p.setProperty(PropertyKey.customCharsetMapping.getKeyName(), "custom:Cp1252"); + Connection c = getConnectionWithProps(p); + // Compare server-side and client-side collation defaults. this.rs = this.stmt.executeQuery("SELECT COLLATION_NAME, CHARACTER_SET_NAME, ID FROM INFORMATION_SCHEMA.COLLATIONS WHERE IS_DEFAULT = 'Yes'"); while (this.rs.next()) { String collationName = this.rs.getString(1); String charsetName = this.rs.getString(2); int collationId = this.rs.getInt(3); - int mappedCollationId = CharsetMapping.CHARSET_NAME_TO_COLLATION_INDEX.get(charsetName); + + int mappedCollationId = ((MysqlConnection) c).getSession().getServerSession().getCharsetSettings() + .getCollationIndexForMysqlCharsetName(charsetName); defaultCollations.put(charsetName, collationName); @@ -10926,7 +10928,8 @@ public void testBug91317() throws Exception { } assertEquals(collationId, mappedCollationId); - assertEquals(collationName, CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[mappedCollationId]); + assertEquals(collationName, + ((MysqlConnection) c).getSession().getServerSession().getCharsetSettings().getCollationNameForCollationIndex(mappedCollationId)); } ServerVersion sv = ((JdbcConnection) this.conn).getServerVersion(); @@ -10944,8 +10947,9 @@ public void testBug91317() throws Exception { continue; } - String javaEnc = CharsetMapping.getJavaEncodingForMysqlCharset(cs); - String charsetForJavaEnc = CharsetMapping.getMysqlCharsetForJavaEncoding(javaEnc, sv); + String javaEnc = ((MysqlConnection) c).getSession().getServerSession().getCharsetSettings().getJavaEncodingForMysqlCharset(cs); + System.out.println(cs + "->" + javaEnc); + String charsetForJavaEnc = ((MysqlConnection) c).getSession().getServerSession().getCharsetSettings().getMysqlCharsetForJavaEncoding(javaEnc, sv); String expectedCollation = defaultCollations.get(charsetForJavaEnc); if ("UTF-8".equalsIgnoreCase(javaEnc)) { @@ -10953,7 +10957,12 @@ public void testBug91317() throws Exception { expectedCollation = versionMeetsMinimum(8, 0, 1) ? "utf8mb4_0900_ai_ci" : "utf8mb4_general_ci"; } - Connection testConn = getConnectionWithProps("characterEncoding=" + javaEnc); + Properties p2 = new Properties(); + p2.setProperty(PropertyKey.detectCustomCollations.getKeyName(), "true"); + p2.setProperty(PropertyKey.customCharsetMapping.getKeyName(), "custom:Cp1252"); + p2.setProperty(PropertyKey.characterEncoding.getKeyName(), javaEnc); + + Connection testConn = getConnectionWithProps(p2); ResultSet testRs = testConn.createStatement().executeQuery("SHOW VARIABLES LIKE 'collation_connection'"); assertTrue(testRs.next()); assertEquals(expectedCollation, testRs.getString(2)); diff --git a/src/test/java/testsuite/regression/MetaDataRegressionTest.java b/src/test/java/testsuite/regression/MetaDataRegressionTest.java index 800bb66c1..eaf0a80f6 100644 --- a/src/test/java/testsuite/regression/MetaDataRegressionTest.java +++ b/src/test/java/testsuite/regression/MetaDataRegressionTest.java @@ -64,9 +64,10 @@ import org.junit.jupiter.api.Test; import org.opentest4j.AssertionFailedError; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.Constants; import com.mysql.cj.MysqlConnection; +import com.mysql.cj.NativeCharsetSettings; import com.mysql.cj.Query; import com.mysql.cj.conf.PropertyDefinitions.DatabaseTerm; import com.mysql.cj.conf.PropertyKey; @@ -1341,8 +1342,10 @@ private void testBug18554(int columnNameLength) throws Exception { } private void checkRsmdForBug13277(ResultSetMetaData rsmd) throws SQLException { - int i = ((com.mysql.cj.jdbc.ConnectionImpl) this.conn).getSession().getServerSession().getMaxBytesPerChar(CharsetMapping - .getJavaEncodingForMysqlCharset(((com.mysql.cj.jdbc.JdbcConnection) this.conn).getSession().getServerSession().getServerDefaultCharset())); + int i = ((com.mysql.cj.jdbc.ConnectionImpl) this.conn).getSession().getServerSession().getCharsetSettings() + .getMaxBytesPerChar(CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset( + ((NativeCharsetSettings) ((com.mysql.cj.jdbc.JdbcConnection) this.conn).getSession().getServerSession().getCharsetSettings()) + .getServerDefaultCharset())); if (i == 1) { // This is INT field but still processed in // ResultsetMetaData.getColumnDisplaySize @@ -1759,9 +1762,10 @@ private void compareResultSets(ResultSet expected, ResultSet actual) throws Exce } if ("CHAR_OCTET_LENGTH".equals(metadataExpected.getColumnName(i + 1))) { - if (((com.mysql.cj.jdbc.ConnectionImpl) this.conn).getSession().getServerSession() - .getMaxBytesPerChar(CharsetMapping.getJavaEncodingForMysqlCharset( - ((com.mysql.cj.jdbc.JdbcConnection) this.conn).getSession().getServerSession().getServerDefaultCharset())) > 1) { + if (((com.mysql.cj.jdbc.ConnectionImpl) this.conn).getSession().getServerSession().getCharsetSettings() + .getMaxBytesPerChar(CharsetMappingWrapper + .getStaticJavaEncodingForMysqlCharset(((NativeCharsetSettings) ((com.mysql.cj.jdbc.JdbcConnection) this.conn) + .getSession().getServerSession().getCharsetSettings()).getServerDefaultCharset())) > 1) { continue; // SHOW CREATE and CHAR_OCT *will* differ } } diff --git a/src/test/java/testsuite/regression/StatementRegressionTest.java b/src/test/java/testsuite/regression/StatementRegressionTest.java index 1e71070d7..6d252e47e 100644 --- a/src/test/java/testsuite/regression/StatementRegressionTest.java +++ b/src/test/java/testsuite/regression/StatementRegressionTest.java @@ -111,7 +111,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.ClientPreparedQuery; import com.mysql.cj.MysqlConnection; import com.mysql.cj.Query; @@ -2388,7 +2388,7 @@ public void testLoadData() throws Exception { assertThrows(SQLSyntaxErrorException.class, versionMeetsMinimum(8, 0, 19) ? "Loading local data is disabled;.*" : "The used command is not allowed with this MySQL version", () -> { this.stmt.executeUpdate("LOAD DATA LOCAL INFILE '" + fileName + "' INTO TABLE loadDataRegress CHARACTER SET " - + CharsetMapping.getMysqlCharsetForJavaEncoding( + + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding( ((MysqlConnection) this.conn).getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), this.serverVersion)); return null; @@ -2399,7 +2399,7 @@ public void testLoadData() throws Exception { Connection testConn = getConnectionWithProps(props); int updateCount = testConn.createStatement() .executeUpdate("LOAD DATA LOCAL INFILE '" + fileNameBuf.toString() + "' INTO TABLE loadDataRegress CHARACTER SET " - + CharsetMapping.getMysqlCharsetForJavaEncoding( + + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding( ((MysqlConnection) this.conn).getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), this.serverVersion)); assertTrue(updateCount == rowCount); diff --git a/src/test/java/testsuite/regression/SyntaxRegressionTest.java b/src/test/java/testsuite/regression/SyntaxRegressionTest.java index 2a2158820..eb3fd49b3 100644 --- a/src/test/java/testsuite/regression/SyntaxRegressionTest.java +++ b/src/test/java/testsuite/regression/SyntaxRegressionTest.java @@ -2045,9 +2045,7 @@ public void testInnodbTablespaceEncryption() throws Exception { } else { // Syntax can still be tested by with different outcome. System.out.println("Although not required it is recommended that the 'keyring_file' plugin is properly installed and configured to run this test."); - String err = versionMeetsMinimum(8, 0, 4) || versionMeetsMinimum(5, 7, 22) && !versionMeetsMinimum(8, 0, 0) - ? "Can't find master key from keyring, please check in the server log if a keyring.* is loaded and initialized successfully." - : "Can't find master key from keyring, please check keyring plugin is loaded."; + String err = "Can't find master key from keyring.*"; final Statement testStmt = this.conn.createStatement(); assertThrows(SQLException.class, err, new Callable() { diff --git a/src/test/java/testsuite/simple/CharsetTest.java b/src/test/java/testsuite/simple/CharsetTest.java index 3d5fbc05d..651db7821 100644 --- a/src/test/java/testsuite/simple/CharsetTest.java +++ b/src/test/java/testsuite/simple/CharsetTest.java @@ -30,7 +30,6 @@ package testsuite.simple; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -54,6 +53,7 @@ import org.junit.jupiter.api.Test; import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.conf.PropertyKey; import testsuite.BaseTestCase; @@ -244,15 +244,6 @@ public void testInsertCharStatement() throws Exception { } } - @Test - public void testStaticCharsetMappingConsistency() { - for (int i = 1; i < CharsetMapping.MAP_SIZE; i++) { - assertNotNull(CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i], - "Assertion failure: No mapping from charset index " + i + " to a mysql collation"); - assertNotNull(CharsetMapping.COLLATION_INDEX_TO_CHARSET[i], "Assertion failure: No mapping from charset index " + i + " to a Java character set"); - } - } - /** * Prints static mappings for analysis. * @@ -270,43 +261,48 @@ public void testCharsetMapping() throws Exception { java.nio.charset.Charset cs = availableCharsets.get(canonicalName); canonicalName = cs.name(); - int index = CharsetMapping.getCollationIndexForJavaEncoding(canonicalName, this.serverVersion); - String csname = CharsetMapping.getMysqlCharsetNameForCollationIndex(index); + int index = CharsetMappingWrapper.getStaticCollationIndexForJavaEncoding(canonicalName, this.serverVersion); + String csname = CharsetMappingWrapper.getStaticMysqlCharsetNameForCollationIndex(index); System.out.println((canonicalName + " ").substring(0, 26) + " (" + cs.canEncode() + ") --> " - + CharsetMapping.getJavaEncodingForCollationIndex(index) + " : " + index + " : " - + CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[index] + " : " + CharsetMapping.getMysqlCharsetNameForCollationIndex(index) + " : " - + CharsetMapping.CHARSET_NAME_TO_CHARSET.get(csname) + " : " + CharsetMapping.getJavaEncodingForMysqlCharset(csname) + " : " - + CharsetMapping.getMysqlCharsetForJavaEncoding(canonicalName, this.serverVersion) + " : " - + CharsetMapping.getCollationIndexForJavaEncoding(canonicalName, this.serverVersion) + " : " - + CharsetMapping.isMultibyteCharset(canonicalName)); + + CharsetMappingWrapper.getStaticJavaEncodingForCollationIndex(index) + " : " + index + " : " + + CharsetMappingWrapper.getStaticCollationNameForCollationIndex(index) + " : " + + CharsetMappingWrapper.getStaticMysqlCharsetNameForCollationIndex(index) + " : " + + CharsetMappingWrapper.getStaticMysqlCharsetByName(csname) + " : " + CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset(csname) + + " : " + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding(canonicalName, this.serverVersion) + " : " + + CharsetMappingWrapper.getStaticCollationIndexForJavaEncoding(canonicalName, this.serverVersion) + " : " + + CharsetMappingWrapper.isStaticMultibyteCharset(canonicalName)); Set s = cs.aliases(); Iterator j = s.iterator(); while (j.hasNext()) { String alias = j.next(); - index = CharsetMapping.getCollationIndexForJavaEncoding(alias, this.serverVersion); - csname = CharsetMapping.getMysqlCharsetNameForCollationIndex(index); + index = CharsetMappingWrapper.getStaticCollationIndexForJavaEncoding(alias, this.serverVersion); + csname = CharsetMappingWrapper.getStaticMysqlCharsetNameForCollationIndex(index); System.out.println(" " + (alias + " ").substring(0, 30) + " --> " - + CharsetMapping.getJavaEncodingForCollationIndex(index) + " : " + index + " : " - + CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[index] + " : " + CharsetMapping.getMysqlCharsetNameForCollationIndex(index) - + " : " + CharsetMapping.CHARSET_NAME_TO_CHARSET.get(csname) + " : " + CharsetMapping.getJavaEncodingForMysqlCharset(csname) - + " : " + CharsetMapping.getMysqlCharsetForJavaEncoding(alias, this.serverVersion) + " : " - + CharsetMapping.getCollationIndexForJavaEncoding(alias, this.serverVersion) + " : " + CharsetMapping.isMultibyteCharset(alias)); + + CharsetMappingWrapper.getStaticJavaEncodingForCollationIndex(index) + " : " + index + " : " + + CharsetMappingWrapper.getStaticCollationNameForCollationIndex(index) + " : " + + CharsetMappingWrapper.getStaticMysqlCharsetNameForCollationIndex(index) + " : " + + CharsetMappingWrapper.getStaticMysqlCharsetByName(csname) + " : " + + CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset(csname) + " : " + + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding(alias, this.serverVersion) + " : " + + CharsetMappingWrapper.getStaticCollationIndexForJavaEncoding(alias, this.serverVersion) + " : " + + CharsetMappingWrapper.isStaticMultibyteCharset(alias)); } System.out.println("==================================="); } for (int i = 1; i < CharsetMapping.MAP_SIZE; i++) { - String csname = CharsetMapping.getMysqlCharsetNameForCollationIndex(i); - String enc = CharsetMapping.getJavaEncodingForCollationIndex(i); - System.out.println((i + " ").substring(0, 4) + " by index--> " - + (CharsetMapping.COLLATION_INDEX_TO_COLLATION_NAME[i] + " ").substring(0, 20) + " : " - + (csname + " ").substring(0, 10) + " : " + (enc + " ").substring(0, 20) - - + " by charset--> " + (CharsetMapping.getJavaEncodingForMysqlCharset(csname) + " ").substring(0, 20) - - + " by encoding--> " + (CharsetMapping.getCollationIndexForJavaEncoding(enc, this.serverVersion) + " ").substring(0, 4) + " : " - + (CharsetMapping.getMysqlCharsetForJavaEncoding(enc, this.serverVersion) + " ").substring(0, 15)); + String csname = CharsetMappingWrapper.getStaticMysqlCharsetNameForCollationIndex(i); + if (csname != null) { + String enc = CharsetMappingWrapper.getStaticJavaEncodingForCollationIndex(i); + System.out.println((i + " ").substring(0, 4) + " by index--> " + + (CharsetMappingWrapper.getStaticCollationNameForCollationIndex(i) + " ").substring(0, 20) + " : " + + (csname + " ").substring(0, 10) + " : " + (enc + " ").substring(0, 20) + + + " by charset--> " + (CharsetMappingWrapper.getStaticJavaEncodingForMysqlCharset(csname) + " ").substring(0, 20) + + " by encoding--> " + (CharsetMappingWrapper.getStaticCollationIndexForJavaEncoding(enc, this.serverVersion) + " ").substring(0, 4) + + " : " + (CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding(enc, this.serverVersion) + " ").substring(0, 15)); + } } } diff --git a/src/test/java/testsuite/simple/ConnectionTest.java b/src/test/java/testsuite/simple/ConnectionTest.java index ccd9d0b55..cf6e1d3cd 100644 --- a/src/test/java/testsuite/simple/ConnectionTest.java +++ b/src/test/java/testsuite/simple/ConnectionTest.java @@ -85,7 +85,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.MysqlConnection; import com.mysql.cj.NativeSession; import com.mysql.cj.PreparedQuery; @@ -822,7 +822,7 @@ public void testLocalInfileWithUrl() throws Exception { Connection loadConn = getConnectionWithProps(props); Statement loadStmt = loadConn.createStatement(); - String charset = " CHARACTER SET " + CharsetMapping.getMysqlCharsetForJavaEncoding( + String charset = " CHARACTER SET " + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding( ((MysqlConnection) loadConn).getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), ((JdbcConnection) loadConn).getServerVersion()); diff --git a/src/test/java/testsuite/simple/ResultSetTest.java b/src/test/java/testsuite/simple/ResultSetTest.java index beab3464f..29148b773 100644 --- a/src/test/java/testsuite/simple/ResultSetTest.java +++ b/src/test/java/testsuite/simple/ResultSetTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2020, Oracle and/or its affiliates. + * Copyright (c) 2005, 2021, Oracle and/or its affiliates. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License, version 2.0, as published by the @@ -65,7 +65,6 @@ import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; import com.mysql.cj.MysqlConnection; import com.mysql.cj.MysqlType; import com.mysql.cj.conf.PropertyKey; @@ -88,13 +87,7 @@ public void testPadding() throws Exception { this.rs = this.stmt.executeQuery("SHOW COLLATION"); while (this.rs.next()) { int index = ((Number) this.rs.getObject(3)).intValue(); - String charsetName = null; - if (((ConnectionImpl) c).getSession().getProtocol().getServerSession().indexToCustomMysqlCharset != null) { - charsetName = ((ConnectionImpl) c).getSession().getProtocol().getServerSession().indexToCustomMysqlCharset.get(index); - } - if (charsetName == null) { - charsetName = CharsetMapping.getMysqlCharsetNameForCollationIndex(index); - } + String charsetName = ((ConnectionImpl) c).getSession().getServerSession().getCharsetSettings().getMysqlCharsetNameForCollationIndex(index); if (charsetName != null) { charsetsMap.put(charsetName, index); } diff --git a/src/test/java/testsuite/simple/StatementsTest.java b/src/test/java/testsuite/simple/StatementsTest.java index 2f6c3e60f..54bc556e3 100644 --- a/src/test/java/testsuite/simple/StatementsTest.java +++ b/src/test/java/testsuite/simple/StatementsTest.java @@ -77,7 +77,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import com.mysql.cj.CharsetMapping; +import com.mysql.cj.CharsetMappingWrapper; import com.mysql.cj.MysqlConnection; import com.mysql.cj.MysqlType; import com.mysql.cj.conf.PropertyKey; @@ -1872,8 +1872,8 @@ public void testLocalInfileHooked() throws Exception { try { ((com.mysql.cj.jdbc.JdbcStatement) testStmt).setLocalInfileInputStream(stream); - testStmt.execute( - "LOAD DATA LOCAL INFILE 'bogusFileName' INTO TABLE localInfileHooked CHARACTER SET " + CharsetMapping.getMysqlCharsetForJavaEncoding( + testStmt.execute("LOAD DATA LOCAL INFILE 'bogusFileName' INTO TABLE localInfileHooked CHARACTER SET " + + CharsetMappingWrapper.getStaticMysqlCharsetForJavaEncoding( ((MysqlConnection) this.conn).getPropertySet().getStringProperty(PropertyKey.characterEncoding).getValue(), this.serverVersion)); assertEquals(-1, stream.read()); this.rs = testStmt.executeQuery("SELECT field2 FROM localInfileHooked ORDER BY field1 ASC"); diff --git a/src/test/java/testsuite/x/devapi/SessionTest.java b/src/test/java/testsuite/x/devapi/SessionTest.java index 09f2d2daf..033d7222c 100644 --- a/src/test/java/testsuite/x/devapi/SessionTest.java +++ b/src/test/java/testsuite/x/devapi/SessionTest.java @@ -1021,8 +1021,8 @@ public void run() { cli0 = cf.getClient(this.baseUrl, "{\"pooling\": {\"enabled\": true, \"maxSize\" : 2}}"); s0 = cli0.getSession(); s1 = cli0.getSession(); - long id0 = ((SessionImpl) s0).getSession().getServerSession().getThreadId(); - long id1 = ((SessionImpl) s1).getSession().getServerSession().getThreadId(); + long id0 = ((SessionImpl) s0).getSession().getServerSession().getCapabilities().getThreadId(); + long id1 = ((SessionImpl) s1).getSession().getServerSession().getCapabilities().getThreadId(); s0.sql("SET @a='s0'").execute(); s0.sql("CREATE TEMPORARY TABLE testpooledsessionstmps0(x int)").execute(); @@ -1050,8 +1050,8 @@ public void run() { Session s0_new = cli0.getSession(); Session s1_new = cli0.getSession(); - assertEquals(id0, ((SessionImpl) s0_new).getSession().getServerSession().getThreadId()); - assertEquals(id1, ((SessionImpl) s1_new).getSession().getServerSession().getThreadId()); + assertEquals(id0, ((SessionImpl) s0_new).getSession().getServerSession().getCapabilities().getThreadId()); + assertEquals(id1, ((SessionImpl) s1_new).getSession().getServerSession().getCapabilities().getThreadId()); res = s0_new.sql("SELECT @a as a").execute(); assertTrue(res.hasNext());