Merge pull request #715 from kbase/dev-mongo_upgrade

Upgrade to MongoDB 7.0.4

.classpath

@@ -52,7 +52,7 @@
 	<classpathentry kind="lib" path="/jars/lib/jars/kbase/auth/kbase-auth-0.4.4.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/junit/junit-4.12.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/hamcrest/hamcrest-core-1.3.jar"/>
-	<classpathentry kind="lib" path="/jars/lib/jars/kbase/auth2/kbase-auth2test-0.2.4.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/kbase/auth2/kbase-auth2-test-fat-0.6.1.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/kafka/kafka-clients-2.1.0.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/kbase/shock/shock-client-0.1.0.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/amazon/V2/eventstream-1.0.1.jar"/>
@@ -85,7 +85,10 @@
 	<classpathentry kind="lib" path="/jars/lib/jars/apache_commons/http/httpmime-4.5.8.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/picocli/picocli-4.6.1.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/kbase/sample/SampleServiceClient-0.1.1.jar"/>
-	<classpathentry kind="lib" path="/jars/lib/jars/mongo/mongo-java-driver-3.12.10.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/mongo/mongodb-driver-core-4.11.1.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/mongo/mongodb-driver-sync-4.11.1.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/bson/bson-record-codec-4.11.1.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/bson/bson-4.11.1.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/caffeinecache/caffeine-2.9.3.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/slf4j/slf4j-api-1.7.30.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/amazon/V2/annotations-2.17.214.jar"/>
@@ -105,6 +108,6 @@
 	<classpathentry kind="lib" path="/jars/lib/jars/amazon/V2/third-party-jackson-core-2.17.214.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/amazon/V2/url-connection-client-2.17.214.jar"/>
 	<classpathentry kind="lib" path="/jars/lib/jars/amazon/V2/utils-2.17.214.jar"/>
-	<classpathentry kind="lib" path="/jars/lib/jars/kbase/common/kbase-common-0.1.1.jar"/>
+	<classpathentry kind="lib" path="/jars/lib/jars/kbase/common/kbase-common-0.2.0.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
       matrix:
         include:
           - java: '11'
-            mongo: 'mongodb-linux-x86_64-3.6.23'
+            mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4'
             minio: '2019-05-23T00-29-34Z'
             wired_tiger: 'true'
             ant_test: 'test_quick_coverage'

build.xml

@@ -24,7 +24,7 @@
   <property name="jackson.version" value="2.9.9"/>
 
   <fileset dir="${jardir}" id="sourceslib">
-    <include name="kbase/common/kbase-common-0.1.1.jar"/>
+    <include name="kbase/common/kbase-common-0.2.0.jar"/>
   </fileset>
 
   <fileset dir="${jardir}" id="serverlib">
@@ -39,7 +39,7 @@
   <fileset dir="${jardir}" id="clientserverlib">
     <include name="junit/junit-4.12.jar"/>
     <include name="hamcrest/hamcrest-core-1.3.jar"/>
-    <include name="kbase/common/kbase-common-0.1.1.jar"/>
+    <include name="kbase/common/kbase-common-0.2.0.jar"/>
     <!-- joda is required for kbase-common -->
     <include name="joda/joda-time-2.2.jar"/>
     <include name="kbase/auth/kbase-auth-0.4.4.jar"/>
@@ -62,7 +62,10 @@
     <include name="apache_commons/commons-io-2.4.jar"/>
     <include name="apache_commons/commons-lang3-3.1.jar"/>
     <include name="picocli/picocli-4.6.1.jar"/>
-    <include name="mongo/mongo-java-driver-3.12.10.jar"/>
+    <include name="mongo/mongodb-driver-core-4.11.1.jar"/>
+    <include name="mongo/mongodb-driver-sync-4.11.1.jar"/>
+    <include name="bson/bson-record-codec-4.11.1.jar"/>
+    <include name="bson/bson-4.11.1.jar"/>
     <include name="slf4j/slf4j-api-1.7.30.jar"/>
     <include name="logback/logback-core-1.1.2.jar"/>
     <include name="logback/logback-classic-1.1.2.jar"/>
@@ -101,14 +104,15 @@
     <include name="apache_commons/commons-collections-3.2.1.jar"/>
     <include name="jsemver/java-semver-0.9.0.jar"/>
     <include name="equalsverifier/equalsverifier-3.1.10.jar"/>
-    <include name="kbase/auth2/kbase-auth2test-0.2.4.jar"/>
     <include name="arangodb/arangodb-java-driver-6.7.2.jar"/>
     <include name="arangodb/velocypack-2.4.0.jar"/>
     <!-- mockito and dependencies -->
     <include name="mockito/mockito-core-3.0.0.jar"/>
     <include name="bytebuddy/byte-buddy-1.9.10.jar"/>
     <include name="bytebuddy/byte-buddy-agent-1.9.10.jar"/>
     <include name="objenesis/objenesis-2.6.jar"/>
+<!--    put auth2 fat jar in the last entry so that it would not shadow other jars-->
+    <include name="kbase/auth2/kbase-auth2-test-fat-0.6.1.jar"/>
   </fileset>
 
   <union id="applicationjars">

docsource/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '0.14'
 # The full version, including alpha/beta/rc tags.
-release = '0.14.2'
+release = '0.14.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docsource/releasenotes.rst

@@ -3,6 +3,14 @@
 Workspace service release notes
 ===============================
 
+VERSION: 0.14.3 (Released 2/22/2024)
+------------------------------------
+
+UPDATES:
+
+* The MongoDB clients have been updated to the most recent version and the service tested against Mongo 7.
+
+
 VERSION: 0.14.2 (Released 11/9/2023)
 ------------------------------------
 

scripts/workspace_container_test.py

@@ -16,7 +16,7 @@
 
 """
 
-WORKSPACE_VERSION = "0.14.2"
+WORKSPACE_VERSION = "0.14.3"
 
 AUTH_URL = "http://localhost:8080"
 WS_URL = "http://localhost:7058"

src/us/kbase/typedobj/db/test/MongoTypeStorageTest.java

@@ -12,7 +12,8 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 
 import us.kbase.common.test.TestCommon;
@@ -43,7 +44,7 @@ public static void setUp() throws Exception {
 				MONGO.getServerPort());
 
 		@SuppressWarnings("resource")
-		final MongoClient mc = new MongoClient("localhost:" + MONGO.getServerPort());
+		final MongoClient mc = MongoClients.create("mongodb://localhost:" + MONGO.getServerPort());
 		MONGO_DB = mc.getDatabase("test_" + MongoTypeStorageTest.class.getSimpleName());
 	}
 

src/us/kbase/typedobj/db/test/TypeRegisteringTest.java

@@ -33,7 +33,8 @@
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 
 import us.kbase.common.test.TestCommon;
@@ -144,7 +145,7 @@ public static MongoDatabase createMongoDbConnection() throws Exception {
 					mongo.getTempDir());
 		}
 		@SuppressWarnings("resource")
-		final MongoClient mcli = new MongoClient("localhost:" + mongo.getServerPort());
+		final MongoClient mcli = MongoClients.create("mongodb://localhost:" + mongo.getServerPort());
 		final MongoDatabase mdb = mcli.getDatabase("TypeRegisteringTest");
 		TestCommon.destroyDB(mdb);
 		return mdb;

src/us/kbase/workspace/kbase/InitWorkspaceServer.java

@@ -22,11 +22,12 @@
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.slf4j.LoggerFactory;
 
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientOptions;
 import com.mongodb.MongoCredential;
 import com.mongodb.MongoException;
 import com.mongodb.ServerAddress;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 
 import us.kbase.abstracthandle.AbstractHandleClient;
@@ -411,16 +412,18 @@ private static SampleIdHandlerFactory getSampleIdHandlerFactory(
 	public static MongoClient buildMongo(final KBaseWorkspaceConfig c, final String dbName)
 			throws WorkspaceInitException {
 		//TODO ZLATER MONGO handle shards & replica sets
-		final MongoClientOptions opts = MongoClientOptions.builder()
-				.retryWrites(c.getMongoRetryWrites()).build();
+		final MongoClientSettings.Builder mongoBuilder = MongoClientSettings.builder()
+				.retryWrites(c.getMongoRetryWrites())
+				.applyToClusterSettings(builder -> builder.hosts(
+						Arrays.asList(new ServerAddress(c.getHost()))));
 		try {
 			if (c.getMongoUser() != null) {
 				final MongoCredential creds = MongoCredential.createCredential(
 						c.getMongoUser(), dbName, c.getMongoPassword().toCharArray());
 				// unclear if and when it's safe to clear the password
-				return new MongoClient(new ServerAddress(c.getHost()), creds, opts);
+				return MongoClients.create(mongoBuilder.credential(creds).build());
 			} else {
-				return new MongoClient(new ServerAddress(c.getHost()), opts);
+				return MongoClients.create(mongoBuilder.build());
 			}
 		} catch (MongoException e) {
 			LoggerFactory.getLogger(InitWorkspaceServer.class).error(

src/us/kbase/workspace/kbase/SchemaUpdaterCLI.java

@@ -9,7 +9,7 @@
 
 import org.slf4j.LoggerFactory;
 
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
 import com.mongodb.client.MongoDatabase;
 
 import ch.qos.logback.classic.Level;

src/us/kbase/workspace/test/WorkspaceMongoIndex.java

@@ -0,0 +1,25 @@
+package us.kbase.workspace.test;
+
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class WorkspaceMongoIndex {
+
+    public static Set<Document> getAndNormalizeIndexes(final MongoDatabase db, final String collectionName) {
+        final Set<Document> indexes = new HashSet<>();
+        for (final Document index: db.getCollection(collectionName).listIndexes()) {
+            // In MongoDB 4.4, the listIndexes and the mongo shell helper method db.collection.getIndexes()
+            // no longer returns the namespace ns field in the index specification documents.
+            index.remove("ns");
+            // some versions of Mongo return ints, some longs. Convert all to longs.
+            if (index.containsKey("expireAfterSeconds")) {
+                index.put("expireAfterSeconds", ((Number) index.get("expireAfterSeconds")).longValue());
+            }
+            indexes.add(index);
+        }
+        return indexes;
+    }
+}

src/us/kbase/workspace/test/controllers/handle/HandleServiceController.java

@@ -9,17 +9,14 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.LinkedList;
 import java.util.Map;
-import java.util.Scanner;
 import org.apache.commons.io.FileUtils;
 import org.ini4j.Ini;
 import org.ini4j.Profile.Section;
 
 import us.kbase.auth.AuthToken;
 import us.kbase.common.test.controllers.mongo.MongoController;
-import us.kbase.common.test.controllers.shock.ShockController;
 
 
 /** Q&D Utility to run the Handle Service for the purposes of testing from Java.
@@ -137,36 +134,4 @@ public void destroy(final boolean deleteTempFiles, final boolean dumpLogToStdOut
 			FileUtils.deleteDirectory(tempDir.toFile());
 		}
 	}
-
-	public static void main(String[] args) throws Exception {
-		MongoController monc = new MongoController(
-				"/kb/runtime/bin/mongod",
-				Paths.get("workspacetesttemp"), false);
-		ShockController sc = new ShockController(
-				"/kb/deployment/bin/shock-server",
-				"0.9.6",
-				Paths.get("workspacetesttemp"),
-				System.getProperty("test.user1"),
-				"localhost:" + monc.getServerPort(),
-				"shockdb", "foo", "foo", new URL("http://foo.com"));
-		HandleServiceController hsc = new HandleServiceController(
-				monc,
-				"http://localhost:" + sc.getServerPort(),
-				null, //this will break the hm, need a token
-				Paths.get("workspacetesttemp"),
-				new URL("http://foo.com"),
-				"KBASE_ADMIN",
-				"/kb/deployment/lib",
-				"handle_controller_test_handle_db");
-		System.out.println("handlesrv: " + hsc.getHandleServerPort());
-		System.out.println(hsc.getTempDir());
-		Scanner reader = new Scanner(System.in);
-		System.out.println("any char to shut down");
-		//get user input for a
-		reader.next();
-		hsc.destroy(false);
-		sc.destroy(false);
-		monc.destroy(false);
-		reader.close();
-	}
 }

src/us/kbase/workspace/test/database/mongo/GridFSBlobStoreTest.java

@@ -24,7 +24,9 @@
 
 import com.github.zafarkhaja.semver.Version;
 import com.google.common.collect.ImmutableMap;
-import com.mongodb.MongoClient;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.gridfs.GridFSBucket;
 import com.mongodb.client.gridfs.GridFSBuckets;
@@ -58,7 +60,7 @@ public static void setUpClass() throws Exception {
 				mongo.getTempDir());
 		TestCommon.stfuLoggers();
 		@SuppressWarnings("resource")
-		final MongoClient mongoClient = new MongoClient("localhost:" + mongo.getServerPort());
+		final MongoClient mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort());
 		final MongoDatabase db = mongoClient.getDatabase("GridFSBackendTest");
 		gfs = GridFSBuckets.create(db);
 		gfsb = new GridFSBlobStore(db);

src/us/kbase/workspace/test/database/mongo/MongoInternalsTest.java

@@ -85,7 +85,9 @@
 import us.kbase.workspace.test.workspace.WorkspaceTester;
 
 import com.google.common.collect.ImmutableMap;
-import com.mongodb.MongoClient;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
 
@@ -125,8 +127,7 @@ public static void setUpClass() throws Exception {
 		System.out.println("Using mongo temp dir " +
 				mongo.getTempDir());
 		TestCommon.stfuLoggers();
-		String mongohost = "localhost:" + mongo.getServerPort();
-		mongoClient = new MongoClient(mongohost);
+		mongoClient = MongoClients.create("mongodb://localhost:" + mongo.getServerPort());
 		db = mongoClient.getDatabase("MongoInternalsTest");
 		final MongoDatabase tdb = mongoClient.getDatabase("MongoInternalsTest_types");
 		TestCommon.destroyDB(db);