Flaky tests (#6032)

* Update email password test
* Looking into flaky tests
* Skip nonsense test
* Using 'this.retries()' to accommodate race conditions

----
Co-authored-by: LJ <[email protected]>
Co-authored-by: Kenneth Geisshirt <[email protected]>

integration-tests/tests/src/tests/sync/flexible.ts

@@ -1681,6 +1681,12 @@ describe.skipIf(environment.missingServer, "Flexible sync", function () {
         });
 
         it("does not wait for objects to sync when timeout is shorter", async function (this: RealmContext) {
+          // We're rerunning this test to prevent the subscription set state from
+          // completing faster than expected (from the point when `subscribe()` is
+          // called with timeout `0`, to the next line when the state is checked).
+          // (This is due to the test being flaky only on CI.)
+          this.retries(100);
+
           expect(this.realm.subscriptions).to.have.length(0);
 
           const peopleOver10 = this.realm.objects(Person).filtered("age > 10");

integration-tests/tests/src/tests/sync/open-behavior.ts

@@ -506,6 +506,7 @@ describe("OpenBehaviour", function () {
   });
 
   it("canceling promise with multiple realm.open calls active", async function (this: AppContext) {
+    this.retries(3);
     const user = await this.app.logIn(Realm.Credentials.anonymous());
     const partitionValue = generatePartition();
 

integration-tests/tests/src/tests/sync/sync-session.ts

@@ -796,10 +796,11 @@ describe("SessionTest", () => {
   describe("writeCopyTo on synced realms", () => {
     afterEach(() => Realm.clearTestState());
     it("can create encrypted copies", async function (this: AppContext) {
+      this.retries(3);
       /*
-    Test that we can create encrypted copies of a realm, and that only the
-    correct encryption key will allow us to re-open that copy
-  */
+      Test that we can create encrypted copies of a realm, and that only the
+      correct encryption key will allow us to re-open that copy
+    */
       const credentials1 = await getRegisteredEmailPassCredentials(this.app);
       const credentials2 = await getRegisteredEmailPassCredentials(this.app);
       const partition = generatePartition();
@@ -960,9 +961,11 @@ describe("SessionTest", () => {
       // log out the user that created the realm
       await user1.logOut();
 
-      // add another 25 people
+      realm1.syncSession?.pause();
+
+      // add another 2500 people
       realm1.write(() => {
-        for (let i = 0; i < 25; i++) {
+        for (let i = 0; i < 2500; i++) {
           realm1.create("Person", {
             _id: new BSON.ObjectId(),
             age: i,
@@ -993,6 +996,7 @@ describe("SessionTest", () => {
       }).throws("All client changes must be integrated in server before writing copy");
 
       // log back in and upload the changes we made locally
+      realm1.syncSession?.resume();
       user1 = await this.app.logIn(credentials1);
       await realm1.syncSession?.uploadAllLocalChanges();
 

integration-tests/tests/src/tests/sync/user.ts

@@ -102,43 +102,52 @@ describe.skipIf(environment.missingServer, "User", () => {
       const provider = this.app.emailPasswordAuth;
       expect(provider).instanceof(Realm.Auth.EmailPasswordAuth);
     });
+  });
 
-    it("autoverify email password works", async function (this: AppContext & RealmContext) {
-      const validEmail = randomVerifiableEmail();
-      const invalidEmail = randomNonVerifiableEmail();
-      const invalidPassword = "pass"; // too short
-      const validPassword = "password123456";
+  describe("autoverify email password works", () => {
+    // any email address is valid and registerUser() will never fail when auto-confirm is enabled. The combination invalid email
+    // valid password cannot be tested.
+    importAppBefore(buildAppConfig("with-email-password").emailPasswordAuth());
+    removeExistingUsers();
+
+    const validEmail = randomVerifiableEmail();
+    const invalidEmail = randomNonVerifiableEmail();
+    const invalidPassword = "pass"; // too short
+    const validPassword = "password123456";
 
-      // invalid email, invalid password
-      let credentials = Realm.Credentials.emailPassword({ email: invalidEmail, password: invalidPassword });
+    it("invalid email, invalid password", async function (this: AppContext & RealmContext) {
+      const credentials = Realm.Credentials.emailPassword({ email: invalidEmail, password: invalidPassword });
       await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
       await expect(
         this.app.emailPasswordAuth.registerUser({ email: invalidEmail, password: invalidPassword }),
-      ).to.be.rejectedWith("password must be between 6 and 128 characters");
-      await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user did not register
-
-      // invalid email, valid password
-      credentials = Realm.Credentials.emailPassword({ email: invalidEmail, password: validPassword });
-      await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
-      expect(
-        this.app.emailPasswordAuth.registerUser({ email: invalidEmail, password: validPassword }),
-      ).to.be.rejectedWith(`failed to confirm user "${invalidEmail}"`);
+      ).to.eventually.be.rejectedWith("password must be between 6 and 128 characters");
       await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user did not register
+    });
 
-      // valid email, invalid password
-      credentials = Realm.Credentials.emailPassword({ email: validEmail, password: invalidPassword });
+    it("valid email, invalid password", async function (this: AppContext & RealmContext) {
+      const credentials = Realm.Credentials.emailPassword({ email: validEmail, password: invalidPassword });
       await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
       await expect(
         this.app.emailPasswordAuth.registerUser({ email: validEmail, password: invalidPassword }),
-      ).to.be.rejectedWith("password must be between 6 and 128 characters");
+      ).to.eventually.be.rejectedWith("password must be between 6 and 128 characters");
       await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user did not register
     });
 
     it("valid email, valid password", async function (this: AppContext & RealmContext) {
       const validEmail = randomVerifiableEmail();
       const validPassword = "password123456";
 
-      // valid email, valid password
+      const credentials = Realm.Credentials.emailPassword({ email: validEmail, password: validPassword });
+      await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
+      await this.app.emailPasswordAuth.registerUser({ email: validEmail, password: validPassword });
+      const user = await this.app.logIn(credentials);
+      expectIsUser(user);
+      expectIsSameUser(user, this.app.currentUser);
+      await user.logOut();
+    });
+
+    it("valid email, valid password", async function (this: AppContext & RealmContext) {
+      removeExistingUsers();
       const credentials = Realm.Credentials.emailPassword({ email: validEmail, password: validPassword });
       await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
       await this.app.emailPasswordAuth.registerUser({ email: validEmail, password: validPassword });
@@ -567,6 +576,18 @@ describe.skipIf(environment.missingServer, "User", () => {
       await this.app.emailPasswordAuth.retryCustomConfirmation({ email: pendingEmail });
     });
 
+    it("custom confirmation function rejects invalid email", async function (this: AppContext & RealmContext) {
+      const invalidEmail = randomNonVerifiableEmail();
+      const validPassword = "password123456";
+
+      const credentials = Realm.Credentials.emailPassword({ email: invalidEmail, password: validPassword });
+      await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user does not exist yet
+      expect(
+        this.app.emailPasswordAuth.registerUser({ email: invalidEmail, password: validPassword }),
+      ).to.eventually.be.rejectedWith(`failed to confirm user "${invalidEmail}"`);
+      await expect(this.app.logIn(credentials)).to.be.rejectedWith("invalid username/password"); // this user did not register
+    });
+
     it("reset password function works", async function (this: AppContext & RealmContext) {
       const validEmail = randomVerifiableEmail();
       const validPassword = "password123456";