Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PYTHON-1393 Add support for Cassandra 4.1.x and 5.0 releases to CI #1220

Merged
merged 8 commits into from
Jul 30, 2024
8 changes: 6 additions & 2 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import com.datastax.jenkins.drivers.python.Slack

slack = new Slack()

DEFAULT_CASSANDRA = ['3.0', '3.11', '4.0']
DEFAULT_CASSANDRA = ['3.0', '3.11', '4.0', '4.1', '5.0-beta1']
DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30']
DEFAULT_RUNTIME = ['3.8.16', '3.9.16', '3.10.11', '3.11.3', '3.12.0']
DEFAULT_CYTHON = ["True", "False"]
Expand Down Expand Up @@ -557,6 +557,10 @@ pipeline {
<td><strong>4.0</strong></td>
<td>Apache CassandraⓇ v4.0.x</td>
</tr>
<tr>
<td><strong>5.0-beta1</strong></td>
<td>Apache CassandraⓇ v5.0-beta1</td>
</tr>
<tr>
<td><strong>dse-5.1.35</strong></td>
<td>DataStax Enterprise v5.1.x</td>
Expand Down Expand Up @@ -644,7 +648,7 @@ pipeline {
parameterizedCron(branchPatternCron().matcher(env.BRANCH_NAME).matches() ? """
# Every weeknight (Monday - Friday) around 4:00 AM
# These schedules will run with and without Cython enabled for Python 3.8.16 and 3.12.0
H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.12.0;CI_SCHEDULE_SERVER_VERSION=3.11 4.0 dse-5.1.35 dse-6.8.30
H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.16 3.12.0;CI_SCHEDULE_SERVER_VERSION=3.11 4.0 5.0-beta1 dse-5.1.35 dse-6.8.30
""" : "")
}

Expand Down
47 changes: 44 additions & 3 deletions tests/integration/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import os
from cassandra.cluster import Cluster

Expand Down Expand Up @@ -561,18 +562,23 @@ def use_cluster(cluster_name, nodes, ipformat=None, start=True, workloads=None,

CCM_CLUSTER.set_dse_configuration_options(dse_options)
else:
CCM_CLUSTER = CCMCluster(path, cluster_name, **ccm_options)
ccm_cluster_clz = CCMCluster if Version(cassandra_version) < Version('4.1') else Cassandra41CCMCluster
CCM_CLUSTER = ccm_cluster_clz(path, cluster_name, **ccm_options)
CCM_CLUSTER.set_configuration_options({'start_native_transport': True})
if Version(cassandra_version) >= Version('2.2'):
CCM_CLUSTER.set_configuration_options({'enable_user_defined_functions': True})
if Version(cassandra_version) >= Version('3.0'):
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
if Version(cassandra_version) >= Version('4.0-a'):
# The config.yml option below is deprecated in C* 4.0 per CASSANDRA-17280
if Version(cassandra_version) < Version('4.0'):
CCM_CLUSTER.set_configuration_options({'enable_scripted_user_defined_functions': True})
else:
# Cassandra version >= 4.0
CCM_CLUSTER.set_configuration_options({
'enable_materialized_views': True,
'enable_sasi_indexes': True,
'enable_transient_replication': True,
})

common.switch_cluster(path, cluster_name)
CCM_CLUSTER.set_configuration_options(configuration_options)
CCM_CLUSTER.populate(nodes, ipformat=ipformat, use_single_interface=use_single_interface)
Expand Down Expand Up @@ -1011,3 +1017,38 @@ def __new__(cls, **kwargs):
kwargs['allow_beta_protocol_version'] = cls.DEFAULT_ALLOW_BETA
return Cluster(**kwargs)

# Subclass of CCMCluster (i.e. ccmlib.cluster.Cluster) which transparently performs
# conversion of cassandra.yml directives into something matching the new syntax
# introduced by CASSANDRA-15234
class Cassandra41CCMCluster(CCMCluster):
__test__ = False
IN_MS_REGEX = re.compile('^(\w+)_in_ms$')
IN_KB_REGEX = re.compile('^(\w+)_in_kb$')
ENABLE_REGEX = re.compile('^enable_(\w+)$')

def _get_config_key(self, k, v):
if "." in k:
return k
m = self.IN_MS_REGEX.match(k)
if m:
return m.group(1)
m = self.ENABLE_REGEX.search(k)
if m:
return "%s_enabled" % (m.group(1))
m = self.IN_KB_REGEX.match(k)
if m:
return m.group(1)
return k

def _get_config_val(self, k, v):
m = self.IN_MS_REGEX.match(k)
if m:
return "%sms" % (v)
m = self.IN_KB_REGEX.match(k)
if m:
return "%sKiB" % (v)
return v

def set_configuration_options(self, values=None, *args, **kwargs):
new_values = {self._get_config_key(k, str(v)):self._get_config_val(k, str(v)) for (k,v) in values.items()}
super(Cassandra41CCMCluster, self).set_configuration_options(values=new_values, *args, **kwargs)
14 changes: 12 additions & 2 deletions tests/integration/cqlengine/query/test_queryset.py
Original file line number Diff line number Diff line change
Expand Up @@ -1362,11 +1362,21 @@ def tearDownClass(cls):
super(TestModelQueryWithFetchSize, cls).tearDownClass()
drop_table(TestModelSmall)

@execute_count(9)
@execute_count(19)
def test_defaultFetchSize(self):
# Use smaller batch sizes to avoid hitting the max. We trigger an InvalidRequest
# response for Cassandra 4.1.x and 5.0.x if we just do the whole thing as one
# large batch. We're just using this to populate values for a test, however,
# so shifting to smaller batches should be fine.
for i in range(0, 5000, 500):
with BatchQuery() as b:
range_max = i + 500
for j in range(i, range_max):
TestModelSmall.batch(b).create(test_id=j)
with BatchQuery() as b:
for i in range(5100):
for i in range(5000, 5100):
TestModelSmall.batch(b).create(test_id=i)

self.assertEqual(len(TestModelSmall.objects.fetch_size(1)), 5100)
self.assertEqual(len(TestModelSmall.objects.fetch_size(500)), 5100)
self.assertEqual(len(TestModelSmall.objects.fetch_size(4999)), 5100)
Expand Down