Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support bigint during quota evaluation #5729

Open
wants to merge 4 commits into
base: development/8.8
Choose a base branch
from
Open

Support bigint during quota evaluation #5729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
15e07e4
Activate es2020 globals in eslint config
williamlardier Jan 15, 2025
28dbb47
Support bigint metrics returned by the scuba client
williamlardier Jan 15, 2025
49e96cd
Use more recent Java version
williamlardier Jan 16, 2025
711e9b7
Bump cloudserver version
williamlardier Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .eslintrc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"plugins": [
"mocha"
],
"env": {
"es2020": true
},
"rules": {
"import/extensions": "off",
"lines-around-directive": "off",
Expand Down
7 changes: 5 additions & 2 deletions lib/api/apiUtils/quotas/quotaUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ function _evaluateQuotas(
return parallelDone(err);
}
if (!isMetricStale(bucketMetrics, 'bucket', bucket.getName(), action, inflight, log) &&
bucketMetrics.bytesTotal + inflightForCheck > bucketQuota) {
BigInt(bucketMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) >
BigInt(bucketQuota || 0)) {
log.debug('Bucket quota exceeded', {
bucket: bucket.getName(),
action,
Expand All @@ -147,8 +148,10 @@ function _evaluateQuotas(
if (err || inflight < 0) {
return parallelDone(err);
}
// Metrics are served as BigInt strings
if (!isMetricStale(accountMetrics, 'account', account.account, action, inflight, log) &&
accountMetrics.bytesTotal + inflightForCheck > accountQuota) {
BigInt(accountMetrics.bytesTotal || 0) + BigInt(inflightForCheck || 0) >
BigInt(accountQuota || 0)) {
log.debug('Account quota exceeded', {
accountId: account.account,
action,
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@zenko/cloudserver",
"version": "8.8.41",
"version": "8.8.42",
"description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
"main": "index.js",
"engines": {
Expand Down
11 changes: 10 additions & 1 deletion tests/functional/jaws/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
<version>1.0.0</version>
<properties>
<project.custom.encoding>UTF-8</project.custom.encoding>
<project.custom.java.version>1.6</project.custom.java.version>
<project.custom.java.version>1.8</project.custom.java.version>
<maven.compiler.source>${project.custom.java.version}</maven.compiler.source>
<maven.compiler.target>${project.custom.java.version}</maven.compiler.target>
<project.build.sourceEncoding>${project.custom.encoding}</project.build.sourceEncoding>
Expand All @@ -53,6 +53,15 @@
<useSystemClassLoader>false</useSystemClassLoader>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>${maven.compiler.source}</source>
<target>${maven.compiler.target}</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
Expand Down
36 changes: 18 additions & 18 deletions tests/quota/awsNodeSdk.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
next => abortMPU(bucket, key, uploadId, 0, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0n);
return next();
},
next => deleteBucket(bucket, next),
Expand Down Expand Up @@ -553,13 +553,13 @@ function multiObjectDelete(bucket, keys, size, callback) {
}, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => restoreObject(bucket, key, 0, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
Expand Down Expand Up @@ -590,7 +590,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size * 2);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size * 2));
return next();
},
next => wait(inflightFlushFrequencyMS * 2, next),
Expand Down Expand Up @@ -643,7 +643,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
next => wait(inflightFlushFrequencyMS * 2, next),
// Should still have 0 as inflight
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0n);
return next();
},
next => wait(inflightFlushFrequencyMS * 2, next),
Expand Down Expand Up @@ -683,7 +683,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size - 100);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size - 100));
return next();
},
next => deleteObject(bucket, key, size, next),
Expand Down Expand Up @@ -715,7 +715,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0n);
return next();
},
next => deleteBucket(bucket, next),
Expand Down Expand Up @@ -752,7 +752,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0n);
return next();
},
next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => {
Expand Down Expand Up @@ -786,7 +786,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => deleteVersionID(bucket, key, vID, size, err => {
Expand All @@ -795,7 +795,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
], done);
Expand All @@ -818,7 +818,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
Expand All @@ -832,7 +832,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
Expand All @@ -857,7 +857,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => fakeMetadataArchive(bucket, key, vID, {
Expand All @@ -876,7 +876,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
return next();
}),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), size);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(size));
return next();
},
next => deleteVersionID(bucket, key, vID, size, next),
Expand Down Expand Up @@ -932,7 +932,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
// Verify all parts are counted in inflights
assert.strictEqual(scuba.getInflightsForBucket(bucket), totalSize);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(totalSize));
return next();
},
next => {
Expand All @@ -954,7 +954,7 @@ function multiObjectDelete(bucket, keys, size, callback) {
next => {
// Verify inflights reduced by dropped part
const expectedInflights = usedParts * partSize;
assert.strictEqual(scuba.getInflightsForBucket(bucket), expectedInflights);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(expectedInflights));
return next();
},
next => deleteObject(bucket, key, usedParts * partSize, next),
Expand Down Expand Up @@ -1002,14 +1002,14 @@ function multiObjectDelete(bucket, keys, size, callback) {
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
// Verify all parts are counted in inflights
assert.strictEqual(scuba.getInflightsForBucket(bucket), totalSize);
assert.strictEqual(scuba.getInflightsForBucket(bucket), BigInt(totalSize));
return next();
},
next => abortMPU(bucket, key, uploadId, totalSize, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
// Verify inflights reduced to zero after abort
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0n);
return next();
},
next => deleteBucket(bucket, next),
Expand Down
Loading
Loading