Skip to content

Commit

Permalink
Revert "Merge pull request webbukkit#4030 from ChimneySwift/generic-s3"
Browse files Browse the repository at this point in the history 
This reverts commit b0e56d3, reversing
changes made to 20700c2.

The reason for reverting this is that it breaks zoom tile generation for Amazon's s3 in an insidious way.
No errors are reported, and the zoom tile thread infinitely iterates over calling the s3 API,
throwing an error when s3 replies and then swallowing the error and immediately trying again.
This generates about 2-3MB/s of useless traffic to s3 and the immense amount of API calls made incurs a significant cost.
  • Loading branch information
@stewi1014
stewi1014 committed Oct 30, 2024
1 parent 080a84f commit a4a924f
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 119 deletions.
24 changes: 10 additions & 14 deletions DynmapCore/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,11 @@ dependencies {
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
implementation 'org.postgresql:postgresql:42.2.18'
implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT'
implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT'
implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT'
implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT'
implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT'
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1'
implementation 'com.sun.xml.bind:jaxb-impl:3.0.0'
implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
}

processResources {
Expand Down Expand Up @@ -59,13 +57,11 @@ shadowJar {
include(dependency('org.eclipse.jetty::'))
include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
include(dependency('org.postgresql:postgresql:'))
include(dependency('io.github.linktosriram.s3lite:core:'))
include(dependency('io.github.linktosriram.s3lite:api:'))
include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:'))
include(dependency('io.github.linktosriram.s3lite:http-client-spi:'))
include(dependency('io.github.linktosriram.s3lite:util:'))
include(dependency('jakarta.xml.bind::'))
include(dependency('com.sun.xml.bind::'))
include(dependency('io.github.linktosriram:s3-lite-core:'))
include(dependency('io.github.linktosriram:s3-lite-api:'))
include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
include(dependency('io.github.linktosriram:s3-lite-util:'))
include(dependency(':DynmapCoreAPI'))
exclude("META-INF/maven/**")
exclude("META-INF/services/**")
Expand Down
231 changes: 128 additions & 103 deletions DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,32 @@
package org.dynmap.storage.aws_s3;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.dynmap.DynmapCore;
import org.dynmap.DynmapWorld;
import org.dynmap.Log;
import org.dynmap.MapType;
import org.dynmap.MapType.ImageEncoding;
import org.dynmap.MapType.ImageVariant;
import org.dynmap.PlayerFaces.FaceType;
import org.dynmap.WebAuthManager;
import org.dynmap.storage.MapStorage;
import org.dynmap.storage.MapStorageTile;
import org.dynmap.storage.MapStorageTileEnumCB;
import org.dynmap.storage.MapStorageBaseTileEnumCB;
import org.dynmap.storage.MapStorageTileSearchEndCB;
import org.dynmap.utils.BufferInputStream;
import org.dynmap.utils.BufferOutputStream;

import io.github.linktosriram.s3lite.api.client.S3Client;
import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
import io.github.linktosriram.s3lite.api.exception.S3Exception;
Expand Down Expand Up @@ -118,19 +145,20 @@ public TileRead read() {

@Override
public boolean write(long hash, BufferOutputStream encImage, long timestamp) {
boolean done = false;
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(req);
} else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
boolean done = false;
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(req);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} catch (StorageShutdownException x) {
Expand Down Expand Up @@ -214,7 +242,7 @@ public String toString() {
}

private String bucketname;
private Region region;
private String region;
private String access_key_id;
private String secret_access_key;
private String prefix;
Expand All @@ -241,22 +269,12 @@ public boolean init(DynmapCore core) {
}
// Get our settings
bucketname = core.configuration.getString("storage/bucketname", "dynmap");
region = core.configuration.getString("storage/region", "us-east-1");
access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
prefix = core.configuration.getString("storage/prefix", "");

// Either use a custom region, or one of the default AWS regions
String region_name = core.configuration.getString("storage/region", "us-east-1");
String region_endpoint = core.configuration.getString("storage/override_endpoint", "");

if (region_endpoint.length() > 0) {
region = Region.of(region_name, URI.create(region_endpoint));
} else {
region = Region.fromString(region_name);
}

if ((prefix.length() > 0) && (prefix.charAt(prefix.length() - 1) != '/')) {
prefix += '/';
if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) {
prefix += '/';
}
// Now creste the access client for the S3 service
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region);
Expand Down Expand Up @@ -503,20 +521,21 @@ public void purgeMapTiles(DynmapWorld world, MapType map) {

@Override
public boolean setPlayerFaceImage(String playername, FaceType facetype,
BufferOutputStream encImage) {
boolean done = false;
String baseKey = prefix + "tiles/faces/" + facetype.id + "/" + playername + ".png";
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
BufferOutputStream encImage) {
boolean done = false;
String baseKey = prefix + "tiles/faces/" + facetype.id + "/" + playername + ".png";
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} catch (StorageShutdownException x) {
Expand Down Expand Up @@ -556,19 +575,20 @@ public boolean hasPlayerFaceImage(String playername, FaceType facetype) {

@Override
public boolean setMarkerImage(String markerid, BufferOutputStream encImage) {
boolean done = false;
String baseKey = prefix + "tiles/_markers_/" + markerid + ".png";
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
} else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
}
done = true;
boolean done = false;
String baseKey = prefix + "tiles/_markers_/" + markerid + ".png";
S3Client s3 = null;
try {
s3 = getConnection();
if (encImage == null) { // Delete?
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
}
else {
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
}
done = true;
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} catch (StorageShutdownException x) {
Expand Down Expand Up @@ -688,51 +708,56 @@ public boolean needsStaticWebFiles() {
* @return true if successful
*/
public boolean setStaticWebFile(String fileid, BufferOutputStream content) {

boolean done = false;
String baseKey = prefix + fileid;
S3Client s3 = null;
try {
s3 = getConnection();
byte[] cacheval = standalone_cache.get(fileid);

if (content == null) { // Delete?
if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
return true;
}
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
standalone_cache.put(fileid, new byte[0]); // Mark in cache
} else {
byte[] digest = content.buf;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(content.buf);
digest = md.digest();
} catch (NoSuchAlgorithmException nsax) {

}
// If cached and same, just return
if (Arrays.equals(digest, cacheval)) {
return true;
}
String ct = "text/plain";
if (fileid.endsWith(".json")) {
ct = "application/json";
} else if (fileid.endsWith(".php")) {
ct = "application/x-httpd-php";
} else if (fileid.endsWith(".html")) {
ct = "text/html";
} else if (fileid.endsWith(".css")) {
ct = "text/css";
} else if (fileid.endsWith(".js")) {
ct = "application/x-javascript";
}
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
s3.putObject(req, RequestBody.fromBytes(content.buf));
standalone_cache.put(fileid, digest);
}
done = true;

boolean done = false;
String baseKey = prefix + fileid;
S3Client s3 = null;
try {
s3 = getConnection();
byte[] cacheval = standalone_cache.get(fileid);

if (content == null) { // Delete?
if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
return true;
}
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
s3.deleteObject(delreq);
standalone_cache.put(fileid, new byte[0]); // Mark in cache
}
else {
byte[] digest = content.buf;
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(content.buf);
digest = md.digest();
} catch (NoSuchAlgorithmException nsax) {

}
// If cached and same, just return
if (Arrays.equals(digest, cacheval)) {
return true;
}
String ct = "text/plain";
if (fileid.endsWith(".json")) {
ct = "application/json";
}
else if (fileid.endsWith(".php")) {
ct = "application/x-httpd-php";
}
else if (fileid.endsWith(".html")) {
ct = "text/html";
}
else if (fileid.endsWith(".css")) {
ct = "text/css";
}
else if (fileid.endsWith(".js")) {
ct = "application/x-javascript";
}
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
standalone_cache.put(fileid, digest);
}
done = true;
} catch (S3Exception x) {
Log.severe("AWS Exception", x);
} catch (StorageShutdownException x) {
Expand All @@ -757,10 +782,10 @@ private S3Client getConnection() throws S3Exception, StorageShutdownException {
if (c == null) {
if (cpoolCount < POOLSIZE) { // Still more we can have
c = new DefaultS3ClientBuilder()
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
.region(region)
.httpClient(URLConnectionSdkHttpClient.create())
.build();
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
.region(Region.fromString(region))
.httpClient(URLConnectionSdkHttpClient.create())
.build();
if (c == null) {
Log.severe("Error creating S3 access client");
return null;
Expand Down
1 change: 0 additions & 1 deletion fabric-1.21.1/src/main/resources/configuration.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ storage:
#aws_access_key_id: "<aws-access-key-id>"
#aws_secret_access_key: "<aws-secret-access-key>"
#prefix: ""
#override_endpoint: ""

components:
- class: org.dynmap.ClientConfigurationComponent
Expand Down
1 change: 0 additions & 1 deletion forge-1.21/src/main/resources/configuration.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ storage:
#aws_access_key_id: "<aws-access-key-id>"
#aws_secret_access_key: "<aws-secret-access-key>"
#prefix: ""
#override_endpoint: ""

components:
- class: org.dynmap.ClientConfigurationComponent
Expand Down

0 comments on commit a4a924f

Please sign in to comment.