From bbbeed7a13b9b2166ca560ce5f107e486fa6bb93 Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 23 Feb 2023 11:53:08 +0100 Subject: [PATCH 1/3] #9408: restored HugeResourceTest: - fixed double-release bug in MultiPartFormData - used ByteBufferPool.NonPooling pool to work around #9311 Signed-off-by: Ludovic Orban --- .../eclipse/jetty/http/MultiPartFormData.java | 1 + .../jetty/ee10/webapp/HugeResourceTest.java | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormData.java b/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormData.java index 9e03566e2bb2..c4611d495a90 100644 --- a/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormData.java +++ b/jetty-core/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormData.java @@ -469,6 +469,7 @@ public void onPartContent(Content.Chunk chunk) } partChunks.forEach(Content.Chunk::release); + partChunks.clear(); return; } } diff --git a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java index d80dc300b04f..d0903452a730 100644 --- a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java +++ b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java @@ -52,6 +52,7 @@ import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.http.MultiPart; +import org.eclipse.jetty.io.ByteBufferPool; import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.io.Content; import org.eclipse.jetty.server.HttpConfiguration; @@ -70,7 +71,6 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -83,7 +83,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @Tag("large-disk-resource") -@Disabled // TODO investigate public class HugeResourceTest { private static final long KB = 1024; @@ -114,11 +113,10 @@ public static void prepareStaticFiles() throws IOException String.format("FileStore %s of %s needs at least 30GB of free space for this test (only had %,.2fGB)", baseFileStore, staticBase, (double)(baseFileStore.getUnallocatedSpace() / GB))); - makeStaticFile(staticBase.resolve("test-1M.dat"), MB); - makeStaticFile(staticBase.resolve("test-100M.dat"), 100 * MB); - makeStaticFile(staticBase.resolve("test-1G.dat"), GB); - // makeStaticFile(staticBase.resolve("test-4G.dat"), 4 * GB); - // makeStaticFile(staticBase.resolve("test-10G.dat"), 10 * GB); + makeStaticFile(staticBase.resolve("test-1m.dat"), MB); + makeStaticFile(staticBase.resolve("test-1g.dat"), GB); + // makeStaticFile(staticBase.resolve("test-4g.dat"), 4 * GB); + // makeStaticFile(staticBase.resolve("test-10g.dat"), 10 * GB); outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-outputdir"); FS.ensureEmpty(outputDir); @@ -131,11 +129,10 @@ public static Stream staticFiles() { ArrayList ret = new ArrayList<>(); - ret.add(Arguments.of("test-1M.dat", MB)); - ret.add(Arguments.of("test-100M.dat", 100 * MB)); - ret.add(Arguments.of("test-1G.dat", GB)); - // ret.add(Arguments.of("test-4G.dat", 4 * GB)); - // ret.add(Arguments.of("test-1Gg.dat", 10 * GB)); + ret.add(Arguments.of("test-1m.dat", MB)); + ret.add(Arguments.of("test-1g.dat", GB)); + // ret.add(Arguments.of("test-4g.dat", 4 * GB)); + // ret.add(Arguments.of("test-10g.dat", 10 * GB)); return ret.stream(); } @@ -202,7 +199,8 @@ public void startServer() throws Exception QueuedThreadPool serverThreads = new QueuedThreadPool(); serverThreads.setDetailedDump(true); serverThreads.setName("server"); - server = new Server(serverThreads); + // TODO: Use normal pool when a fix for https://github.com/eclipse/jetty.project/issues/9311 is merged. + server = new Server(serverThreads, null, new ByteBufferPool.NonPooling()); httpConfig = new HttpConfiguration(); ServerConnector connector = new ServerConnector(server, 1, 1, new HttpConnectionFactory(httpConfig)); connector.setPort(0); @@ -249,6 +247,8 @@ public void startClient() throws Exception connector.setSelectors(1); connector.setExecutor(clientThreads); client = new HttpClient(new HttpClientTransportOverHTTP(connector)); + // TODO: Use normal pool when a fix for https://github.com/eclipse/jetty.project/issues/9311 is merged. + client.setByteBufferPool(new ByteBufferPool.NonPooling()); client.start(); } @@ -603,8 +603,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S IO.copy(inputStream, byteCounting); out.printf("part[%s].inputStream.length=%d%n", part.getName(), byteCounting.getCount()); } - catch (IOException e) + catch (Throwable e) { + resp.setStatus(500); e.printStackTrace(out); } }); From 7e8b5771e5fd4b8fb332b3907be5a9f15329101c Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Thu, 23 Feb 2023 14:34:07 +0100 Subject: [PATCH 2/3] #9408: handle review comments Signed-off-by: Ludovic Orban --- .../org/eclipse/jetty/ee10/webapp/HugeResourceTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java index d0903452a730..2bb3bc7780cf 100644 --- a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java +++ b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java @@ -115,6 +115,8 @@ public static void prepareStaticFiles() throws IOException makeStaticFile(staticBase.resolve("test-1m.dat"), MB); makeStaticFile(staticBase.resolve("test-1g.dat"), GB); + // The reason for testing 4GB and 10GB were because of various filesystem handling bugs + // we had in our code (the 2GB threshold and the 8GB threshold in various FileSystem APIs). // makeStaticFile(staticBase.resolve("test-4g.dat"), 4 * GB); // makeStaticFile(staticBase.resolve("test-10g.dat"), 10 * GB); @@ -603,10 +605,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S IO.copy(inputStream, byteCounting); out.printf("part[%s].inputStream.length=%d%n", part.getName(), byteCounting.getCount()); } - catch (Throwable e) + catch (Throwable x) { - resp.setStatus(500); - e.printStackTrace(out); + throw new AssertionError(x); } }); } From 98822f9e4a7f08387e267a9da22470eb258c836a Mon Sep 17 00:00:00 2001 From: Ludovic Orban Date: Fri, 24 Feb 2023 10:49:21 +0100 Subject: [PATCH 3/3] #9408: re-enable 4G and 10G tests Signed-off-by: Ludovic Orban --- .../eclipse/jetty/ee10/webapp/HugeResourceTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java index 2bb3bc7780cf..958a53bd0c75 100644 --- a/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java +++ b/jetty-ee10/jetty-ee10-webapp/src/test/java/org/eclipse/jetty/ee10/webapp/HugeResourceTest.java @@ -117,8 +117,8 @@ public static void prepareStaticFiles() throws IOException makeStaticFile(staticBase.resolve("test-1g.dat"), GB); // The reason for testing 4GB and 10GB were because of various filesystem handling bugs // we had in our code (the 2GB threshold and the 8GB threshold in various FileSystem APIs). - // makeStaticFile(staticBase.resolve("test-4g.dat"), 4 * GB); - // makeStaticFile(staticBase.resolve("test-10g.dat"), 10 * GB); + makeStaticFile(staticBase.resolve("test-4g.dat"), 4 * GB); + makeStaticFile(staticBase.resolve("test-10g.dat"), 10 * GB); outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-outputdir"); FS.ensureEmpty(outputDir); @@ -133,8 +133,8 @@ public static Stream staticFiles() ret.add(Arguments.of("test-1m.dat", MB)); ret.add(Arguments.of("test-1g.dat", GB)); - // ret.add(Arguments.of("test-4g.dat", 4 * GB)); - // ret.add(Arguments.of("test-10g.dat", 10 * GB)); + ret.add(Arguments.of("test-4g.dat", 4 * GB)); + ret.add(Arguments.of("test-10g.dat", 10 * GB)); return ret.stream(); } @@ -430,7 +430,7 @@ public void demand(Runnable demandCallback) Thread.sleep(100); stalled.set(false); demand.get().run(); - assertTrue(complete.await(30, TimeUnit.SECONDS)); + assertTrue(complete.await(60, TimeUnit.SECONDS)); Response response = responseRef.get(); assertThat("HTTP Response Code", response.getStatus(), is(200));