Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commit 954acb1

Browse files
authored
support zstd compression (#1950)
1 parent 7d71dec commit 954acb1

File tree

6 files changed

+227
-0
lines changed

6 files changed

+227
-0
lines changed

‎LICENSES/LICENSE.zstd-jni.txt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
Zstd-jni: JNI bindings to Zstd Library
2+
3+
Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
4+
5+
BSD License
6+
7+
Redistribution and use in source and binary forms, with or without modification,
8+
are permitted provided that the following conditions are met:
9+
10+
* Redistributions of source code must retain the above copyright notice, this
11+
list of conditions and the following disclaimer.
12+
13+
* Redistributions in binary form must reproduce the above copyright notice, this
14+
list of conditions and the following disclaimer in the documentation and/or
15+
other materials provided with the distribution.
16+
17+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
21+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
24+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import io.netty.buffer.ByteBuf;
1919
import io.netty.buffer.Unpooled;
2020
import io.netty.handler.codec.compression.Brotli;
21+
import io.netty.handler.codec.compression.Zstd;
2122
import io.netty.handler.codec.http.DefaultFullHttpRequest;
2223
import io.netty.handler.codec.http.DefaultHttpRequest;
2324
import io.netty.handler.codec.http.HttpHeaderValues;
@@ -67,6 +68,7 @@
6768
import static org.asynchttpclient.util.HttpUtils.ACCEPT_ALL_HEADER_VALUE;
6869
import static org.asynchttpclient.util.HttpUtils.GZIP_DEFLATE;
6970
import static org.asynchttpclient.util.HttpUtils.filterOutBrotliFromAcceptEncoding;
71+
import static org.asynchttpclient.util.HttpUtils.filterOutZstdFromAcceptEncoding;
7072
import static org.asynchttpclient.util.HttpUtils.hostHeader;
7173
import static org.asynchttpclient.util.HttpUtils.originHeader;
7274
import static org.asynchttpclient.util.HttpUtils.urlEncodeFormParams;
@@ -182,13 +184,21 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque
182184
// For manual decompression by user, any encoding may suite, so leave untouched
183185
headers.set(ACCEPT_ENCODING, filterOutBrotliFromAcceptEncoding(userDefinedAcceptEncoding));
184186
}
187+
if (!Zstd.isAvailable()) {
188+
// zstd is not available.
189+
// For manual decompression by user, any encoding may suit, so leave untouched
190+
headers.set(ACCEPT_ENCODING, filterOutZstdFromAcceptEncoding(userDefinedAcceptEncoding));
191+
}
185192
}
186193
} else if (config.isCompressionEnforced()) {
187194
// Add Accept Encoding header if compression is enforced
188195
headers.set(ACCEPT_ENCODING, GZIP_DEFLATE);
189196
if (Brotli.isAvailable()) {
190197
headers.add(ACCEPT_ENCODING, HttpHeaderValues.BR);
191198
}
199+
if (Zstd.isAvailable()) {
200+
headers.add(ACCEPT_ENCODING, HttpHeaderValues.ZSTD);
201+
}
192202
}
193203
}
194204

‎client/src/main/java/org/asynchttpclient/util/HttpUtils.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ public final class HttpUtils {
4040
private static final String CONTENT_TYPE_CHARSET_ATTRIBUTE = "charset=";
4141
private static final String CONTENT_TYPE_BOUNDARY_ATTRIBUTE = "boundary=";
4242
private static final String BROTLY_ACCEPT_ENCODING_SUFFIX = ", br";
43+
private static final String ZSTD_ACCEPT_ENCODING_SUFFIX = ", zstd";
4344

4445
private HttpUtils() {
4546
// Prevent outside initialization
@@ -173,4 +174,12 @@ public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncodi
173174
}
174175
return acceptEncoding;
175176
}
177+
178+
public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding) {
179+
// we don't support zstd ATM
180+
if (acceptEncoding.endsWith(ZSTD_ACCEPT_ENCODING_SUFFIX)) {
181+
return acceptEncoding.subSequence(0, acceptEncoding.length() - ZSTD_ACCEPT_ENCODING_SUFFIX.length());
182+
}
183+
return acceptEncoding;
184+
}
176185
}
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Copyright (c) 2015-2024 AsyncHttpClient Project. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.asynchttpclient;
17+
18+
import com.aayushatharva.brotli4j.encoder.BrotliOutputStream;
19+
import com.aayushatharva.brotli4j.encoder.Encoder;
20+
import com.sun.net.httpserver.Headers;
21+
import com.sun.net.httpserver.HttpExchange;
22+
import com.sun.net.httpserver.HttpHandler;
23+
import com.sun.net.httpserver.HttpServer;
24+
import java.io.IOException;
25+
import java.io.OutputStream;
26+
import java.net.InetSocketAddress;
27+
import java.nio.charset.StandardCharsets;
28+
import java.util.Arrays;
29+
import java.util.List;
30+
import java.util.stream.Collectors;
31+
import java.util.zip.GZIPOutputStream;
32+
import org.junit.jupiter.api.AfterAll;
33+
import org.junit.jupiter.api.BeforeAll;
34+
import org.junit.jupiter.api.Test;
35+
import com.github.luben.zstd.Zstd;
36+
37+
import static org.junit.jupiter.api.Assertions.assertEquals;
38+
39+
public class AutomaticDecompressionTest {
40+
private static final String UNCOMPRESSED_PAYLOAD = "a".repeat(500);
41+
42+
private static HttpServer HTTP_SERVER;
43+
44+
private static AsyncHttpClient createClient() {
45+
AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder()
46+
.setEnableAutomaticDecompression(true)
47+
.setCompressionEnforced(true)
48+
.build();
49+
return new DefaultAsyncHttpClient(config);
50+
}
51+
52+
@BeforeAll
53+
static void setupServer() throws Exception {
54+
HTTP_SERVER = HttpServer.create(new InetSocketAddress(0), 0);
55+
56+
HTTP_SERVER.createContext("/br").setHandler(new HttpHandler() {
57+
@Override
58+
public void handle(HttpExchange exchange)
59+
throws IOException {
60+
validateAcceptEncodingHeader(exchange);
61+
exchange.getResponseHeaders().set("Content-Encoding", "br");
62+
exchange.sendResponseHeaders(200, 0);
63+
OutputStream out = exchange.getResponseBody();
64+
Encoder.Parameters params = new Encoder.Parameters();
65+
BrotliOutputStream brotliOutputStream = new BrotliOutputStream(out, params);
66+
brotliOutputStream.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8));
67+
brotliOutputStream.flush();
68+
brotliOutputStream.close();
69+
}
70+
});
71+
72+
HTTP_SERVER.createContext("/zstd").setHandler(new HttpHandler() {
73+
@Override
74+
public void handle(HttpExchange exchange)
75+
throws IOException {
76+
validateAcceptEncodingHeader(exchange);
77+
exchange.getResponseHeaders().set("Content-Encoding", "zstd");
78+
byte[] compressedData = new byte[UNCOMPRESSED_PAYLOAD.length()];
79+
long n = Zstd.compress(compressedData, UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8), 2, true);
80+
exchange.sendResponseHeaders(200, n);
81+
OutputStream out = exchange.getResponseBody();
82+
out.write(compressedData, 0, (int) n);
83+
out.flush();
84+
out.close();
85+
}
86+
});
87+
88+
HTTP_SERVER.createContext("/gzip").setHandler(new HttpHandler() {
89+
@Override
90+
public void handle(HttpExchange exchange)
91+
throws IOException {
92+
validateAcceptEncodingHeader(exchange);
93+
exchange.getResponseHeaders().set("Content-Encoding", "gzip");
94+
exchange.sendResponseHeaders(200, 0);
95+
OutputStream out = exchange.getResponseBody();
96+
GZIPOutputStream gzip = new GZIPOutputStream(out);
97+
gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8));
98+
gzip.flush();
99+
gzip.close();
100+
}
101+
});
102+
103+
HTTP_SERVER.start();
104+
}
105+
106+
private static void validateAcceptEncodingHeader(HttpExchange exchange) {
107+
Headers requestHeaders = exchange.getRequestHeaders();
108+
List<String> acceptEncodingList = requestHeaders.get("Accept-Encoding")
109+
.stream()
110+
.flatMap(x -> Arrays.asList(x.split(",")).stream())
111+
.collect(Collectors.toList());
112+
assertEquals(List.of("gzip", "deflate", "br", "zstd"), acceptEncodingList);
113+
}
114+
115+
@AfterAll
116+
static void stopServer() {
117+
if (HTTP_SERVER != null) {
118+
HTTP_SERVER.stop(0);
119+
}
120+
}
121+
122+
@Test
123+
void zstd() throws Throwable {
124+
io.netty.handler.codec.compression.Zstd.ensureAvailability();
125+
try (AsyncHttpClient client = createClient()) {
126+
Request request = new RequestBuilder("GET")
127+
.setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/zstd")
128+
.build();
129+
Response response = client.executeRequest(request).get();
130+
assertEquals(200, response.getStatusCode());
131+
assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
132+
}
133+
}
134+
135+
@Test
136+
void brotli() throws Throwable {
137+
io.netty.handler.codec.compression.Brotli.ensureAvailability();
138+
try (AsyncHttpClient client = createClient()) {
139+
Request request = new RequestBuilder("GET")
140+
.setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/br")
141+
.build();
142+
Response response = client.executeRequest(request).get();
143+
assertEquals(200, response.getStatusCode());
144+
assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
145+
}
146+
}
147+
148+
@Test
149+
void gzip() throws Throwable {
150+
try (AsyncHttpClient client = createClient()) {
151+
Request request = new RequestBuilder("GET")
152+
.setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/gzip")
153+
.build();
154+
Response response = client.executeRequest(request).get();
155+
assertEquals(200, response.getStatusCode());
156+
assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
157+
}
158+
}
159+
160+
161+
}

‎client/src/test/java/org/asynchttpclient/netty/NettyTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import io.netty.channel.epoll.Epoll;
44
import io.netty.channel.kqueue.KQueue;
55
import io.netty.handler.codec.compression.Brotli;
6+
import io.netty.handler.codec.compression.Zstd;
67
import io.netty.incubator.channel.uring.IOUring;
78
import org.junit.jupiter.api.Test;
89
import org.junit.jupiter.api.condition.EnabledOnOs;
@@ -40,4 +41,16 @@ public void brotliIsAvailableOnLinux() {
4041
public void brotliIsAvailableOnMac() {
4142
assertTrue(Brotli.isAvailable());
4243
}
44+
45+
@Test
46+
@EnabledOnOs(value = OS.LINUX)
47+
public void zstdIsAvailableOnLinux() {
48+
assertTrue(Zstd.isAvailable());
49+
}
50+
51+
@Test
52+
@EnabledOnOs(value = OS.MAC)
53+
public void zstdIsAvailableOnMac() {
54+
assertTrue(Zstd.isAvailable());
55+
}
4356
}

‎pom.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<netty.iouring>0.0.25.Final</netty.iouring>
6363
<brotli4j.version>1.16.0</brotli4j.version>
6464
<slf4j.version>2.0.13</slf4j.version>
65+
<zstd-jni.version>1.5.6-3</zstd-jni.version>
6566
<activation.version>2.0.1</activation.version>
6667
<logback.version>1.4.11</logback.version>
6768
<jetbrains-annotations.version>24.0.1</jetbrains-annotations.version>
@@ -224,6 +225,13 @@
224225
<optional>true</optional>
225226
</dependency>
226227

228+
<dependency>
229+
<groupId>com.github.luben</groupId>
230+
<artifactId>zstd-jni</artifactId>
231+
<version>${zstd-jni.version}</version>
232+
<optional>true</optional>
233+
</dependency>
234+
227235
<dependency>
228236
<groupId>com.aayushatharva.brotli4j</groupId>
229237
<artifactId>brotli4j</artifactId>

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /