Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 471ea9b

Browse files
committed
Implemented new mechanism how to automatically expand internal buffer size. This fixes bug #241 and #124 issues.
1 parent 4504c2e commit 471ea9b

File tree

3 files changed

+72
-26
lines changed

3 files changed

+72
-26
lines changed

‎src/main/java/com/jsoniter/IterImplForStreaming.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -276,12 +276,14 @@ public final static boolean loadMore(JsonIterator iter) throws IOException {
276276
private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOException {
277277
int offset = iter.tail - iter.skipStartedAt;
278278
byte[] srcBuffer = iter.buf;
279-
// Double the size of internal buffer
280-
// TODO: Fix NegativeArraySizeException that happens if source stream doesnt return as much
281-
// of output as was requested i.e. when n < iter.buf.length - offset. Anyhow doubling the buffer
282-
// size seems to be pretty dangerous idea and should be either disabled or solved safely.
283-
if (iter.skipStartedAt == 0 || iter.skipStartedAt < iter.tail / 2) {
284-
iter.buf = new byte[iter.buf.length * 2];
279+
// Check there is no unused buffer capacity
280+
if (iter.buf.length - iter.tail == 0) {
281+
// If auto expand buffer enabled, then create larger buffer
282+
if (iter.autoExpandBufferStep > 0) {
283+
iter.buf = new byte[iter.buf.length + iter.autoExpandBufferStep];
284+
} else {
285+
throw iter.reportError("loadMore", "buffer is full and autoexpansion is disabled");
286+
}
285287
}
286288
System.arraycopy(srcBuffer, iter.skipStartedAt, iter.buf, 0, offset);
287289
int n = iter.in.read(iter.buf, offset, iter.buf.length - offset);

‎src/main/java/com/jsoniter/JsonIterator.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ public class JsonIterator implements Closeable {
2121
final static ValueType[] valueTypes = new ValueType[256];
2222
InputStream in;
2323
byte[] buf;
24+
// Whenever buf is not large enough new one is created with size of
25+
// buf.length + autoExpandBufferStep. Set to < 1 to disable auto expanding.
26+
int autoExpandBufferStep;
2427
int head;
2528
int tail;
2629
int skipStartedAt = -1; // skip should keep bytes starting at this pos
@@ -60,13 +63,22 @@ private JsonIterator(InputStream in, byte[] buf, int head, int tail) {
6063
this.tail = tail;
6164
}
6265

66+
private JsonIterator(InputStream in, byte[] buf, int autoExpandBufferStep) {
67+
this(in, buf, 0, 0);
68+
this.autoExpandBufferStep = autoExpandBufferStep;
69+
}
70+
6371
public JsonIterator() {
6472
this(null, new byte[0], 0, 0);
6573
}
6674

6775
public static JsonIterator parse(InputStream in, int bufSize) {
76+
return parse(in, bufSize, bufSize);
77+
}
78+
79+
public static JsonIterator parse(InputStream in, int bufSize, int autoExpandBufferStep) {
6880
enableStreamingSupport();
69-
return new JsonIterator(in, new byte[bufSize], 0, 0);
81+
return new JsonIterator(in, new byte[bufSize], autoExpandBufferStep);
7082
}
7183

7284
public static JsonIterator parse(byte[] buf) {
Lines changed: 51 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package com.jsoniter;
22

33
import com.jsoniter.any.Any;
4+
import com.jsoniter.spi.JsonException;
45
import java.io.IOException;
56
import java.io.InputStream;
67
import junit.framework.TestCase;
78
import org.junit.experimental.categories.Category;
9+
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
810

911
public class IterImplForStreamingTest extends TestCase {
1012

@@ -18,34 +20,64 @@ public void testReadMaxDouble() throws Exception {
1820

1921
@Category(StreamingCategory.class)
2022
public void testLoadMore() throws IOException {
21-
final String originalContent = "1234";
23+
final String originalContent = "1234567890";
2224
final byte[] src = ("{\"a\":\"" + originalContent + "\"}").getBytes();
23-
InputStream slowStream = new InputStream() {
25+
26+
int initialBufferSize;
27+
Any parsedString;
28+
// Case #1: Data fits into initial buffer, autoresizing on
29+
// Input must definitely fit into such large buffer
30+
initialBufferSize = src.length * 2;
31+
JsonIterator jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 512);
32+
jsonIterator.readObject();
33+
parsedString = jsonIterator.readAny();
34+
assertEquals(originalContent, parsedString.toString());
35+
// Check buffer was not expanded
36+
assertEquals(initialBufferSize, jsonIterator.buf.length);
37+
38+
// Case #2: Data does fit into initial buffer, autoresizing off
39+
initialBufferSize = originalContent.length() / 2;
40+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 0);
41+
jsonIterator.readObject();
42+
try {
43+
jsonIterator.readAny();
44+
fail("Expect to fail because buffer is too small.");
45+
} catch (JsonException e) {
46+
if (!e.getMessage().startsWith("loadMore")) {
47+
throw e;
48+
}
49+
}
50+
// Check buffer was not expanded
51+
assertEquals(initialBufferSize, jsonIterator.buf.length);
52+
53+
// Case #3: Data does fit into initial buffer, autoresizing on
54+
initialBufferSize = originalContent.length() / 2;
55+
int autoExpandBufferStep = initialBufferSize * 3;
56+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, autoExpandBufferStep);
57+
jsonIterator.readObject();
58+
parsedString = jsonIterator.readAny();
59+
assertEquals(originalContent, parsedString.toString());
60+
// Check buffer was expanded exactly once
61+
assertEquals(initialBufferSize + autoExpandBufferStep, jsonIterator.buf.length);
62+
}
63+
64+
private static InputStream getSluggishInputStream(final byte[] src) {
65+
return new InputStream() {
2466
int position = 0;
25-
boolean pretendEmptyNextRead = false;
2667

2768
@Override
2869
public int read() throws IOException {
70+
throw new NotImplementedException();
71+
}
72+
73+
@Override
74+
public int read(byte[] b, int off, int len) throws IOException {
2975
if (position < src.length) {
30-
if (pretendEmptyNextRead) {
31-
pretendEmptyNextRead = false;
32-
return -1;
33-
} else {
34-
pretendEmptyNextRead = true;
35-
return src[position++];
36-
}
76+
b[off] = src[position++];
77+
return 1;
3778
}
3879
return -1;
3980
}
4081
};
41-
42-
// Input must definitely fit into such large buffer
43-
final int initialBufferSize = src.length * 2;
44-
JsonIterator jsonIterator = JsonIterator.parse(slowStream, initialBufferSize);
45-
jsonIterator.readObject();
46-
Any parsedString = jsonIterator.readAny();
47-
assertEquals(originalContent, parsedString.toString());
48-
// Check buffer was not expanded prematurely
49-
assertEquals(initialBufferSize, jsonIterator.buf.length);
5082
}
5183
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /