Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 9928ac5

Browse files
Merge pull request #243 from svobol13/master
Fixed leaky buffer when streaming.
2 parents 04bde57 + ba73426 commit 9928ac5

File tree

6 files changed

+137
-22
lines changed

6 files changed

+137
-22
lines changed

‎src/main/java/com/jsoniter/IterImpl.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
5959
case '[': // If open symbol, increase level
6060
level++;
6161
break;
62-
case ']': // If close symbol, increase level
62+
case ']': // If close symbol, decrease level
6363
level--;
6464

6565
// If we have returned to the original level, we're done
@@ -85,7 +85,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
8585
case '{': // If open symbol, increase level
8686
level++;
8787
break;
88-
case '}': // If close symbol, increase level
88+
case '}': // If close symbol, decrease level
8989
level--;
9090

9191
// If we have returned to the original level, we're done

‎src/main/java/com/jsoniter/IterImplForStreaming.java

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
7171
case '[': // If open symbol, increase level
7272
level++;
7373
break;
74-
case ']': // If close symbol, increase level
74+
case ']': // If close symbol, decrease level
7575
level--;
7676

7777
// If we have returned to the original level, we're done
@@ -101,7 +101,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
101101
case '{': // If open symbol, increase level
102102
level++;
103103
break;
104-
case '}': // If close symbol, increase level
104+
case '}': // If close symbol, decrease level
105105
level--;
106106

107107
// If we have returned to the original level, we're done
@@ -147,7 +147,8 @@ final static void skipString(JsonIterator iter) throws IOException {
147147
throw iter.reportError("skipString", "incomplete string");
148148
}
149149
if (escaped) {
150-
iter.head = 1; // skip the first char as last char is \
150+
// TODO add unit test to prove/verify bug
151+
iter.head += 1; // skip the first char as last char is \
151152
}
152153
} else {
153154
iter.head = end;
@@ -274,19 +275,19 @@ public final static boolean loadMore(JsonIterator iter) throws IOException {
274275
}
275276

276277
private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOException {
277-
int n;
278-
intoffset;
279-
if (iter.skipStartedAt == 0 || iter.skipStartedAt < iter.tail / 2) {
280-
byte[] newBuf = newbyte[iter.buf.length * 2];
281-
offset = iter.tail - iter.skipStartedAt;
282-
System.arraycopy(iter.buf, iter.skipStartedAt, newBuf, 0, offset);
283-
iter.buf = newBuf;
284-
n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
285-
} else {
286-
offset = iter.tail - iter.skipStartedAt;
287-
System.arraycopy(iter.buf, iter.skipStartedAt, iter.buf, 0, offset);
288-
n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
289-
}
278+
int offset = iter.tail - iter.skipStartedAt;
279+
byte[] srcBuffer = iter.buf;
280+
// Check there is no unused buffer capacity
281+
if ((getUnusedBufferByteCount(iter)) == 0) {
282+
// If auto expand buffer enabled, then create larger buffer
283+
if(iter.autoExpandBufferStep > 0) {
284+
iter.buf = newbyte[iter.buf.length + iter.autoExpandBufferStep];
285+
} else {
286+
throwiter.reportError("loadMore", String.format("buffer is full and autoexpansion is disabled. tail: [%s] skipStartedAt: [%s]", iter.tail, iter.skipStartedAt));
287+
}
288+
}
289+
System.arraycopy(srcBuffer, iter.skipStartedAt, iter.buf, 0, offset);
290+
intn = iter.in.read(iter.buf, offset, iter.buf.length - offset);
290291
iter.skipStartedAt = 0;
291292
if (n < 1) {
292293
if (n == -1) {
@@ -301,6 +302,11 @@ private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOExce
301302
return true;
302303
}
303304

305+
private static int getUnusedBufferByteCount(JsonIterator iter) {
306+
// Get bytes from 0 to skipStart + from tail till end
307+
return iter.buf.length - iter.tail + iter.skipStartedAt;
308+
}
309+
304310
final static byte readByte(JsonIterator iter) throws IOException {
305311
if (iter.head == iter.tail) {
306312
if (!loadMore(iter)) {
@@ -643,8 +649,7 @@ static final int readInt(final JsonIterator iter, final byte c) throws IOExcepti
643649

644650
static void assertNotLeadingZero(JsonIterator iter) throws IOException {
645651
try {
646-
byte nextByte = IterImpl.readByte(iter);
647-
iter.unreadByte();
652+
byte nextByte = iter.buf[iter.head];
648653
int ind2 = IterImplNumber.intDigits[nextByte];
649654
if (ind2 == IterImplNumber.INVALID_CHAR_FOR_NUMBER) {
650655
return;

‎src/main/java/com/jsoniter/JsonIterator.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ public class JsonIterator implements Closeable {
2121
final static ValueType[] valueTypes = new ValueType[256];
2222
InputStream in;
2323
byte[] buf;
24+
// Whenever buf is not large enough new one is created with size of
25+
// buf.length + autoExpandBufferStep. Set to < 1 to disable auto expanding.
26+
int autoExpandBufferStep;
2427
int head;
2528
int tail;
2629
int skipStartedAt = -1; // skip should keep bytes starting at this pos
@@ -60,13 +63,22 @@ private JsonIterator(InputStream in, byte[] buf, int head, int tail) {
6063
this.tail = tail;
6164
}
6265

66+
private JsonIterator(InputStream in, byte[] buf, int autoExpandBufferStep) {
67+
this(in, buf, 0, 0);
68+
this.autoExpandBufferStep = autoExpandBufferStep;
69+
}
70+
6371
public JsonIterator() {
6472
this(null, new byte[0], 0, 0);
6573
}
6674

6775
public static JsonIterator parse(InputStream in, int bufSize) {
76+
return parse(in, bufSize, bufSize);
77+
}
78+
79+
public static JsonIterator parse(InputStream in, int bufSize, int autoExpandBufferStep) {
6880
enableStreamingSupport();
69-
return new JsonIterator(in, new byte[bufSize], 0, 0);
81+
return new JsonIterator(in, new byte[bufSize], autoExpandBufferStep);
7082
}
7183

7284
public static JsonIterator parse(byte[] buf) {

‎src/test/java/com/jsoniter/IterImplForStreamingTest.java

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
package com.jsoniter;
22

3+
import com.jsoniter.any.Any;
4+
import com.jsoniter.spi.JsonException;
5+
import java.io.ByteArrayInputStream;
6+
import java.io.IOException;
7+
import java.io.InputStream;
38
import junit.framework.TestCase;
9+
import org.junit.experimental.categories.Category;
10+
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
411

512
public class IterImplForStreamingTest extends TestCase {
613

@@ -11,4 +18,76 @@ public void testReadMaxDouble() throws Exception {
1118
String number = new String(numberChars.chars, 0, numberChars.charsLength);
1219
assertEquals(maxDouble, number);
1320
}
14-
}
21+
22+
@Category(StreamingCategory.class)
23+
public void testLoadMore() throws IOException {
24+
final String originalContent = "1234567890";
25+
final byte[] src = ("{\"a\":\"" + originalContent + "\"}").getBytes();
26+
27+
int initialBufferSize;
28+
Any parsedString;
29+
// Case #1: Data fits into initial buffer, autoresizing on
30+
// Input must definitely fit into such large buffer
31+
initialBufferSize = src.length * 2;
32+
JsonIterator jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 512);
33+
jsonIterator.readObject();
34+
parsedString = jsonIterator.readAny();
35+
assertEquals(originalContent, parsedString.toString());
36+
// Check buffer was not expanded
37+
assertEquals(initialBufferSize, jsonIterator.buf.length);
38+
39+
// Case #2: Data does not fit into initial buffer, autoresizing off
40+
initialBufferSize = originalContent.length() / 2;
41+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 0);
42+
jsonIterator.readObject();
43+
try {
44+
jsonIterator.readAny();
45+
fail("Expect to fail because buffer is too small.");
46+
} catch (JsonException e) {
47+
if (!e.getMessage().startsWith("loadMore")) {
48+
throw e;
49+
}
50+
}
51+
// Check buffer was not expanded
52+
assertEquals(initialBufferSize, jsonIterator.buf.length);
53+
54+
// Case #3: Data does fit into initial buffer, autoresizing on
55+
initialBufferSize = originalContent.length() / 2;
56+
int autoExpandBufferStep = initialBufferSize * 3;
57+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, autoExpandBufferStep);
58+
jsonIterator.readObject();
59+
parsedString = jsonIterator.readAny();
60+
assertEquals(originalContent, parsedString.toString());
61+
// Check buffer was expanded exactly once
62+
assertEquals(initialBufferSize + autoExpandBufferStep, jsonIterator.buf.length);
63+
64+
// Case #4: Data does not fit (but largest string does) into initial buffer, autoresizing on
65+
initialBufferSize = originalContent.length() + 2;
66+
jsonIterator = JsonIterator.parse(new ByteArrayInputStream(src), initialBufferSize, 0);
67+
jsonIterator.readObject();
68+
parsedString = jsonIterator.readAny();
69+
assertEquals(originalContent, parsedString.toString());
70+
// Check buffer was expanded exactly once
71+
assertEquals(initialBufferSize, jsonIterator.buf.length);
72+
}
73+
74+
private static InputStream getSluggishInputStream(final byte[] src) {
75+
return new InputStream() {
76+
int position = 0;
77+
78+
@Override
79+
public int read() throws IOException {
80+
throw new NotImplementedException();
81+
}
82+
83+
@Override
84+
public int read(byte[] b, int off, int len) throws IOException {
85+
if (position < src.length) {
86+
b[off] = src[position++];
87+
return 1;
88+
}
89+
return -1;
90+
}
91+
};
92+
}
93+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,28 @@
11
package com.jsoniter.any;
22

3+
import com.jsoniter.spi.JsonException;
34
import junit.framework.TestCase;
45

56
public class TestLong extends TestCase {
67
public void test_to_string_should_trim() {
78
Any any = Any.lazyLong(" 1000".getBytes(), 0, " 1000".length());
89
assertEquals("1000", any.toString());
910
}
11+
12+
public void test_should_fail_with_leading_zero() {
13+
byte[] bytes = "01".getBytes();
14+
Any any = Any.lazyLong(bytes, 0, bytes.length);
15+
try {
16+
any.toLong();
17+
fail("This should fail.");
18+
} catch (JsonException e) {
19+
20+
}
21+
}
22+
23+
public void test_should_work_with_zero() {
24+
byte[] bytes = "0".getBytes();
25+
Any any = Any.lazyLong(bytes, 0, bytes.length);
26+
assertEquals(0L, any.toLong());
27+
}
1028
}

‎src/test/java/com/jsoniter/suite/AllTestCases.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
TestGson.class,
5454
com.jsoniter.output.TestGson.class,
5555
TestStreamBuffer.class,
56+
IterImplForStreamingTest.class,
5657
TestCollection.class,
5758
TestList.class,
5859
TestAnnotationJsonObject.class,

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /