Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 5a1c765

Browse files
committed
feat: Add batchSize option to RedisItemReader for MGET optimization
- Add batchSize parameter to RedisItemReader with default value of 1 - Implement batch fetching using Redis MGET for better performance - Maintain backward compatibility with existing code - Add comprehensive test cases for batch reading functionality This enhancement addresses the N+1 problem by reducing network round-trips when reading multiple keys from Redis. With batchSize > 1, the reader uses MGET operations instead of individual GET calls. Resolves #4941 Signed-off-by: khj68 <junthewise@gmail.com>
1 parent 3bcc525 commit 5a1c765

File tree

3 files changed

+182
-5
lines changed

3 files changed

+182
-5
lines changed

‎spring-batch-infrastructure/src/main/java/org/springframework/batch/item/redis/RedisItemReader.java

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
*/
1616
package org.springframework.batch.item.redis;
1717

18+
import java.util.ArrayList;
19+
import java.util.LinkedList;
20+
import java.util.List;
21+
import java.util.Queue;
22+
1823
import org.springframework.batch.item.ExecutionContext;
1924
import org.springframework.batch.item.ItemStreamException;
2025
import org.springframework.batch.item.ItemStreamReader;
@@ -29,10 +34,17 @@
2934
* query.
3035
*
3136
* <p>
37+
* The reader supports batch fetching using Redis MGET operations when batchSize is
38+
* greater than 1, which significantly improves performance by reducing network
39+
* round-trips.
40+
* </p>
41+
*
42+
* <p>
3243
* The implementation is not thread-safe and not restartable.
3344
* </p>
3445
*
3546
* @author Mahmoud Ben Hassine
47+
* @author Jun Kim
3648
* @since 5.1
3749
* @param <K> type of keys
3850
* @param <V> type of values
@@ -43,13 +55,24 @@ public class RedisItemReader<K, V> implements ItemStreamReader<V> {
4355

4456
private final ScanOptions scanOptions;
4557

58+
private final int batchSize;
59+
60+
private final Queue<V> valueBuffer;
61+
4662
private Cursor<K> cursor;
4763

4864
public RedisItemReader(RedisTemplate<K, V> redisTemplate, ScanOptions scanOptions) {
65+
this(redisTemplate, scanOptions, 1);
66+
}
67+
68+
public RedisItemReader(RedisTemplate<K, V> redisTemplate, ScanOptions scanOptions, int batchSize) {
4969
Assert.notNull(redisTemplate, "redisTemplate must not be null");
5070
Assert.notNull(scanOptions, "scanOptions must no be null");
71+
Assert.isTrue(batchSize > 0 && batchSize <= 1000, "batchSize must be between 1 and 1000");
5172
this.redisTemplate = redisTemplate;
5273
this.scanOptions = scanOptions;
74+
this.batchSize = batchSize;
75+
this.valueBuffer = new LinkedList<>();
5376
}
5477

5578
@Override
@@ -59,13 +82,46 @@ public void open(ExecutionContext executionContext) throws ItemStreamException {
5982

6083
@Override
6184
public V read() throws Exception {
62-
if (this.cursor.hasNext()) {
63-
KnextKey = this.cursor.next();
64-
return this.redisTemplate.opsForValue().get(nextKey);
85+
// If buffer has values, return from buffer first
86+
if (!this.valueBuffer.isEmpty()) {
87+
return this.valueBuffer.poll();
6588
}
66-
else {
89+
90+
// If batch size is 1, use single GET operation (backward compatibility)
91+
if (this.batchSize == 1) {
92+
if (this.cursor.hasNext()) {
93+
K nextKey = this.cursor.next();
94+
return this.redisTemplate.opsForValue().get(nextKey);
95+
}
96+
else {
97+
return null;
98+
}
99+
}
100+
101+
// Batch mode: collect keys and use MGET
102+
List<K> keysToFetch = new ArrayList<>();
103+
while (this.cursor.hasNext() && keysToFetch.size() < this.batchSize) {
104+
keysToFetch.add(this.cursor.next());
105+
}
106+
107+
if (keysToFetch.isEmpty()) {
67108
return null;
68109
}
110+
111+
// Use multiGet for batch fetching
112+
List<V> values = this.redisTemplate.opsForValue().multiGet(keysToFetch);
113+
114+
if (values != null) {
115+
// Filter out null values and add to buffer
116+
for (V value : values) {
117+
if (value != null) {
118+
this.valueBuffer.offer(value);
119+
}
120+
}
121+
}
122+
123+
// Return first value from buffer or null if empty
124+
return this.valueBuffer.poll();
69125
}
70126

71127
@Override

‎spring-batch-infrastructure/src/main/java/org/springframework/batch/item/redis/builder/RedisItemReaderBuilder.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
* Builder for {@link RedisItemReader}.
2424
*
2525
* @author Mahmoud Ben Hassine
26+
* @author Jun Kim
2627
* @since 5.1
2728
* @param <K> type of keys
2829
* @param <V> type of values
@@ -33,6 +34,8 @@ public class RedisItemReaderBuilder<K, V> {
3334

3435
private ScanOptions scanOptions;
3536

37+
private int batchSize = 1;
38+
3639
/**
3740
* Set the {@link RedisTemplate} to use in the reader.
3841
* @param redisTemplate the template to use
@@ -53,12 +56,25 @@ public RedisItemReaderBuilder<K, V> scanOptions(ScanOptions scanOptions) {
5356
return this;
5457
}
5558

59+
/**
60+
* Set the batch size for fetching values from Redis. When set to a value greater than
61+
* 1, the reader will use Redis MGET operations to fetch multiple values in a single
62+
* network round-trip, significantly improving performance.
63+
* @param batchSize the number of keys to fetch in each batch (must be between 1 and
64+
* 1000)
65+
* @return the current builder instance for fluent chaining
66+
*/
67+
public RedisItemReaderBuilder<K, V> batchSize(int batchSize) {
68+
this.batchSize = batchSize;
69+
return this;
70+
}
71+
5672
/**
5773
* Build a new {@link RedisItemReader}.
5874
* @return a new item reader
5975
*/
6076
public RedisItemReader<K, V> build() {
61-
return new RedisItemReader<>(this.redisTemplate, this.scanOptions);
77+
return new RedisItemReader<>(this.redisTemplate, this.scanOptions, this.batchSize);
6278
}
6379

6480
}

‎spring-batch-infrastructure/src/test/java/org/springframework/batch/item/redis/RedisItemReaderTests.java

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
*/
1616
package org.springframework.batch.item.redis;
1717

18+
import java.util.Arrays;
19+
import java.util.Collections;
20+
import java.util.List;
21+
1822
import org.junit.jupiter.api.Assertions;
1923
import org.junit.jupiter.api.Test;
2024
import org.junit.jupiter.api.extension.ExtendWith;
@@ -62,4 +66,105 @@ void testRead() throws Exception {
6266
Assertions.assertNull(item3);
6367
}
6468

69+
@Test
70+
void testReadWithBatchSize() throws Exception {
71+
// given
72+
int batchSize = 2;
73+
Mockito.when(this.redisTemplate.scan(this.scanOptions)).thenReturn(this.cursor);
74+
Mockito.when(this.cursor.hasNext()).thenReturn(true, true, true, true, false);
75+
Mockito.when(this.cursor.next()).thenReturn("person:1", "person:2", "person:3", "person:4");
76+
77+
// Setup multiGet - batch of 2 keys each time
78+
Mockito.when(this.redisTemplate.opsForValue().multiGet(Mockito.anyList())).thenAnswer(invocation -> {
79+
List<String> keys = invocation.getArgument(0);
80+
if (keys.size() == 2) {
81+
if (keys.get(0).equals("person:1") && keys.get(1).equals("person:2")) {
82+
return Arrays.asList("foo", "bar");
83+
}
84+
if (keys.get(0).equals("person:3") && keys.get(1).equals("person:4")) {
85+
return Arrays.asList("baz", "qux");
86+
}
87+
}
88+
return Collections.emptyList();
89+
});
90+
91+
RedisItemReader<String, String> redisItemReader = new RedisItemReader<>(this.redisTemplate, this.scanOptions,
92+
batchSize);
93+
redisItemReader.open(new ExecutionContext());
94+
95+
// when
96+
String item1 = redisItemReader.read();
97+
String item2 = redisItemReader.read();
98+
String item3 = redisItemReader.read();
99+
String item4 = redisItemReader.read();
100+
String item5 = redisItemReader.read();
101+
102+
// then
103+
Assertions.assertEquals("foo", item1);
104+
Assertions.assertEquals("bar", item2);
105+
Assertions.assertEquals("baz", item3);
106+
Assertions.assertEquals("qux", item4);
107+
Assertions.assertNull(item5);
108+
109+
// Verify multiGet was called instead of individual get
110+
Mockito.verify(this.redisTemplate.opsForValue(), Mockito.times(2)).multiGet(Mockito.any());
111+
Mockito.verify(this.redisTemplate.opsForValue(), Mockito.never()).get(Mockito.any());
112+
}
113+
114+
@Test
115+
void testReadWithBatchSizeHandlesNullValues() throws Exception {
116+
// given
117+
int batchSize = 3;
118+
Mockito.when(this.redisTemplate.scan(this.scanOptions)).thenReturn(this.cursor);
119+
Mockito.when(this.cursor.hasNext()).thenReturn(true, true, true, false);
120+
Mockito.when(this.cursor.next()).thenReturn("person:1", "person:2", "person:3");
121+
122+
// multiGet returns some null values
123+
List<String> keys = Arrays.asList("person:1", "person:2", "person:3");
124+
List<String> values = Arrays.asList("foo", null, "baz"); // person:2 is null
125+
Mockito.when(this.redisTemplate.opsForValue().multiGet(keys)).thenReturn(values);
126+
127+
RedisItemReader<String, String> redisItemReader = new RedisItemReader<>(this.redisTemplate, this.scanOptions,
128+
batchSize);
129+
redisItemReader.open(new ExecutionContext());
130+
131+
// when
132+
String item1 = redisItemReader.read();
133+
String item2 = redisItemReader.read();
134+
String item3 = redisItemReader.read();
135+
136+
// then - null values should be filtered out
137+
Assertions.assertEquals("foo", item1);
138+
Assertions.assertEquals("baz", item2);
139+
Assertions.assertNull(item3);
140+
}
141+
142+
@Test
143+
void testBackwardCompatibilityWithDefaultBatchSize() throws Exception {
144+
// given - using constructor without batchSize (defaults to 1)
145+
Mockito.when(this.redisTemplate.scan(this.scanOptions)).thenReturn(this.cursor);
146+
Mockito.when(this.cursor.hasNext()).thenReturn(true, true, false);
147+
Mockito.when(this.cursor.next()).thenReturn("person:1", "person:2");
148+
Mockito.when(this.redisTemplate.opsForValue().get("person:1")).thenReturn("foo");
149+
Mockito.when(this.redisTemplate.opsForValue().get("person:2")).thenReturn("bar");
150+
151+
// Using old constructor
152+
RedisItemReader<String, String> redisItemReader = new RedisItemReader<>(this.redisTemplate, this.scanOptions);
153+
redisItemReader.open(new ExecutionContext());
154+
155+
// when
156+
String item1 = redisItemReader.read();
157+
String item2 = redisItemReader.read();
158+
String item3 = redisItemReader.read();
159+
160+
// then
161+
Assertions.assertEquals("foo", item1);
162+
Assertions.assertEquals("bar", item2);
163+
Assertions.assertNull(item3);
164+
165+
// Verify individual get was called, not multiGet (backward compatibility)
166+
Mockito.verify(this.redisTemplate.opsForValue(), Mockito.times(2)).get(Mockito.any());
167+
Mockito.verify(this.redisTemplate.opsForValue(), Mockito.never()).multiGet(Mockito.any());
168+
}
169+
65170
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /