Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit fee3d47

Browse files
DATAREDIS-803 - Work around Redis parameter limitation.
Redis has a [limitation of 1024 * 1024 parameters](https://github.com/antirez/redis/blob/4.0.9/src/networking.c#L1200) for bulk operations. To insert more than 1024 * 1024 / 2 - 1 entries with putAll(), they need to be split up in multiple HMSET commands. To reveive more than 1024 * 1024 - 1 entries with entrySet(), we can directly use the HGETALL command instead of first fetching the keys with HKEYS and then fetching the values with HMGET.
1 parent e0b73c7 commit fee3d47

File tree

4 files changed

+40
-21
lines changed

4 files changed

+40
-21
lines changed

‎src/main/java/org/springframework/data/redis/core/DefaultHashOperations.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import java.util.Collection;
1919
import java.util.Collections;
20+
import java.util.Iterator;
2021
import java.util.LinkedHashMap;
2122
import java.util.List;
2223
import java.util.Map;
@@ -32,6 +33,7 @@
3233
* @author Costin Leau
3334
* @author Christoph Strobl
3435
* @author Ninad Divadkar
36+
* @author Christian Bühler
3537
*/
3638
class DefaultHashOperations<K, HK, HV> extends AbstractOperations<K, Object> implements HashOperations<K, HK, HV> {
3739

@@ -141,16 +143,22 @@ public void putAll(K key, Map<? extends HK, ? extends HV> m) {
141143

142144
byte[] rawKey = rawKey(key);
143145

144-
Map<byte[], byte[]> hashes = new LinkedHashMap<>(m.size());
146+
int size = Math.min(RedisTemplate.REDIS_MAX_ARGS / 2 - 1, m.size());
147+
Map<byte[], byte[]> hashes = new LinkedHashMap<>(size);
145148

146-
for (Map.Entry<? extends HK, ? extends HV> entry : m.entrySet()) {
149+
Iterator<? extends Entry<? extends HK, ? extends HV>> entries = m.entrySet().iterator();
150+
while (entries.hasNext()) {
151+
Entry<? extends HK, ? extends HV> entry = entries.next();
147152
hashes.put(rawHashKey(entry.getKey()), rawHashValue(entry.getValue()));
148-
}
149153

150-
execute(connection -> {
151-
connection.hMSet(rawKey, hashes);
152-
return null;
153-
}, true);
154+
if (!entries.hasNext() || hashes.size() == size) {
155+
execute(connection -> {
156+
connection.hMSet(rawKey, hashes);
157+
return null;
158+
}, true);
159+
hashes.clear();
160+
}
161+
}
154162
}
155163

156164
/*

‎src/main/java/org/springframework/data/redis/core/RedisTemplate.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,15 @@
7979
* @author Ninad Divadkar
8080
* @author Anqing Shao
8181
* @author Mark Paluch
82+
* @author Christian Bühler
8283
* @param <K> the Redis key type against which the template works (usually a String)
8384
* @param <V> the Redis value type against which the template works
8485
* @see StringRedisTemplate
8586
*/
8687
public class RedisTemplate<K, V> extends RedisAccessor implements RedisOperations<K, V>, BeanClassLoaderAware {
8788

89+
public static final int REDIS_MAX_ARGS = 1024 * 1024;
90+
8891
private boolean enableTransactionSupport = false;
8992
private boolean exposeConnection = false;
9093
private boolean initialized = false;

‎src/main/java/org/springframework/data/redis/support/collections/DefaultRedisMap.java‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
*
3939
* @author Costin Leau
4040
* @author Christoph Strobl
41+
* @author Christian Bühler
4142
*/
4243
public class DefaultRedisMap<K, V> implements RedisMap<K, V> {
4344

@@ -155,20 +156,7 @@ public boolean containsValue(Object value) {
155156
*/
156157
@Override
157158
public Set<java.util.Map.Entry<K, V>> entrySet() {
158-
159-
Set<K> keySet = keySet();
160-
checkResult(keySet);
161-
Collection<V> multiGet = hashOps.multiGet(keySet);
162-
163-
Iterator<K> keys = keySet.iterator();
164-
Iterator<V> values = multiGet.iterator();
165-
166-
Set<Map.Entry<K, V>> entries = new LinkedHashSet<>();
167-
while (keys.hasNext()) {
168-
entries.add(new DefaultRedisMapEntry(keys.next(), values.next()));
169-
}
170-
171-
return entries;
159+
return hashOps.entries().entrySet();
172160
}
173161

174162
/*

‎src/test/java/org/springframework/data/redis/support/collections/AbstractRedisMapTests.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.ArrayList;
2626
import java.util.Collection;
2727
import java.util.Collections;
28+
import java.util.HashMap;
2829
import java.util.LinkedHashMap;
2930
import java.util.LinkedHashSet;
3031
import java.util.Map;
@@ -62,6 +63,7 @@
6263
* @author Jennifer Hickey
6364
* @author Christoph Strobl
6465
* @author Thomas Darimont
66+
* @author Christian Bühler
6567
*/
6668
@RunWith(Parameterized.class)
6769
public abstract class AbstractRedisMapTests<K, V> {
@@ -396,6 +398,24 @@ public void testEntrySet() {
396398
assertThat(values, not(hasItem(v2)));
397399
}
398400

401+
@Test // DATAREDIS-803
402+
@IfProfileValue(name = "runLongTests", value = "true")
403+
public void testBigEntrySet() {
404+
405+
Set<Entry<K, V>> entries = map.entrySet();
406+
assertTrue(entries.isEmpty());
407+
408+
Map<K, V> m = new HashMap<>();
409+
for (int i = 0; i < RedisTemplate.REDIS_MAX_ARGS - 1; i++) {
410+
m.put(getKey(), getValue());
411+
}
412+
map.putAll(m);
413+
414+
entries = map.entrySet();
415+
416+
assertEquals(RedisTemplate.REDIS_MAX_ARGS - 1, entries.size());
417+
}
418+
399419
@Test
400420
public void testPutIfAbsent() {
401421

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /