Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit d6ceec8

Browse files
Work around Redis parameter limitation
Redis has a [limitation of 1024*1024 parameters](https://github.com/antirez/redis/blob/4.0.9/src/networking.c#L1200) for bulk operations. To insert more than 1024*1024/2-1 entries with putAll(), they need to be split up in multiple HMSET commands. To reveive more than 1024*1024-1 entries with entrySet(), we can directly use the HGETALL command instead of first fetching the keys with HKEYS and then fetching the values with HMGET.
1 parent e0b73c7 commit d6ceec8

File tree

4 files changed

+34
-34
lines changed

4 files changed

+34
-34
lines changed

‎src/main/java/org/springframework/data/redis/core/DefaultHashOperations.java‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,8 @@
1515
*/
1616
package org.springframework.data.redis.core;
1717

18-
import java.util.Collection;
19-
import java.util.Collections;
20-
import java.util.LinkedHashMap;
21-
import java.util.List;
22-
import java.util.Map;
18+
import java.util.*;
2319
import java.util.Map.Entry;
24-
import java.util.Set;
2520

2621
import org.springframework.core.convert.converter.Converter;
2722
import org.springframework.lang.Nullable;
@@ -141,16 +136,22 @@ public void putAll(K key, Map<? extends HK, ? extends HV> m) {
141136

142137
byte[] rawKey = rawKey(key);
143138

144-
Map<byte[], byte[]> hashes = new LinkedHashMap<>(m.size());
139+
int size = Math.min(RedisTemplate.REDIS_MAX_ARGS / 2 - 1, m.size());
140+
Map<byte[], byte[]> hashes = new LinkedHashMap<>(size);
145141

146-
for (Map.Entry<? extends HK, ? extends HV> entry : m.entrySet()) {
142+
Iterator<? extends Entry<? extends HK, ? extends HV>> entries = m.entrySet().iterator();
143+
while (entries.hasNext()) {
144+
Entry<? extends HK, ? extends HV> entry = entries.next();
147145
hashes.put(rawHashKey(entry.getKey()), rawHashValue(entry.getValue()));
148-
}
149146

150-
execute(connection -> {
151-
connection.hMSet(rawKey, hashes);
152-
return null;
153-
}, true);
147+
if (!entries.hasNext() || hashes.size() == size) {
148+
execute(connection -> {
149+
connection.hMSet(rawKey, hashes);
150+
return null;
151+
}, true);
152+
hashes.clear();
153+
}
154+
}
154155
}
155156

156157
/*

‎src/main/java/org/springframework/data/redis/core/RedisTemplate.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@
8585
*/
8686
public class RedisTemplate<K, V> extends RedisAccessor implements RedisOperations<K, V>, BeanClassLoaderAware {
8787

88+
public static final int REDIS_MAX_ARGS = 1024 * 1024;
89+
8890
private boolean enableTransactionSupport = false;
8991
private boolean exposeConnection = false;
9092
private boolean initialized = false;

‎src/main/java/org/springframework/data/redis/support/collections/DefaultRedisMap.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -155,20 +155,7 @@ public boolean containsValue(Object value) {
155155
*/
156156
@Override
157157
public Set<java.util.Map.Entry<K, V>> entrySet() {
158-
159-
Set<K> keySet = keySet();
160-
checkResult(keySet);
161-
Collection<V> multiGet = hashOps.multiGet(keySet);
162-
163-
Iterator<K> keys = keySet.iterator();
164-
Iterator<V> values = multiGet.iterator();
165-
166-
Set<Map.Entry<K, V>> entries = new LinkedHashSet<>();
167-
while (keys.hasNext()) {
168-
entries.add(new DefaultRedisMapEntry(keys.next(), values.next()));
169-
}
170-
171-
return entries;
158+
return hashOps.entries().entrySet();
172159
}
173160

174161
/*

‎src/test/java/org/springframework/data/redis/support/collections/AbstractRedisMapTests.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,8 @@
2222

2323
import java.io.IOException;
2424
import java.text.DecimalFormat;
25-
import java.util.ArrayList;
26-
import java.util.Collection;
27-
import java.util.Collections;
28-
import java.util.LinkedHashMap;
29-
import java.util.LinkedHashSet;
30-
import java.util.Map;
25+
import java.util.*;
3126
import java.util.Map.Entry;
32-
import java.util.Set;
3327

3428
import org.junit.After;
3529
import org.junit.AfterClass;
@@ -396,6 +390,22 @@ public void testEntrySet() {
396390
assertThat(values, not(hasItem(v2)));
397391
}
398392

393+
@Test
394+
public void testBigEntrySet() {
395+
Set<Entry<K, V>> entries = map.entrySet();
396+
assertTrue(entries.isEmpty());
397+
398+
Map<K, V> m = new HashMap<>();
399+
for (int i = 0; i < RedisTemplate.REDIS_MAX_ARGS - 1; i++) {
400+
m.put(getKey(), getValue());
401+
}
402+
map.putAll(m);
403+
404+
entries = map.entrySet();
405+
406+
assertEquals(RedisTemplate.REDIS_MAX_ARGS - 1, entries.size());
407+
}
408+
399409
@Test
400410
public void testPutIfAbsent() {
401411

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /