Skip to content

Commit aaaacfb

Browse files
drmdrm
committed
ScanIterator no longer stops on empty pages
1 parent c4cef43 commit aaaacfb

5 files changed

Lines changed: 120 additions & 50 deletions

File tree

‎src/nl/melp/redis/collections/ScanIterator.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import nl.melp.redis.Redis;
44

55
import java.io.IOException;
6+
import java.util.ArrayList;
67
import java.util.Iterator;
78
import java.util.LinkedList;
89
import java.util.List;
@@ -20,8 +21,11 @@ public ScanIterator(Redis redis, byte[]operation, byte[] keyName) {
2021
this(redis, operation, null, keyName);
2122
}
2223

23-
2424
public ScanIterator(Redis redis, byte[]operation, byte[] match, byte[] keyName) {
25+
this(redis, 1000, operation, match, keyName);
26+
}
27+
28+
public ScanIterator(Redis redis, int bufferSize, byte[]operation, byte[] match, byte[] keyName) {
2529
this.redis = redis;
2630
this.operation = operation;
2731
this.keyName = keyName;
@@ -30,7 +34,6 @@ public ScanIterator(Redis redis, byte[]operation, byte[] match, byte[] keyName)
3034
this.match = match;
3135
}
3236

33-
3437
@Override
3538
public boolean hasNext() {
3639
if (this.buffer == null || this.localCursor >= this.buffer.size()) {
@@ -40,21 +43,26 @@ public boolean hasNext() {
4043
}
4144
try {
4245
List<Object> result;
43-
List<Object> args = new LinkedList<>();
46+
List<Object> args = new ArrayList<>();
4447
args.add(this.operation);
4548
if (keyName != null) {
4649
args.add(this.keyName);
4750
}
48-
args.add(Integer.toString(this.cursor));
51+
int cursorPos = args.size();
52+
args.add(null);
4953
if (match != null) {
5054
args.add("MATCH");
5155
args.add(match);
5256
}
53-
synchronized (redis) {
54-
result = redis.call(args.toArray());
55-
}
56-
this.cursor = Integer.valueOf(new String((byte[]) result.get(0)));
57-
this.buffer = (List<byte[]>) result.get(1);
57+
do {
58+
args.set(cursorPos, Integer.toString(this.cursor));
59+
synchronized (redis) {
60+
result = redis.call(args.toArray());
61+
}
62+
this.cursor = Integer.parseInt(new String((byte[]) result.get(0)));
63+
this.buffer = (List<byte[]>) result.get(1);
64+
// skip over empty pages
65+
} while (this.cursor != 0 && this.buffer.size() == 0);
5866
this.localCursor = 0;
5967
} catch (IOException e) {
6068
e.printStackTrace();

‎src/nl/melp/redis/collections/SerializedMappedSet.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,11 +150,11 @@ public void putAll(Map<? extends K, ? extends Set<V>> map) {
150150
@Override
151151
public void clear() {
152152
synchronized (redis) {
153-
byte[] copy = new byte[prefixLength + 2];
154-
System.arraycopy(prefix, 0, copy, 0, prefixLength);
155-
copy[prefixLength] = ':';
156-
copy[prefixLength + 1] = '*';
157-
Iterator<byte[]> i = new ScanIterator(redis, "SCAN".getBytes(), copy, null);
153+
byte[] prefix = new byte[prefixLength + 2];
154+
System.arraycopy(this.prefix, 0, prefix, 0, prefixLength);
155+
prefix[prefixLength] = ':';
156+
prefix[prefixLength + 1] = '*';
157+
Iterator<byte[]> i = new ScanIterator(redis, "SCAN".getBytes(), prefix, null);
158158

159159
while (i.hasNext()) {
160160
try {

‎test/nl/melp/redis/collections/AbstractIntegrationTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package nl.melp.redis.collections;
2+
3+
import nl.melp.redis.Redis;
4+
import org.junit.After;
5+
import org.junit.Assert;
6+
import org.junit.Before;
7+
8+
import java.io.IOException;
9+
import java.net.Socket;
10+
import java.util.List;
11+
12+
abstract public class AbstractIntegrationTest {
13+
protected Redis redis;
14+
private Socket socket;
15+
16+
@Before
17+
public void init() throws IOException {
18+
socket = new Socket("localhost", 6379);
19+
redis = new Redis(socket);
20+
redis.call("SELECT", "15");
21+
Assert.assertEquals("Refusing to run on non-empty database", 0, ((List<?>) redis.call("KEYS", "*")).size());
22+
}
23+
24+
@After
25+
public void cleanup() throws IOException {
26+
int size = ((List<?>) redis.call("KEYS", "*")).size();
27+
redis.call("QUIT");
28+
if (!socket.isClosed()) {
29+
socket.close();
30+
}
31+
if (size > 0) {
32+
Assert.fail("This test is littering");
33+
}
34+
}
35+
}

‎test/nl/melp/redis/collections/IntegrationTest.java‎

Lines changed: 4 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,44 +24,13 @@
2424
import java.util.concurrent.ScheduledExecutorService;
2525
import java.util.concurrent.TimeUnit;
2626

27-
import static junit.framework.Assert.assertEquals;
27+
import static org.junit.Assert.assertEquals;
2828
import static org.junit.Assert.assertFalse;
2929
import static org.junit.Assert.assertNull;
3030
import static org.junit.Assert.assertTrue;
3131

32-
public class IntegrationTest {
32+
public class IntegrationTest extends AbstractIntegrationTest {
3333
private final String keyName = IntegrationTest.class.getCanonicalName();
34-
private Redis redis;
35-
private Socket socket;
36-
37-
@Test
38-
public void tmp () throws IOException {
39-
redis.call("SELECT", "0");
40-
System.out.println(
41-
new SerializedHashMap<>(redis, "nl.melp.linkchecker.LinkChecker.report.statuses").keySet().size()
42-
);
43-
}
44-
45-
@Before
46-
public void init() throws IOException {
47-
socket = new Socket("localhost", 6379);
48-
redis = new Redis(socket);
49-
redis.call("SELECT", "15");
50-
Assert.assertEquals("Refusing to run on non-empty database", 0, ((List<?>) redis.call("KEYS", "*")).size());
51-
}
52-
53-
@After
54-
public void cleanup() throws IOException {
55-
int size = ((List<?>) redis.call("KEYS", "*")).size();
56-
redis.call("QUIT");
57-
if (!socket.isClosed()) {
58-
socket.close();
59-
}
60-
if (size > 0) {
61-
Assert.fail("This test is littering");
62-
}
63-
}
64-
6534

6635
@Test
6736
public void testList() throws IOException {
@@ -293,8 +262,8 @@ public void testSerializedMappedSet() throws IOException {
293262
assertTrue(values.get("foo").contains("bar"));
294263
assertTrue(secondary.get("foo").contains("bar"));
295264
values.get("foo").remove("bar");
296-
assertEquals(1, values.size());
297-
assertEquals(1, secondary.size());
265+
assertEquals(0, values.size());
266+
assertEquals(0, secondary.size());
298267
values.clear();
299268
assertEquals(0, values.size());
300269
assertEquals(0, secondary.size());
@@ -383,6 +352,5 @@ public void testSerializedMappedSetKeySynchronization() throws IOException {
383352

384353
Assert.assertEquals(0, map.size());
385354
Assert.assertEquals(0, secondary.size());
386-
387355
}
388356
}

‎test/nl/melp/redis/collections/ScanIteratorTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package nl.melp.redis.collections;
2+
3+
import org.junit.Assert;
4+
import org.junit.Test;
5+
6+
import java.io.IOException;
7+
import java.util.List;
8+
9+
10+
public class ScanIteratorTest extends AbstractIntegrationTest {
11+
@Test
12+
public void testScan() throws IOException {
13+
for (int i = 0; i < 1000; i ++) {
14+
redis.call("SET", "foo:" + i, "Test");
15+
}
16+
17+
ScanIterator it = new ScanIterator(redis, "SCAN".getBytes(), "foo:*".getBytes(), null);
18+
while (it.hasNext()) {
19+
byte[] r = it.next();
20+
redis.call("DEL", r);
21+
}
22+
23+
Assert.assertEquals(0, ((List)redis.call("KEYS", "foo:*")).size());
24+
}
25+
26+
@Test
27+
public void testScanEmptyPages() throws IOException {
28+
for (int i = 0; i < 1000; i ++) {
29+
redis.call("SET", "foo:" + i, "Test");
30+
}
31+
32+
ScanIterator it = new ScanIterator(redis, 1, "SCAN".getBytes(), "foo:9*".getBytes(), null);
33+
int i = 0;
34+
while (it.hasNext()) {
35+
it.next();
36+
i ++;
37+
}
38+
Assert.assertEquals(111, i);
39+
40+
redis.call("FLUSHDB");
41+
}
42+
43+
@Test
44+
public void testHscan() throws IOException {
45+
for (int i = 0; i < 1000; i ++) {
46+
redis.call("HSET", "foo", Integer.toString(i), "Test");
47+
}
48+
49+
Assert.assertEquals(Long.valueOf(1000L), redis.call("HLEN", "foo"));
50+
51+
ScanIterator it = new ScanIterator(redis, "HSCAN".getBytes(), "foo".getBytes());
52+
while (it.hasNext()) {
53+
byte[] r = it.next();
54+
redis.call("HDEL", "foo", r);
55+
}
56+
57+
Assert.assertEquals(Long.valueOf(0), redis.call("HLEN", "foo"));
58+
}
59+
}

0 commit comments

Comments
 (0)