Skip to content

Commit 91d1609

Browse files
authored
Add native fast reset compressors (#44)
* Add native fast reset compressors * Add native fast reset compressor tests * Simplify byte buffer paths * Add constructor normalization for compression level and acceleration * Switch to a lock for guarding the statePtr * Add missing HC test * Throw on concurrent close attempt * Throw on unsupported buffer attempt * Move shared logic to an abstract base class
1 parent e122114 commit 91d1609

10 files changed

Lines changed: 1396 additions & 4 deletions

pom.xml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -950,6 +950,112 @@
950950
</configuration>
951951
</execution>
952952

953+
<!-- LZ4CompressorTest fast-reset compressor (array) -->
954+
<execution>
955+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_array</id>
956+
<goals><goal>test</goal></goals>
957+
<configuration>
958+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_array</test>
959+
</configuration>
960+
</execution>
961+
<execution>
962+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a9_array</id>
963+
<goals><goal>test</goal></goals>
964+
<configuration>
965+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a9_array</test>
966+
</configuration>
967+
</execution>
968+
<execution>
969+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a17_array</id>
970+
<goals><goal>test</goal></goals>
971+
<configuration>
972+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a17_array</test>
973+
</configuration>
974+
</execution>
975+
976+
<!-- LZ4CompressorTest fast-reset compressor (bytebuffer) -->
977+
<execution>
978+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_bytebuffer</id>
979+
<goals><goal>test</goal></goals>
980+
<configuration>
981+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_bytebuffer</test>
982+
</configuration>
983+
</execution>
984+
<execution>
985+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a9_bytebuffer</id>
986+
<goals><goal>test</goal></goals>
987+
<configuration>
988+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a9_bytebuffer</test>
989+
</configuration>
990+
</execution>
991+
<execution>
992+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a17_bytebuffer</id>
993+
<goals><goal>test</goal></goals>
994+
<configuration>
995+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_fastReset_a17_bytebuffer</test>
996+
</configuration>
997+
</execution>
998+
999+
<!-- LZ4CompressorTest high fast-reset compressor (array) -->
1000+
<execution>
1001+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_array</id>
1002+
<goals><goal>test</goal></goals>
1003+
<configuration>
1004+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_array</test>
1005+
</configuration>
1006+
</execution>
1007+
<execution>
1008+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l1_array</id>
1009+
<goals><goal>test</goal></goals>
1010+
<configuration>
1011+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l1_array</test>
1012+
</configuration>
1013+
</execution>
1014+
<execution>
1015+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l9_array</id>
1016+
<goals><goal>test</goal></goals>
1017+
<configuration>
1018+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l9_array</test>
1019+
</configuration>
1020+
</execution>
1021+
<execution>
1022+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l17_array</id>
1023+
<goals><goal>test</goal></goals>
1024+
<configuration>
1025+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l17_array</test>
1026+
</configuration>
1027+
</execution>
1028+
1029+
<!-- LZ4CompressorTest high fast-reset compressor (bytebuffer) -->
1030+
<execution>
1031+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_bytebuffer</id>
1032+
<goals><goal>test</goal></goals>
1033+
<configuration>
1034+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_bytebuffer</test>
1035+
</configuration>
1036+
</execution>
1037+
<execution>
1038+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l1_bytebuffer</id>
1039+
<goals><goal>test</goal></goals>
1040+
<configuration>
1041+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l1_bytebuffer</test>
1042+
</configuration>
1043+
</execution>
1044+
<execution>
1045+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l9_bytebuffer</id>
1046+
<goals><goal>test</goal></goals>
1047+
<configuration>
1048+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l9_bytebuffer</test>
1049+
</configuration>
1050+
</execution>
1051+
<execution>
1052+
<id>fuzz-net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l17_bytebuffer</id>
1053+
<goals><goal>test</goal></goals>
1054+
<configuration>
1055+
<test>net.jpountz.fuzz.LZ4CompressorTest#native_highFastReset_l17_bytebuffer</test>
1056+
</configuration>
1057+
</execution>
1058+
9531059
<!-- XXHash32Test (array) -->
9541060
<execution>
9551061
<id>fuzz-net.jpountz.fuzz.XXHash32Test#safe_array</id>
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package net.jpountz.lz4;
2+
3+
/*
4+
* Copyright 2020 Adrien Grand and the lz4-java contributors.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
import net.jpountz.util.ByteBufferUtils;
20+
import net.jpountz.util.SafeUtils;
21+
22+
import java.nio.ByteBuffer;
23+
import java.util.concurrent.locks.ReentrantLock;
24+
25+
abstract class AbstractLZ4JNIFastResetCompressor extends LZ4Compressor implements AutoCloseable {
26+
27+
private static final String IN_USE_ERROR = "This compressor is not thread-safe and is already in use";
28+
private static final String CLOSED_ERROR = "Compressor has been closed";
29+
private static final String UNSUPPORTED_BUFFER_ERROR = "ByteBuffer must be direct or array-backed";
30+
31+
private final ReentrantLock lock = new ReentrantLock();
32+
private long statePtr;
33+
34+
AbstractLZ4JNIFastResetCompressor(long statePtr, String allocationFailureMessage) {
35+
if (statePtr == 0) {
36+
throw new LZ4Exception(allocationFailureMessage);
37+
}
38+
this.statePtr = statePtr;
39+
}
40+
41+
/**
42+
* Compresses {@code src[srcOff:srcOff+srcLen]} into
43+
* {@code dest[destOff:destOff+maxDestLen]}.
44+
*
45+
* @param src source data
46+
* @param srcOff the start offset in src
47+
* @param srcLen the number of bytes to compress
48+
* @param dest destination buffer
49+
* @param destOff the start offset in dest
50+
* @param maxDestLen the maximum number of bytes to write in dest
51+
* @return the compressed size
52+
* @throws LZ4Exception if maxDestLen is too small
53+
* @throws IllegalStateException if the compressor has been closed or is already in use
54+
*/
55+
@Override
56+
public final int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
57+
if (!lock.tryLock()) {
58+
throw new IllegalStateException(IN_USE_ERROR);
59+
}
60+
61+
try {
62+
long ptr = checkOpen();
63+
SafeUtils.checkRange(src, srcOff, srcLen);
64+
SafeUtils.checkRange(dest, destOff, maxDestLen);
65+
66+
final int result = compressNative(
67+
ptr, src, null, srcOff, srcLen,
68+
dest, null, destOff, maxDestLen);
69+
return checkResult(result);
70+
} finally {
71+
lock.unlock();
72+
}
73+
}
74+
75+
/**
76+
* Compresses {@code src[srcOff:srcOff+srcLen]} into
77+
* {@code dest[destOff:destOff+maxDestLen]}.
78+
* <p>
79+
* Both buffers must be either direct or array-backed.
80+
* {@link ByteBuffer} positions remain unchanged.
81+
*
82+
* @param src source data
83+
* @param srcOff the start offset in src
84+
* @param srcLen the number of bytes to compress
85+
* @param dest destination buffer
86+
* @param destOff the start offset in dest
87+
* @param maxDestLen the maximum number of bytes to write in dest
88+
* @return the compressed size
89+
* @throws LZ4Exception if maxDestLen is too small
90+
* @throws IllegalArgumentException if src or dest is neither array-backed nor direct
91+
* @throws IllegalStateException if the compressor has been closed or is already in use
92+
*/
93+
@Override
94+
public final int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
95+
if (!lock.tryLock()) {
96+
throw new IllegalStateException(IN_USE_ERROR);
97+
}
98+
99+
try {
100+
long ptr = checkOpen();
101+
checkByteBuffer(src);
102+
checkByteBuffer(dest);
103+
ByteBufferUtils.checkNotReadOnly(dest);
104+
ByteBufferUtils.checkRange(src, srcOff, srcLen);
105+
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
106+
107+
byte[] srcArr = src.hasArray() ? src.array() : null;
108+
byte[] destArr = dest.hasArray() ? dest.array() : null;
109+
ByteBuffer srcBuf = srcArr == null ? src : null;
110+
ByteBuffer destBuf = destArr == null ? dest : null;
111+
int srcBufferOff = srcOff + (srcArr != null ? src.arrayOffset() : 0);
112+
int destBufferOff = destOff + (destArr != null ? dest.arrayOffset() : 0);
113+
114+
final int result = compressNative(
115+
ptr, srcArr, srcBuf, srcBufferOff, srcLen,
116+
destArr, destBuf, destBufferOff, maxDestLen);
117+
return checkResult(result);
118+
} finally {
119+
lock.unlock();
120+
}
121+
}
122+
123+
public final boolean isClosed() {
124+
lock.lock();
125+
try {
126+
return statePtr == 0;
127+
} finally {
128+
lock.unlock();
129+
}
130+
}
131+
132+
/**
133+
* Closes this compressor and releases native resources.
134+
* After calling this method, all compress methods will throw {@link IllegalStateException}.
135+
*
136+
* @throws IllegalStateException if the compressor is in use by another thread
137+
*/
138+
@Override
139+
public final void close() {
140+
if (!lock.tryLock()) {
141+
throw new IllegalStateException(IN_USE_ERROR);
142+
}
143+
144+
try {
145+
long ptr = statePtr;
146+
statePtr = 0;
147+
if (ptr != 0) {
148+
freeState(ptr);
149+
}
150+
} finally {
151+
lock.unlock();
152+
}
153+
}
154+
155+
private long checkOpen() {
156+
if (statePtr == 0) {
157+
throw new IllegalStateException(CLOSED_ERROR);
158+
}
159+
return statePtr;
160+
}
161+
162+
private static int checkResult(int result) {
163+
if (result <= 0) {
164+
throw new LZ4Exception("maxDestLen is too small");
165+
}
166+
return result;
167+
}
168+
169+
private static void checkByteBuffer(ByteBuffer buffer) {
170+
if (!(buffer.hasArray() || buffer.isDirect())) {
171+
throw new IllegalArgumentException(UNSUPPORTED_BUFFER_ERROR);
172+
}
173+
}
174+
175+
protected abstract int compressNative(long ptr, byte[] srcArr, ByteBuffer srcBuf, int srcOff, int srcLen,
176+
byte[] destArr, ByteBuffer destBuf, int destOff, int maxDestLen);
177+
178+
protected abstract void freeState(long ptr);
179+
}
180+
181+
182+

src/java/net/jpountz/lz4/LZ4Constants.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ enum LZ4Constants {
2222

2323
static final int DEFAULT_COMPRESSION_LEVEL = 8 + 1;
2424
static final int MAX_COMPRESSION_LEVEL = 16 + 1;
25+
static final int DEFAULT_ACCELERATION = 1;
26+
static final int MIN_ACCELERATION = 1;
27+
static final int MAX_ACCELERATION = 65537;
2528

2629
static final int MEMORY_USAGE = 14;
2730
static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6;

0 commit comments

Comments
 (0)