Skip to content

Commit cd57c31

Browse files
lemirelemire
committed
Some cleaning.
1 parent f002db1 commit cd57c31

16 files changed

Lines changed: 437 additions & 108 deletions

‎Makefile‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,19 @@ ifeq ($(INTEL), 1)
99
CC ?= /opt/intel/bin/icpc
1010
ifeq ($(DEBUG),1)
1111
CXXFLAGS = -fpic -std=c++11 -O3 -Wall -ansi -xAVX -DDEBUG=1 -D_GLIBCXX_DEBUG -ggdb
12-
CCFLAGS = -fpic -std=c99 -O3 -Wall -ansi -xAVX -DDEBUG=1 -D_GLIBCXX_DEBUG -ggdb
12+
CCFLAGS = -fpic -std=c99 -O3 -Wall -ansi -xAVX -DDEBUG=1 -D_GLIBCXX_DEBUG -ggdb
1313
else
14-
CXXFLAGS = -fpic -std=c++11 -O2 -Wall -ansi -xAVX -DNDEBUG=1 -ggdb
15-
CCFLAGS = -fpic -std=c99 -O2 -Wall -ansi -xAVX -DNDEBUG=1 -ggdb
14+
CXXFLAGS = -fpic -std=c++11 -O2 -Wall -ansi -xAVX -DNDEBUG=1 -ggdb
15+
CCFLAGS = -fpic -std=c99 -O2 -Wall -ansi -xAVX -DNDEBUG=1 -ggdb
1616
endif # debug
1717
else #intel
1818
CXX ?= g++-4.7
1919
ifeq ($(DEBUG),1)
20-
CXXFLAGS = -fpic -mavx -std=c++11 -Weffc++ -pedantic -ggdb -DDEBUG=1 -D_GLIBCXX_DEBUG -Wall -Wextra
21-
CCFLAGS = -fpic -mavx -std=c99 -pedantic -ggdb -DDEBUG=1 -D_GLIBCXX_DEBUG -Wall -Wextra
20+
CXXFLAGS = -fpic -mavx -std=c++11 -Weffc++ -pedantic -ggdb -DDEBUG=1 -D_GLIBCXX_DEBUG -Wall -Wextra -Wextra -Wsign-compare -Wwrite-strings -Wpointer-arith -Winit-self -Wno-sign-conversion
21+
CCFLAGS = -fpic -mavx -std=c99 -pedantic -ggdb -DDEBUG=1 -D_GLIBCXX_DEBUG -Wall -Wextra -Wsign-compare -Wwrite-strings -Wpointer-arith -Winit-self -Wno-sign-conversion
2222
else
23-
CXXFLAGS = -fpic -mavx -std=c++11 -Weffc++ -pedantic -O3 -Wall -Wextra
24-
CCFLAGS = -fpic -mavx -std=c99 -pedantic -O3 -Wall -Wextra
23+
CXXFLAGS = -fpic -mavx -std=c++11 -Weffc++ -pedantic -O3 -Wall -Wextra -Wsign-compare -Wwrite-strings -Wpointer-arith -Winit-self -Wno-sign-conversion
24+
CCFLAGS = -fpic -mavx -std=c99 -pedantic -O3 -Wall -Wextra -Wsign-compare -Wwrite-strings -Wpointer-arith -Winit-self -Wno-sign-conversion
2525
endif #debug
2626
endif #intel
2727

‎advancedbenchmarking/src/budgetedtest.cpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -190,10 +190,10 @@ class TestHelper {
190190
if (recbuffer != uncompressed) {
191191
cout << "size = " << uncompressed.size() << endl;
192192
int display = 10;
193-
for (size_t i = 0; i < uncompressed.size(); ++i) {
194-
if (uncompressed[i] != recbuffer[i]) {
195-
cout << " i = " << i << " expected " << uncompressed[i]
196-
<< " but got " << recbuffer[i] << endl;
193+
for (size_t ii = 0; ii < uncompressed.size(); ++ii) {
194+
if (uncompressed[ii] != recbuffer[ii]) {
195+
cout << " i = " << ii << " expected " << uncompressed[ii]
196+
<< " but got " << recbuffer[ii] << endl;
197197
display--;
198198
if (display == 0)
199199
break;
@@ -581,7 +581,7 @@ class TestHelper {
581581
vector<uint32_t> &onePost = uncompPosts.getOnePost(id);
582582
z.reset();
583583
compPostings.emplace(id, shared_ptr<Skipping>(new Skipping(
584-
SkipLog, onePost.data(), onePost.size())));
584+
SkipLog, onePost.data(), (uint32_t) onePost.size())));
585585
packTime += static_cast<double>(z.split());
586586
size_t qty = onePost.size();
587587
if (MaxPostingSize < qty) {
@@ -622,7 +622,7 @@ class TestHelper {
622622
for (size_t k = 2; (intersize > 0) && (k < sizeids.size()); ++k) {
623623
unpackVolume += compPostings[sizeids[k].second]->Length;
624624
intersize = compPostings[sizeids[k].second]->intersect(
625-
intersection_result.data(), intersize,
625+
intersection_result.data(), (uint32_t) intersize,
626626
intersection_result.data());
627627
}
628628
}
@@ -705,7 +705,7 @@ class TestHelper {
705705

706706
z.reset(); // actually this should be very quick
707707
CompressedSizeDuringPacking += hybridPart[part]->load(
708-
id, emptyPost.data(), emptyPost.size());
708+
id, emptyPost.data(), (uint32_t) emptyPost.size());
709709
packTime += static_cast<double>(z.split());
710710

711711
packVolume += 0;
@@ -755,7 +755,7 @@ class TestHelper {
755755
}
756756

757757
CompressedSizeDuringPacking +=
758-
hybridPart[part]->load(id, dirtyCopy.data(), dirtyCopy.size());
758+
hybridPart[part]->load(id, dirtyCopy.data(), (uint32_t) dirtyCopy.size());
759759
packTime += static_cast<double>(z.split());
760760

761761
packVolume += thissize;
@@ -861,7 +861,7 @@ class TestHelper {
861861
vector<uint32_t> &onePost = uncompPosts.getOnePost(id);
862862
z.reset();
863863
CompressedSizeDuringPacking +=
864-
hybrid.load(id, onePost.data(), onePost.size());
864+
hybrid.load(id, onePost.data(), (uint32_t) onePost.size());
865865
packTime += static_cast<double>(z.split());
866866
size_t qty = onePost.size();
867867
packVolume += qty;
@@ -920,7 +920,7 @@ class TestHelper {
920920
vector<uint32_t> &onePost = uncompPosts.getOnePost(id);
921921
z.reset();
922922
CompressedSizeDuringPacking +=
923-
hybrid.load(id, onePost.data(), onePost.size());
923+
hybrid.load(id, onePost.data(), (uint32_t) onePost.size());
924924
packTime += static_cast<double>(z.split());
925925
size_t qty = onePost.size();
926926
packVolume += qty;
@@ -977,7 +977,7 @@ class TestHelper {
977977
vector<uint32_t> &onePost = uncompPosts.getOnePost(id);
978978
z.reset();
979979
CompressedSizeDuringPacking +=
980-
hybrid.load(id, onePost.data(), onePost.size());
980+
hybrid.load(id, onePost.data(), (uint32_t) onePost.size());
981981
packTime += static_cast<double>(z.split());
982982
size_t qty = onePost.size();
983983
packVolume += qty;

‎advancedbenchmarking/src/ramtocache.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ void blockedcompress(shared_ptr<IntegerCODEC> c, vector<uint32_t> &buffer,
6767
l = blocksize;
6868
size_t lo = obuffer.size() - outpos;
6969
c->encodeArray(buffer.data() + inpos, l, obuffer.data() + outpos, lo);
70-
obuffer[outcounter++] = lo; // saving compressed length
70+
obuffer[outcounter++] = (uint32_t)lo; // saving compressed length
7171
outpos += lo;
7272
inpos += l;
7373
}
@@ -155,9 +155,9 @@ void processArray(map<pair<uint32_t, shared_ptr<IntegerCODEC>>, stats> &mystats,
155155
throw runtime_error("failed to recover right length");
156156
for (size_t k = 0; k < l; ++k)
157157
if (l1buf[k] != buffer[k + inpos]) {
158-
for (size_t K = 0; K < l; ++K)
159-
cout << K << ": expected is " << buffer[K + inpos]
160-
<< ", actual is " << l1buf[K] << endl;
158+
for (size_t KK = 0; KK < l; ++KK)
159+
cout << KK << ": expected is " << buffer[KK + inpos]
160+
<< ", actual is " << l1buf[KK] << endl;
161161
throw runtime_error("bug decoded values do not match");
162162
}
163163
inpos += l;

‎example.cpp‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,19 @@
1414

1515
using namespace SIMDCompressionLib;
1616

17+
1718
int main() {
1819
// We pick a CODEC
19-
IntegerCODEC &codec = *CODECFactory::getFromName("s4-bp128-dm");
20-
// could use others, e.g., "varint", "s-fastpfor-1"
20+
IntegerCODEC &codec = *CODECFactory::getFromName("s4-fastpfor-d1");
21+
// could use others, e.g., frameofreference, ibp32, maskedvbyte, s4-bp128-d1, s4-bp128-d2, s4-bp128-d4, s4-bp128-dm, simdframeofreference, streamvbyte
22+
//
23+
// Note that some codecs compute the differential coding in-place, thus modifying part of the input, replacing it with a differentially coded version:
24+
// bp32, fastpfor, s4-bp128-d1-ni, s4-bp128-d2-ni, s4-bp128-d4-ni, s4-bp128-dm-ni, s4-fastpfor-d1, s4-fastpfor-d2, s4-fastpfor-d4, s4-fastpfor-dm
25+
// Other codecs do the differential coding "in passing", such as
26+
// for, frameofreference, ibp32, maskedvbyte, s4-bp128-d1, s4-bp128-d2, s4-bp128-d4, s4-bp128-dm, simdframeofreference, streamvbyte, varint, varintg8iu, varintgb, vbyte
27+
//
28+
29+
2130
////////////
2231
//
2332
// create a container with some integers in it
@@ -30,6 +39,11 @@ int main() {
3039
vector<uint32_t> mydata(N);
3140
for (uint32_t i = 0; i < N; ++i)
3241
mydata[i] = 3 * i;
42+
43+
44+
45+
// we make a copy
46+
std::vector<uint32_t> original_data(mydata);
3347
///////////
3448
//
3549
// You need some "output" container. You are responsible
@@ -67,7 +81,7 @@ int main() {
6781
//
6882
// That's it for compression!
6983
//
70-
if (mydataback != mydata)
84+
if (mydataback != original_data)
7185
throw runtime_error("bug!");
7286

7387
//

‎include/hybm2.h‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* This is an implementation of the hyb+m2 method proposed in:
33
*
44
* J. S. Culpepper and A. Moffat. Efficient set intersection for
5-
* inverted indexing. ACM Trans. Inf. Syst., 29(1):1:1Ð1:25, Dec. 2010.
5+
* inverted indexing. ACM Trans. Inf. Syst., 29(1):1:1�1:25, Dec. 2010.
66
*
77
* Implemented by Daniel Lemire
88
*/
@@ -139,11 +139,11 @@ class HybM2 {
139139
unpackVolume += bitmaps[i].first;
140140
shared_ptr<BoolArray> &ba = bitmaps[i].second;
141141
pos = 0;
142-
for (uint32_t i = 0; i < sizeout; ++i) {
143-
if (!ba->get(out[i]))
142+
for (uint32_t ii = 0; ii < sizeout; ++ii) {
143+
if (!ba->get(out[ii]))
144144
continue;
145145
else
146-
out[pos++] = out[i];
146+
out[pos++] = out[ii];
147147
}
148148
sizeout = pos;
149149
}
@@ -373,11 +373,11 @@ class UncompressedHybM2 {
373373
unpackVolume += bitmaps[i].first;
374374
shared_ptr<BoolArray> &ba = bitmaps[i].second;
375375
pos = 0;
376-
for (uint32_t i = 0; i < sizeout; ++i) {
377-
if (!ba->get(out[i]))
376+
for (uint32_t ii = 0; ii < sizeout; ++ii) {
377+
if (!ba->get(out[ii]))
378378
continue;
379379
else
380-
out[pos++] = out[i];
380+
out[pos++] = out[ii];
381381
}
382382
sizeout = pos;
383383
}
@@ -557,19 +557,19 @@ class SkippingHybM2 {
557557
sizeout = shortlists[0].second->intersect(*shortlists[1].second, out);
558558
for (uint32_t i = 2; (sizeout > 0) && (i < shortlists.size()); ++i) {
559559
unpackVolume += shortlists[i].first;
560-
sizeout = shortlists[i].second->intersect(out, sizeout, out);
560+
sizeout = shortlists[i].second->intersect(out, (uint32_t)sizeout, out);
561561
}
562562
}
563563
size_t pos = 0;
564564
for (uint32_t i = 0; (sizeout > 0) && (i < bitmaps.size()); ++i) {
565565
unpackVolume += bitmaps[i].first;
566566
shared_ptr<BoolArray> &ba = bitmaps[i].second;
567567
pos = 0;
568-
for (uint32_t i = 0; i < sizeout; ++i) {
569-
if (!ba->get(out[i]))
568+
for (uint32_t ii = 0; ii < sizeout; ++ii) {
569+
if (!ba->get(out[ii]))
570570
continue;
571571
else
572-
out[pos++] = out[i];
572+
out[pos++] = out[ii];
573573
}
574574
sizeout = pos;
575575
}

‎include/streamvariablebyte.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ class StreamVByteD1 : public IntegerCODEC {
9090
static_cast<uint32_t>(std::min<size_t>(
9191
count, std::numeric_limits<uint32_t>::max())),
9292
1, 1));
93-
*out = 4 + bytesWritten;
93+
*out = uint8_t(4 + bytesWritten);
9494
nvalue = 4 + bytesWritten;
9595
}
9696

‎include/synthetic.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,10 +216,10 @@ class ZipfianGenerator {
216216
init(_items, _zipfianconstant);
217217
}
218218

219-
double zeta(int n, double theta) {
219+
double zeta(int nn, double ttheta) {
220220
double sum = 0;
221-
for (long i = 0; i < n; i++) {
222-
sum += 1 / (pow(i + 1, theta));
221+
for (long i = 0; i < nn; i++) {
222+
sum += 1 / (pow(i + 1, ttheta));
223223
}
224224
return sum;
225225
}

‎include/varintgb.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ template <bool delta = false> class VarIntGB : public IntegerCODEC {
270270
} else {
271271
memcpy(inbyte, block1->data, block1->length);
272272
inbyte += block1->length;
273-
inbyte[0] = newsel;
273+
inbyte[0] = (uint8_t)newsel;
274274
inbyte++;
275275
memcpy(inbyte, &nextval, newsel + 1);
276276
inbyte += newsel + 1;
@@ -904,7 +904,7 @@ template <bool delta = false> class VarIntGB : public IntegerCODEC {
904904
newnextval &= mask[newnextsel];
905905
// uint32_t newnextval = *(reinterpret_cast<const uint32_t*>(b->data +
906906
// offsettolastval)) & mask[newnextsel];
907-
b->data[0] = (b->data[0] << 2) | *newsel;
907+
b->data[0] = uint8_t((b->data[0] << 2) | *newsel);
908908
std::memmove(b->data + 2 + *newsel, b->data + 1,
909909
b->length - 1 - 1 - newnextsel);
910910
b->length = offsettolastval + 1 + *newsel;
@@ -915,7 +915,7 @@ template <bool delta = false> class VarIntGB : public IntegerCODEC {
915915

916916
void finalshiftin(Block *b, uint32_t nextval, uint32_t newsel,
917917
size_t howmany) {
918-
b->data[0] = (b->data[0] << 2) | newsel;
918+
b->data[0] = uint8_t((b->data[0] << 2) | newsel);
919919
std::memmove(b->data + 2 + newsel, b->data + 1, b->length - 1);
920920
b->length = 1;
921921
for (size_t k = 0; k < howmany; ++k)

‎src/benchintersection.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -617,7 +617,7 @@ size_t intersect_partitionedV1(const uint16_t *A, const size_t s_a,
617617
size_t partition_size = _intersectV1_vector16(
618618
&A[i_a + 2], &B[i_b + 2], static_cast<size_t>(A[i_a + 1]) + 1,
619619
static_cast<size_t>(B[i_b + 1]) + 1, &C[counter + 1]);
620-
C[counter++] = partition_size; // write partition size
620+
C[counter++] = (uint16_t) partition_size; // write partition size
621621
counter += partition_size;
622622
i_a += static_cast<size_t>(A[i_a + 1]) + 2 + 1;
623623
i_b += static_cast<size_t>(B[i_b + 1]) + 2 + 1;
@@ -650,7 +650,7 @@ size_t intersect_partitionedscalar(const uint16_t *A, const size_t s_a,
650650
size_t partition_size = _intersectscalar_vector16(
651651
&A[i_a + 2], &B[i_b + 2], static_cast<size_t>(A[i_a + 1]) + 1,
652652
static_cast<size_t>(B[i_b + 1]) + 1, &C[counter + 1]);
653-
C[counter++] = partition_size; // write partition size
653+
C[counter++] = (uint16_t) partition_size; // write partition size
654654
counter += partition_size;
655655
i_a += static_cast<size_t>(A[i_a + 1]) + 2 + 1;
656656
i_b += static_cast<size_t>(B[i_b + 1]) + 2 + 1;
@@ -685,7 +685,7 @@ size_t intersect_partitioned(const uint16_t *A, const size_t s_a,
685685
size_t partition_size = _intersect_vector16(
686686
&A[i_a + 2], &B[i_b + 2], static_cast<size_t>(A[i_a + 1]) + 1,
687687
static_cast<size_t>(B[i_b + 1]) + 1, &C[counter + 1]);
688-
C[counter++] = partition_size; // write partition size
688+
C[counter++] = (uint16_t) partition_size; // write partition size
689689
counter += partition_size;
690690
i_a += static_cast<size_t>(A[i_a + 1]) + 2 + 1;
691691
i_b += static_cast<size_t>(B[i_b + 1]) + 2 + 1;
@@ -717,7 +717,7 @@ size_t original_intersect_partitioned(const uint16_t *A, const size_t s_a,
717717
size_t partition_size = _original_intersect_vector16(
718718
&A[i_a + 2], &B[i_b + 2], static_cast<size_t>(A[i_a + 1]) + 1,
719719
static_cast<size_t>(B[i_b + 1]) + 1, &C[counter + 1]);
720-
C[counter++] = partition_size; // write partition size
720+
C[counter++] = (uint16_t) partition_size; // write partition size
721721
counter += partition_size;
722722
i_a += static_cast<size_t>(A[i_a + 1]) + 2 + 1;
723723
i_b += static_cast<size_t>(B[i_b + 1]) + 2 + 1;
@@ -754,7 +754,7 @@ int main(int argc, char **argv) {
754754
Big = atoi(optarg);
755755
break;
756756
case 'R':
757-
intersectionratio = atof(optarg);
757+
intersectionratio = (float)atof(optarg);
758758
break;
759759
case 'M':
760760
MaxBit = atoi(optarg);
@@ -810,16 +810,16 @@ int main(int argc, char **argv) {
810810
"16-bitscalar ";
811811
cout << "relative-intersection-size " << endl;
812812

813-
for (float ir = 1.001; ir <= 10000; ir = ir * sqrt(1.9)) {
813+
for (double ir = 1.001; ir <= 10000; ir = ir * sqrt(1.9)) {
814814
vector<pair<vector<uint32_t>, vector<uint32_t>>> data(howmany);
815815
uint32_t smallsize =
816816
static_cast<uint32_t>(round(static_cast<float>(1 << Big) / ir));
817817
cout << "#generating data...";
818818
cout.flush();
819819
for (size_t k = 0; k < howmany; ++k) {
820-
data[k] = uniform ? getNaivePair(udg, smallsize, 1U << MaxBit, ir,
820+
data[k] = uniform ? getNaivePair(udg, smallsize, 1U << MaxBit, (float)ir,
821821
intersectionratio)
822-
: getNaivePair(cdg, smallsize, 1U << MaxBit, ir,
822+
: getNaivePair(cdg, smallsize, 1U << MaxBit, (float)ir,
823823
intersectionratio);
824824
}
825825
cout << "ok." << endl;
@@ -858,7 +858,7 @@ int main(int argc, char **argv) {
858858
aratio = interfnc(data[k].first.data(), (data[k].first).size(),
859859
data[k].second.data(), (data[k].second).size(),
860860
buffer.data());
861-
bogus += aratio;
861+
bogus += size_t(aratio);
862862
}
863863
}
864864
cout << setw(10) << setprecision(5)
@@ -876,7 +876,7 @@ int main(int argc, char **argv) {
876876
datapart[k].first.data(), (datapart[k].first).size(),
877877
datapart[k].second.data(), (datapart[k].second).size(),
878878
(uint16_t *)buffer.data());
879-
bogus += aratio;
879+
bogus += size_t(aratio);
880880
}
881881
}
882882
cout << setw(10) << setprecision(5)
@@ -890,7 +890,7 @@ int main(int argc, char **argv) {
890890
datapart[k].first.data(), (datapart[k].first).size(),
891891
datapart[k].second.data(), (datapart[k].second).size(),
892892
(uint16_t *)buffer.data());
893-
bogus += aratio;
893+
bogus += size_t(aratio);
894894
}
895895
}
896896
cout << setw(10) << setprecision(5)
@@ -904,7 +904,7 @@ int main(int argc, char **argv) {
904904
datapart[k].first.data(), (datapart[k].first).size(),
905905
datapart[k].second.data(), (datapart[k].second).size(),
906906
(uint16_t *)buffer.data());
907-
bogus += aratio;
907+
bogus += size_t(aratio);
908908
}
909909
}
910910
cout << setw(10) << setprecision(5)
@@ -918,7 +918,7 @@ int main(int argc, char **argv) {
918918
datapart[k].first.data(), (datapart[k].first).size(),
919919
datapart[k].second.data(), (datapart[k].second).size(),
920920
(uint16_t *)buffer.data());
921-
bogus += aratio;
921+
bogus += size_t(aratio);
922922
}
923923
}
924924
cout << setw(10) << setprecision(5)

0 commit comments

Comments
 (0)