diff --git a/src/xor.h b/src/xor.h index 7997572..2c2ecc4 100644 --- a/src/xor.h +++ b/src/xor.h @@ -73,6 +73,7 @@ struct XorView { void addElem(uint64_t createdAt, std::string_view id) { elems.emplace_back(createdAt, id, idSize); + std::cerr << "ADDELEM " << to_hex(elems.back().getCompare(idSize)) << std::endl; } void finalise() { @@ -81,18 +82,21 @@ struct XorView { std::sort(elems.begin(), elems.end(), [&](const auto &a, const auto &b) { return a.getCompare(idSize) < b.getCompare(idSize); }); ready = true; + + for (auto &e : elems) std::cerr << "FIN " << to_hex(e.getCompare(idSize)) << std::endl; } std::string initialQuery() { if (!ready) throw herr("xor view not ready"); std::string output; - splitRange(elems.begin(), elems.end(), output); + splitRange(elems.begin(), elems.end(), std::string(1, '\x00'), std::string(1, '\xFF'), output); return output; } // FIXME: try/catch everywhere that calls this std::string handleQuery(std::string_view query, std::vector &haveIds, std::vector &needIds) { + std::cerr << "-------------------" << std::endl; if (!ready) throw herr("xor view not ready"); std::string output; @@ -100,34 +104,46 @@ struct XorView { auto cmp = [&](const auto &a, const auto &b){ return a.getCompare(idSize) < b.getCompare(idSize); }; while (query.size()) { + std::cerr << "=========" << std::endl; uint64_t lowerLength = decodeVarInt(query); - if (lowerLength > idSize + 5) throw herr("lower too long"); - XorElem lowerKey(getBytes(query, lowerLength)); + if (lowerLength > idSize + 5) throw herr("lower too long: ", lowerLength); + auto lowerKeyRaw = getBytes(query, lowerLength); + XorElem lowerKey(lowerKeyRaw); + std::cerr << "LWCMP: " << to_hex(lowerKey.getCompare(idSize)) << std::endl; uint64_t upperLength = decodeVarInt(query); if (upperLength > idSize + 5) throw herr("upper too long"); - XorElem upperKey(getBytes(query, upperLength)); + auto upperKeyRaw = getBytes(query, upperLength); + XorElem upperKey(upperKeyRaw); + std::cerr << "UPCMP: " << to_hex(upperKey.getCompare(idSize)) << std::endl; auto lower = std::lower_bound(elems.begin(), elems.end(), lowerKey, cmp); // FIXME: start at prev upper? auto upper = std::upper_bound(elems.begin(), elems.end(), upperKey, cmp); // FIXME: start at lower? + std::cerr << "FOUND: " << size_t(upper-lower) << std::endl; + for (auto it = lower; it < upper; ++it) std::cerr << "MMM: " << to_hex(it->getId(idSize)) << std::endl; uint64_t mode = decodeVarInt(query); // 0 = range, 8 and above = n-8 inline IDs - std::cerr << "BING MODE = " << mode << std::endl; if (mode == 0) { + std::cerr << "MODE 0" << std::endl; XorElem theirXorSet(0, getBytes(query, idSize), idSize); XorElem ourXorSet; for (auto i = lower; i < upper; ++i) ourXorSet.doXor(*i); - if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, output); + std::cerr << "XSETS " << to_hex(theirXorSet.getId(idSize)) << " / " << to_hex(ourXorSet.getId(idSize)) << std::endl; + if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, lowerKeyRaw, upperKeyRaw, output); } else if (mode >= 8) { + std::cerr << "MODE " << (mode - 8) << std::endl; flat_hash_map theirElems; for (uint64_t i = 0; i < mode - 8; i++) { - theirElems.emplace(XorElem(0, getBytes(query, idSize), idSize), false); + auto bb = getBytes(query, idSize); + std::cerr << "INSERTED THEIR " << to_hex(bb) << std::endl; + theirElems.emplace(XorElem(0, bb, idSize), false); } for (auto it = lower; it < upper; ++it) { + std::cerr << "SEARCHING " << to_hex(it->getId(idSize)) << std::endl; auto e = theirElems.find(*it); if (e == theirElems.end()) { @@ -136,6 +152,7 @@ struct XorView { } else { // ID exists on both sides e->second = true; + std::cerr << "ERMM " << to_hex(e->first.getId(idSize)) << std::endl; } } @@ -153,41 +170,45 @@ struct XorView { private: - void splitRange(std::vector::iterator lower, std::vector::iterator upper, std::string &output) { + void splitRange(std::vector::iterator lower, std::vector::iterator upper, const std::string &lowerKey, const std::string &upperKey, std::string &output) { // Split our range uint64_t numElems = upper - lower; const uint64_t buckets = 16; if (numElems < buckets * 2) { - appendBoundKey(getLowerKey(lower), output); - appendBoundKey(getUpperKey(upper), output); + std::cerr << "DUMPING IDs" << std::endl; + appendBoundKey(lowerKey /*getLowerKey(lower)*/, output); + appendBoundKey(upperKey /*getUpperKey(upper)*/, output); output += encodeVarInt(numElems + 8); for (auto it = lower; it < upper; ++it) output += it->getId(idSize); + for (auto it = lower; it < upper; ++it) std::cerr << "DUMP ID: " << to_hex(it->getId(idSize)) << std::endl; } else { + std::cerr << "DOING SPLIT" << std::endl; uint64_t elemsPerBucket = numElems / buckets; uint64_t bucketsWithExtra = numElems % buckets; auto curr = lower; for (uint64_t i = 0; i < buckets; i++) { - appendBoundKey(getLowerKey(curr), output); - - auto bucketEnd = curr + elemsPerBucket; - if (i < bucketsWithExtra) bucketEnd++; + appendBoundKey(i == 0 ? lowerKey : getLowerKey(curr), output); XorElem ourXorSet; for (auto bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) { + std::cerr << "XORING IN " << to_hex(curr->getId(idSize)) << std::endl; ourXorSet.doXor(*curr); } - appendBoundKey(getUpperKey(curr), output); + appendBoundKey(i == buckets - 1 ? upperKey : getUpperKey(curr), output); + output += encodeVarInt(0); // mode = 0 output += ourXorSet.getId(idSize); + std::cerr << "FULL XOR " << to_hex(ourXorSet.getId(idSize)) << std::endl; } } } void appendBoundKey(std::string k, std::string &output) { + std::cerr << "ABK: " << to_hex(k) << std::endl; output += encodeVarInt(k.size()); output += k; } diff --git a/test/xor.cpp b/test/xor.cpp index abb5dd8..804c75c 100644 --- a/test/xor.cpp +++ b/test/xor.cpp @@ -1,31 +1,128 @@ #include +#include #include "golpe.h" #include "xor.h" -int main() { - XorView x1(16); - x1.addElem(1000, std::string(16, 'a')); - x1.addElem(2000, std::string(16, 'b')); - x1.finalise(); - XorView x2(16); - x2.addElem(2000, std::string(16, 'b')); - x2.addElem(3000, std::string(16, 'c')); +std::vector split(const std::string &s, char delim) { + std::vector result; + std::stringstream ss (s); + std::string item; + + while (getline (ss, item, delim)) { + result.push_back (item); + } + + return result; +} + + + +int main() { + const uint64_t idSize = 16; + + // x1 is client, x2 is relay + XorView x1(idSize); + XorView x2(idSize); + + std::set ids1; + std::set ids2; + + std::string line; + while (std::cin) { + std::getline(std::cin, line); + if (!line.size()) continue; + + auto items = split(line, ','); + if (items.size() != 3) throw herr("too few items"); + + int mode = std::stoi(items[0]); + uint64_t created = std::stoull(items[1]); + auto id = from_hex(items[2]); + if (id.size() != idSize) throw herr("unexpected id size"); + + if (mode == 1) { + x1.addElem(created, id); + ids1.insert(id); + } else if (mode == 2) { + x2.addElem(created, id); + ids2.insert(id); + } else if (mode == 3) { + x1.addElem(created, id); + x2.addElem(created, id); + ids1.insert(id); + ids2.insert(id); + } else { + throw herr("unexpected mode"); + } + + if (mode == 1) std::cerr << "CLIENT-ONLY: " << to_hex(id) << std::endl; + if (mode == 2) std::cerr << "RELAY-ONLY : " << to_hex(id) << std::endl; + if (mode == 3) std::cerr << "BOTH : " << to_hex(id) << std::endl; + + } + + std::cerr << "BEGIN RECONCILATION" << std::endl; + + x1.finalise(); x2.finalise(); - { - auto q = x1.initialQuery(); - std::cout << to_hex(q) << std::endl; + std::string q = x1.initialQuery(); - std::vector have, need; - auto q2 = x2.handleQuery(q, have, need); + uint64_t round = 0; - for (auto &s : have) std::cout << "HAVE: " << to_hex(s) << std::endl; - for (auto &s : need) std::cout << "NEED: " << to_hex(s) << std::endl; - std::cout << to_hex(q2) << std::endl; + while (q.size()) { + round++; + std::cerr << "ROUND A " << round << std::endl; + std::cerr << "CLIENT -> RELAY" << std::endl; + { + std::vector have, need; + q = x2.handleQuery(q, have, need); + + // q and have are returned to client + for (auto &id : have) { + ids1.insert(id); + std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl; + } + for (auto &id : need) { + ids2.insert(id); + std::cerr << "ADD RELAY: " << to_hex(id) << std::endl; + } + } + + if (q.size()) { + std::cerr << "ROUND B " << round << std::endl; + std::cerr << "RELAY -> CLIENT" << std::endl; + std::vector have, need; + q = x1.handleQuery(q, have, need); + + for (auto &id : need) { + ids1.insert(id); + std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl; + } + for (auto &id : have) { + ids2.insert(id); + std::cerr << "ADD RELAY: " << to_hex(id) << std::endl; + } + } } + if (ids1 != ids2) { + for (const auto &id : ids1) if (!ids2.contains(id)) std::cerr << "In CLIENT not RELAY: " << to_hex(id) << std::endl; + for (const auto &id : ids2) if (!ids1.contains(id)) std::cerr << "In RELAY not CLIENT: " << to_hex(id) << std::endl; + throw herr("mismatch"); + } + +/* + std::vector have, need; + auto q2 = x2.handleQuery(q, have, need); + + for (auto &s : have) std::cout << "HAVE: " << to_hex(s) << std::endl; + for (auto &s : need) std::cout << "NEED: " << to_hex(s) << std::endl; + std::cout << to_hex(q2) << std::endl; +*/ + return 0; } diff --git a/test/xorTest.pl b/test/xorTest.pl new file mode 100644 index 0000000..e7de36e --- /dev/null +++ b/test/xorTest.pl @@ -0,0 +1,33 @@ +#!/usr/bin/env perl + +use IPC::Open2; +use Session::Token; + +my $idSize = 16; + +srand($ENV{SEED} || 0); +my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2); + +my $pid = open2(my $outfile, my $infile, './test/xor'); + +my $num = rnd(10000) + 1; + +for (1..$num) { + my $mode = rnd(3) + 1; + my $created = 1677970534 + rnd($num); + my $id = $stgen->get; + print $infile "$mode,$created,$id\n"; +} + +close($infile); + +while (<$outfile>) { + print $_; +} + + + +sub rnd { + my $n = shift; + return int(rand() * $n); +}