From 2f59f8ffa3e1aece59eb72ed8e8a5c5d0e9b2898 Mon Sep 17 00:00:00 2001 From: Doug Hoyte Date: Sun, 5 Mar 2023 16:35:43 -0500 Subject: [PATCH] wip --- src/xor.h | 39 ++++----------------------------------- test/xor.cpp | 31 ++++++++----------------------- test/xorTest.pl | 41 +++++++++++++++++++++++++++-------------- 3 files changed, 39 insertions(+), 72 deletions(-) diff --git a/src/xor.h b/src/xor.h index 2c2ecc4..52cd19d 100644 --- a/src/xor.h +++ b/src/xor.h @@ -73,7 +73,6 @@ struct XorView { void addElem(uint64_t createdAt, std::string_view id) { elems.emplace_back(createdAt, id, idSize); - std::cerr << "ADDELEM " << to_hex(elems.back().getCompare(idSize)) << std::endl; } void finalise() { @@ -82,8 +81,6 @@ struct XorView { std::sort(elems.begin(), elems.end(), [&](const auto &a, const auto &b) { return a.getCompare(idSize) < b.getCompare(idSize); }); ready = true; - - for (auto &e : elems) std::cerr << "FIN " << to_hex(e.getCompare(idSize)) << std::endl; } std::string initialQuery() { @@ -96,7 +93,6 @@ struct XorView { // FIXME: try/catch everywhere that calls this std::string handleQuery(std::string_view query, std::vector &haveIds, std::vector &needIds) { - std::cerr << "-------------------" << std::endl; if (!ready) throw herr("xor view not ready"); std::string output; @@ -104,46 +100,36 @@ struct XorView { auto cmp = [&](const auto &a, const auto &b){ return a.getCompare(idSize) < b.getCompare(idSize); }; while (query.size()) { - std::cerr << "=========" << std::endl; uint64_t lowerLength = decodeVarInt(query); if (lowerLength > idSize + 5) throw herr("lower too long: ", lowerLength); auto lowerKeyRaw = getBytes(query, lowerLength); XorElem lowerKey(lowerKeyRaw); - std::cerr << "LWCMP: " << to_hex(lowerKey.getCompare(idSize)) << std::endl; uint64_t upperLength = decodeVarInt(query); if (upperLength > idSize + 5) throw herr("upper too long"); auto upperKeyRaw = getBytes(query, upperLength); XorElem upperKey(upperKeyRaw); - std::cerr << "UPCMP: " << to_hex(upperKey.getCompare(idSize)) << std::endl; auto lower = std::lower_bound(elems.begin(), elems.end(), lowerKey, cmp); // FIXME: start at prev upper? auto upper = std::upper_bound(elems.begin(), elems.end(), upperKey, cmp); // FIXME: start at lower? - std::cerr << "FOUND: " << size_t(upper-lower) << std::endl; - for (auto it = lower; it < upper; ++it) std::cerr << "MMM: " << to_hex(it->getId(idSize)) << std::endl; uint64_t mode = decodeVarInt(query); // 0 = range, 8 and above = n-8 inline IDs if (mode == 0) { - std::cerr << "MODE 0" << std::endl; XorElem theirXorSet(0, getBytes(query, idSize), idSize); XorElem ourXorSet; for (auto i = lower; i < upper; ++i) ourXorSet.doXor(*i); - std::cerr << "XSETS " << to_hex(theirXorSet.getId(idSize)) << " / " << to_hex(ourXorSet.getId(idSize)) << std::endl; if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, lowerKeyRaw, upperKeyRaw, output); } else if (mode >= 8) { - std::cerr << "MODE " << (mode - 8) << std::endl; flat_hash_map theirElems; for (uint64_t i = 0; i < mode - 8; i++) { auto bb = getBytes(query, idSize); - std::cerr << "INSERTED THEIR " << to_hex(bb) << std::endl; theirElems.emplace(XorElem(0, bb, idSize), false); } for (auto it = lower; it < upper; ++it) { - std::cerr << "SEARCHING " << to_hex(it->getId(idSize)) << std::endl; auto e = theirElems.find(*it); if (e == theirElems.end()) { @@ -152,7 +138,6 @@ struct XorView { } else { // ID exists on both sides e->second = true; - std::cerr << "ERMM " << to_hex(e->first.getId(idSize)) << std::endl; } } @@ -176,53 +161,37 @@ struct XorView { const uint64_t buckets = 16; if (numElems < buckets * 2) { - std::cerr << "DUMPING IDs" << std::endl; - appendBoundKey(lowerKey /*getLowerKey(lower)*/, output); - appendBoundKey(upperKey /*getUpperKey(upper)*/, output); + appendBoundKey(lowerKey, output); + appendBoundKey(upperKey, output); output += encodeVarInt(numElems + 8); for (auto it = lower; it < upper; ++it) output += it->getId(idSize); - for (auto it = lower; it < upper; ++it) std::cerr << "DUMP ID: " << to_hex(it->getId(idSize)) << std::endl; } else { - std::cerr << "DOING SPLIT" << std::endl; uint64_t elemsPerBucket = numElems / buckets; uint64_t bucketsWithExtra = numElems % buckets; auto curr = lower; for (uint64_t i = 0; i < buckets; i++) { - appendBoundKey(i == 0 ? lowerKey : getLowerKey(curr), output); + appendBoundKey(i == 0 ? lowerKey : minimalKeyDiff(curr->getCompare(idSize), std::prev(curr)->getCompare(idSize)), output); XorElem ourXorSet; for (auto bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) { - std::cerr << "XORING IN " << to_hex(curr->getId(idSize)) << std::endl; ourXorSet.doXor(*curr); } - appendBoundKey(i == buckets - 1 ? upperKey : getUpperKey(curr), output); + appendBoundKey(i == buckets - 1 ? upperKey : minimalKeyDiff(curr->getCompare(idSize), std::prev(curr)->getCompare(idSize)), output); output += encodeVarInt(0); // mode = 0 output += ourXorSet.getId(idSize); - std::cerr << "FULL XOR " << to_hex(ourXorSet.getId(idSize)) << std::endl; } } } void appendBoundKey(std::string k, std::string &output) { - std::cerr << "ABK: " << to_hex(k) << std::endl; output += encodeVarInt(k.size()); output += k; } - std::string getLowerKey(std::vector::iterator it) { - if (it == elems.begin()) return std::string(1, '\0'); - return minimalKeyDiff(it->getCompare(idSize), std::prev(it)->getCompare(idSize)); - } - - std::string getUpperKey(std::vector::iterator it) { - if (it == elems.end()) return std::string(1, '\xFF'); - return minimalKeyDiff(it->getCompare(idSize), std::prev(it)->getCompare(idSize)); - } - std::string minimalKeyDiff(std::string_view key, std::string_view prevKey) { for (uint64_t i = 0; i < idSize + 5; i++) { if (key[i] != prevKey[i]) return std::string(key.substr(0, i + 1)); diff --git a/test/xor.cpp b/test/xor.cpp index 804c75c..14d8e37 100644 --- a/test/xor.cpp +++ b/test/xor.cpp @@ -57,15 +57,8 @@ int main() { } else { throw herr("unexpected mode"); } - - if (mode == 1) std::cerr << "CLIENT-ONLY: " << to_hex(id) << std::endl; - if (mode == 2) std::cerr << "RELAY-ONLY : " << to_hex(id) << std::endl; - if (mode == 3) std::cerr << "BOTH : " << to_hex(id) << std::endl; - } - std::cerr << "BEGIN RECONCILATION" << std::endl; - x1.finalise(); x2.finalise(); @@ -76,35 +69,36 @@ int main() { while (q.size()) { round++; std::cerr << "ROUND A " << round << std::endl; - std::cerr << "CLIENT -> RELAY" << std::endl; + std::cerr << "CLIENT -> RELAY: " << q.size() << " bytes" << std::endl; { std::vector have, need; q = x2.handleQuery(q, have, need); // q and have are returned to client for (auto &id : have) { + if (ids1.contains(id)) throw herr("redundant set"); ids1.insert(id); - std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl; } for (auto &id : need) { + if (ids2.contains(id)) throw herr("redundant set"); ids2.insert(id); - std::cerr << "ADD RELAY: " << to_hex(id) << std::endl; } } if (q.size()) { - std::cerr << "ROUND B " << round << std::endl; - std::cerr << "RELAY -> CLIENT" << std::endl; + std::cerr << "ROUND B " << round << std::endl; + std::cerr << "RELAY -> CLIENT: " << q.size() << " bytes" << std::endl; + std::vector have, need; q = x1.handleQuery(q, have, need); for (auto &id : need) { + if (ids1.contains(id)) throw herr("redundant set"); ids1.insert(id); - std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl; } for (auto &id : have) { + if (ids2.contains(id)) throw herr("redundant set"); ids2.insert(id); - std::cerr << "ADD RELAY: " << to_hex(id) << std::endl; } } } @@ -115,14 +109,5 @@ int main() { throw herr("mismatch"); } -/* - std::vector have, need; - auto q2 = x2.handleQuery(q, have, need); - - for (auto &s : have) std::cout << "HAVE: " << to_hex(s) << std::endl; - for (auto &s : need) std::cout << "NEED: " << to_hex(s) << std::endl; - std::cout << to_hex(q2) << std::endl; -*/ - return 0; } diff --git a/test/xorTest.pl b/test/xorTest.pl index e7de36e..ea88071 100644 --- a/test/xorTest.pl +++ b/test/xorTest.pl @@ -8,24 +8,37 @@ my $idSize = 16; srand($ENV{SEED} || 0); my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2); -my $pid = open2(my $outfile, my $infile, './test/xor'); -my $num = rnd(10000) + 1; +while(1) { + my $pid = open2(my $outfile, my $infile, './test/xor'); -for (1..$num) { - my $mode = rnd(3) + 1; - my $created = 1677970534 + rnd($num); - my $id = $stgen->get; - print $infile "$mode,$created,$id\n"; + my $num = rnd(10000) + 1; + + for (1..$num) { + my $mode; + if (rand() < 0.001) { + $mode = rnd(2) + 1; + } else { + $mode = 3; + } + my $created = 1677970534 + rnd($num); + my $id = $stgen->get; + print $infile "$mode,$created,$id\n"; + } + + close($infile); + + while (<$outfile>) { + print $_; + } + + waitpid($pid, 0); + my $child_exit_status = $?; + die "failure" if $child_exit_status; + + print "\n-----------OK-----------\n"; } -close($infile); - -while (<$outfile>) { - print $_; -} - - sub rnd { my $n = shift;