wip

src/xor.h

@@ -73,6 +73,7 @@ struct XorView {
 
     void addElem(uint64_t createdAt, std::string_view id) {
         elems.emplace_back(createdAt, id, idSize);
+        std::cerr << "ADDELEM " << to_hex(elems.back().getCompare(idSize)) << std::endl;
     }
 
     void finalise() {
@@ -81,18 +82,21 @@ struct XorView {
         std::sort(elems.begin(), elems.end(), [&](const auto &a, const auto &b) { return a.getCompare(idSize) < b.getCompare(idSize); });
 
         ready = true;
+
+        for (auto &e : elems) std::cerr << "FIN " << to_hex(e.getCompare(idSize)) << std::endl;
     }
 
     std::string initialQuery() {
         if (!ready) throw herr("xor view not ready");
 
         std::string output;
-        splitRange(elems.begin(), elems.end(), output);
+        splitRange(elems.begin(), elems.end(), std::string(1, '\x00'), std::string(1, '\xFF'), output);
         return output;
     }
 
     // FIXME: try/catch everywhere that calls this
     std::string handleQuery(std::string_view query, std::vector<std::string> &haveIds, std::vector<std::string> &needIds) {
+    std::cerr << "-------------------" << std::endl;
         if (!ready) throw herr("xor view not ready");
 
         std::string output;
@@ -100,34 +104,46 @@ struct XorView {
         auto cmp = [&](const auto &a, const auto &b){ return a.getCompare(idSize) < b.getCompare(idSize); };
 
         while (query.size()) {
+    std::cerr << "=========" << std::endl;
             uint64_t lowerLength = decodeVarInt(query);
-            if (lowerLength > idSize + 5) throw herr("lower too long");
+            if (lowerLength > idSize + 5) throw herr("lower too long: ", lowerLength);
-            XorElem lowerKey(getBytes(query, lowerLength));
+            auto lowerKeyRaw = getBytes(query, lowerLength);
+            XorElem lowerKey(lowerKeyRaw);
+            std::cerr << "LWCMP: " << to_hex(lowerKey.getCompare(idSize)) << std::endl;
 
             uint64_t upperLength = decodeVarInt(query);
             if (upperLength > idSize + 5) throw herr("upper too long");
-            XorElem upperKey(getBytes(query, upperLength));
+            auto upperKeyRaw = getBytes(query, upperLength);
+            XorElem upperKey(upperKeyRaw);
+            std::cerr << "UPCMP: " << to_hex(upperKey.getCompare(idSize)) << std::endl;
 
             auto lower = std::lower_bound(elems.begin(), elems.end(), lowerKey, cmp); // FIXME: start at prev upper?
             auto upper = std::upper_bound(elems.begin(), elems.end(), upperKey, cmp); // FIXME: start at lower?
+            std::cerr << "FOUND: " << size_t(upper-lower) << std::endl;
+            for (auto it = lower; it < upper; ++it) std::cerr << "MMM: " << to_hex(it->getId(idSize)) << std::endl;
 
             uint64_t mode = decodeVarInt(query); // 0 = range, 8 and above = n-8 inline IDs
-            std::cerr << "BING MODE = " << mode << std::endl;
 
             if (mode == 0) {
+            std::cerr << "MODE 0" << std::endl;
                 XorElem theirXorSet(0, getBytes(query, idSize), idSize);
 
                 XorElem ourXorSet;
                 for (auto i = lower; i < upper; ++i) ourXorSet.doXor(*i);
 
-                if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, output);
+                std::cerr << "XSETS " << to_hex(theirXorSet.getId(idSize)) << " / " << to_hex(ourXorSet.getId(idSize)) << std::endl;
+                if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, lowerKeyRaw, upperKeyRaw, output);
             } else if (mode >= 8) {
+            std::cerr << "MODE " << (mode - 8) << std::endl;
                 flat_hash_map<XorElem, bool> theirElems;
                 for (uint64_t i = 0; i < mode - 8; i++) {
-                    theirElems.emplace(XorElem(0, getBytes(query, idSize), idSize), false);
+                    auto bb = getBytes(query, idSize);
+                    std::cerr << "INSERTED THEIR " << to_hex(bb) << std::endl;
+                    theirElems.emplace(XorElem(0, bb, idSize), false);
                 }
 
                 for (auto it = lower; it < upper; ++it) {
+                    std::cerr << "SEARCHING " << to_hex(it->getId(idSize)) << std::endl;
                     auto e = theirElems.find(*it);
 
                     if (e == theirElems.end()) {
@@ -136,6 +152,7 @@ struct XorView {
                     } else {
                         // ID exists on both sides
                         e->second = true;
+                        std::cerr << "ERMM " << to_hex(e->first.getId(idSize)) << std::endl;
                     }
                 }
 
@@ -153,41 +170,45 @@ struct XorView {
 
   private:
 
-    void splitRange(std::vector<XorElem>::iterator lower, std::vector<XorElem>::iterator upper, std::string &output) {
+    void splitRange(std::vector<XorElem>::iterator lower, std::vector<XorElem>::iterator upper, const std::string &lowerKey, const std::string &upperKey, std::string &output) {
         // Split our range
         uint64_t numElems = upper - lower;
         const uint64_t buckets = 16;
 
         if (numElems < buckets * 2) {
-            appendBoundKey(getLowerKey(lower), output);
+            std::cerr << "DUMPING IDs" << std::endl;
-            appendBoundKey(getUpperKey(upper), output);
+            appendBoundKey(lowerKey /*getLowerKey(lower)*/, output);
+            appendBoundKey(upperKey /*getUpperKey(upper)*/, output);
 
             output += encodeVarInt(numElems + 8);
             for (auto it = lower; it < upper; ++it) output += it->getId(idSize);
+            for (auto it = lower; it < upper; ++it) std::cerr << "DUMP ID: " << to_hex(it->getId(idSize)) << std::endl;
         } else {
+            std::cerr << "DOING SPLIT" << std::endl;
             uint64_t elemsPerBucket = numElems / buckets;
             uint64_t bucketsWithExtra = numElems % buckets;
             auto curr = lower;
 
             for (uint64_t i = 0; i < buckets; i++) {
-                appendBoundKey(getLowerKey(curr), output);
+                appendBoundKey(i == 0 ? lowerKey : getLowerKey(curr), output);
-
-                auto bucketEnd = curr + elemsPerBucket;
-                if (i < bucketsWithExtra) bucketEnd++;
 
                 XorElem ourXorSet;
                 for (auto bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) {
+                std::cerr << "XORING IN " << to_hex(curr->getId(idSize)) << std::endl;
                     ourXorSet.doXor(*curr);
                 }
 
-                appendBoundKey(getUpperKey(curr), output);
+                appendBoundKey(i == buckets - 1 ? upperKey : getUpperKey(curr), output);
 
+                output += encodeVarInt(0); // mode = 0
                 output += ourXorSet.getId(idSize);
+                std::cerr << "FULL XOR " << to_hex(ourXorSet.getId(idSize)) << std::endl;
             }
         }
     }
 
     void appendBoundKey(std::string k, std::string &output) {
+        std::cerr << "ABK: " << to_hex(k) << std::endl;
         output += encodeVarInt(k.size());
         output += k;
     }

test/xor.cpp

@@ -1,31 +1,128 @@
 #include <iostream>
+#include <sstream>
 
 #include "golpe.h"
 #include "xor.h"
 
 
-int main() {
-    XorView x1(16);
-    x1.addElem(1000, std::string(16, 'a'));
-    x1.addElem(2000, std::string(16, 'b'));
-    x1.finalise();
 
-    XorView x2(16);
+std::vector<std::string> split(const std::string &s, char delim) {
-    x2.addElem(2000, std::string(16, 'b'));
+    std::vector<std::string> result;
-    x2.addElem(3000, std::string(16, 'c'));
+    std::stringstream ss (s);
+    std::string item;
+
+    while (getline (ss, item, delim)) {
+        result.push_back (item);
+    }
+
+    return result;
+}
+
+
+
+int main() {
+    const uint64_t idSize = 16;
+
+    // x1 is client, x2 is relay
+    XorView x1(idSize);
+    XorView x2(idSize);
+
+    std::set<std::string> ids1;
+    std::set<std::string> ids2;
+
+    std::string line;
+    while (std::cin) {
+        std::getline(std::cin, line);
+        if (!line.size()) continue;
+
+        auto items = split(line, ',');
+        if (items.size() != 3) throw herr("too few items");
+
+        int mode = std::stoi(items[0]);
+        uint64_t created = std::stoull(items[1]);
+        auto id = from_hex(items[2]);
+        if (id.size() != idSize) throw herr("unexpected id size");
+
+        if (mode == 1) {
+            x1.addElem(created, id);
+            ids1.insert(id);
+        } else if (mode == 2) {
+            x2.addElem(created, id);
+            ids2.insert(id);
+        } else if (mode == 3) {
+            x1.addElem(created, id);
+            x2.addElem(created, id);
+            ids1.insert(id);
+            ids2.insert(id);
+        } else {
+            throw herr("unexpected mode");
+        }
+
+        if (mode == 1) std::cerr << "CLIENT-ONLY: " << to_hex(id) << std::endl;
+        if (mode == 2) std::cerr << "RELAY-ONLY : " << to_hex(id) << std::endl;
+        if (mode == 3) std::cerr << "BOTH       : " << to_hex(id) << std::endl;
+
+    }
+
+    std::cerr << "BEGIN RECONCILATION" << std::endl;
+
+    x1.finalise();
     x2.finalise();
 
-    {
+    std::string q = x1.initialQuery();
-        auto q = x1.initialQuery();
-        std::cout << to_hex(q) << std::endl;
 
+    uint64_t round = 0;
+
+    while (q.size()) {
+        round++;
+        std::cerr << "ROUND A " << round << std::endl;
+        std::cerr << "CLIENT -> RELAY" << std::endl;
+        {
+            std::vector<std::string> have, need;
+            q = x2.handleQuery(q, have, need);
+
+            // q and have are returned to client
+            for (auto &id : have) {
+                ids1.insert(id);
+                std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl;
+            }
+            for (auto &id : need) {
+                ids2.insert(id);
+                std::cerr << "ADD RELAY:  " << to_hex(id) << std::endl;
+            }
+        }
+
+        if (q.size()) {
+        std::cerr << "ROUND B " << round << std::endl;
+        std::cerr << "RELAY -> CLIENT" << std::endl;
+            std::vector<std::string> have, need;
+            q = x1.handleQuery(q, have, need);
+
+            for (auto &id : need) {
+                ids1.insert(id);
+                std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl;
+            }
+            for (auto &id : have) {
+                ids2.insert(id);
+                std::cerr << "ADD RELAY:  " << to_hex(id) << std::endl;
+            }
+        }
+    }
+
+    if (ids1 != ids2) {
+        for (const auto &id : ids1) if (!ids2.contains(id)) std::cerr << "In CLIENT not RELAY: " << to_hex(id) << std::endl;
+        for (const auto &id : ids2) if (!ids1.contains(id)) std::cerr << "In RELAY not CLIENT: " << to_hex(id) << std::endl;
+        throw herr("mismatch");
+    }
+
+/*
     std::vector<std::string> have, need;
     auto q2 = x2.handleQuery(q, have, need);
 
     for (auto &s : have) std::cout << "HAVE: " << to_hex(s) << std::endl;
     for (auto &s : need) std::cout << "NEED: " << to_hex(s) << std::endl;
     std::cout << to_hex(q2) << std::endl;
-    }
+*/
 
     return 0;
 }

test/xorTest.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/env perl
+
+use IPC::Open2;
+use Session::Token;
+
+my $idSize = 16;
+
+srand($ENV{SEED} || 0);
+my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2);
+
+my $pid = open2(my $outfile, my $infile, './test/xor');
+
+my $num = rnd(10000) + 1;
+
+for (1..$num) {
+    my $mode = rnd(3) + 1;
+    my $created = 1677970534 + rnd($num);
+    my $id = $stgen->get;
+    print $infile "$mode,$created,$id\n";
+}
+
+close($infile);
+
+while (<$outfile>) {
+    print $_;
+}
+
+
+
+sub rnd {
+    my $n = shift;
+    return int(rand() * $n);
+}