wip

js/harness.js

@@ -0,0 +1,76 @@
+const readline = require('readline');
+const XorView = require('./xor.js');
+
+const idSize = 16;
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout,
+  terminal: false
+});
+
+let n = 0;
+let x1 = new XorView(idSize);
+let x2 = new XorView(idSize);
+
+rl.on('line', (line) => {
+    let items = line.split(',');
+    if (items.length !== 3) throw Error("too few items");
+
+    let mode = parseInt(items[0]);
+    let created = parseInt(items[1]);
+    let id = fromHexString(items[2]);
+    if (id.length !== idSize) throw Error("unexpected id size");
+
+    if (mode === 1) {
+        x1.addElem(created, id);
+    } else if (mode === 2) {
+        x2.addElem(created, id);
+    } else if (mode === 3) {
+        x1.addElem(created, id);
+        x2.addElem(created, id);
+    } else {
+        throw Error("unexpected mode");
+    }
+
+    n++;
+});
+
+rl.once('close', () => {
+    x1.finalise();
+    x2.finalise();
+
+    let q = x1.initial();
+
+    while (q.length !== 0) {
+        {
+            let [newQ, haveIds, needIds] = x2.reconcile(q);
+            q = newQ;
+
+            for (let id of haveIds) console.log(`xor,2,HAVE,${id}`);
+            for (let id of needIds) console.log(`xor,2,NEED,${id}`);
+        }
+
+        if (q.length !== 0) {
+            let [newQ, haveIds, needIds] = x1.reconcile(q);
+            q = newQ;
+
+            for (let id of haveIds) console.log(`xor,1,HAVE,${id}`);
+            for (let id of needIds) console.log(`xor,1,NEED,${id}`);
+        }
+    }
+});
+
+
+
+
+// FIXME: copied from xor.js
+
+function fromHexString(hexString) {
+    if ((hexString.length % 2) !== 0) throw Error("uneven length of hex string");
+    return hexString.match(/../g).map((byte) => parseInt(byte, 16));
+}
+
+function toHexString(buf) {
+    return buf.reduce((str, byte) => str + byte.toString(16).padStart(2, '0'), '');
+}

js/xor.js

@@ -5,7 +5,7 @@ class XorView {
     }
 
     addElem(timestamp, id) {
-        elems.push({ timestamp, id, });
+        this.elems.push({ timestamp, id, });
     }
 
     finalise() {
@@ -13,7 +13,18 @@ class XorView {
         this.ready = true;
     }
 
+    initial() {
+        if (!this.ready) throw Error("xor view not ready");
+
+        let output = [];
+        let lastTimestampOut = [0]; // wrapped in array so we can modify it by reference
+        this._splitRange(0, this.elems.length, { timestamp: 0, id: [], }, { timestamp: Number.MAX_VALUE, id: [], }, lastTimestampOut, output);
+        return toHexString(output);
+    }
+
     reconcile(query) {
+        if (!this.ready) throw Error("xor view not ready");
+
         query = fromHexString(query);
         let output = [];
         let haveIds = [], needIds = [];
@@ -69,20 +80,24 @@ class XorView {
                     }
                 }
 
-                if (!matches) this._splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output);
+                if (!matches) this._splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output);
             } else if (mode >= 8) {
                 let theirElems = {};
-                for (let i = 0; i < mode - 8; i++) theirElems[getBytes(query, this.idSize)] = false;
+                for (let i = 0; i < mode - 8; i++) {
+                    let id = toHexString(getBytes(query, this.idSize));
+                    theirElems[id] = false;
+                }
 
-                for (let it = lower; it < upper; it++) {
+                for (let i = lower; i < upper; i++) {
-                    let e = theirElems[this.elems[i]];
+                    let id = toHexString(this.elems[i].id);
+                    let e = theirElems[id];
 
                     if (e === undefined) {
                         // ID exists on our side, but not their side
-                        haveIds.push(e.id);
+                        haveIds.push(id);
                     } else {
                         // ID exists on both sides
-                        theirElems[this.elems[i]] = true;
+                        theirElems[id] = true;
                     }
                 }
 
@@ -97,10 +112,10 @@ class XorView {
             }
         }
 
-        return [output, haveIds, needIds];
+        return [toHexString(output), haveIds, needIds];
     }
 
-    _splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output) {
+    _splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output) {
         let encodeTimestampOut = (timestamp) => {
             if (timestamp === Number.MAX_VALUE) {
                 lastTimestampOut[0] = Number.MAX_VALUE;
@@ -113,7 +128,7 @@ class XorView {
             return encodeVarInt(timestamp + 1);
         };
 
-        let appendBoundKey = (key, output) => {
+        let appendBoundKey = (key) => {
             output.push(...encodeTimestampOut(key.timestamp));
             output.push(...encodeVarInt(key.id.length));
             output.push(...key.id);
@@ -142,8 +157,8 @@ class XorView {
         let buckets = 16;
 
         if (numElems < buckets * 2) {
-            appendBoundKey(lowerKey);
+            appendBoundKey(lowerBoundKey);
-            appendBoundKey(upperKey);
+            appendBoundKey(upperBoundKey);
 
             output.push(...encodeVarInt(numElems + 8));
             for (let it = lower; it < upper; ++it) output.push(...this.elems[it].id);
@@ -153,19 +168,19 @@ class XorView {
             let curr = lower;
 
             for (let i = 0; i < buckets; i++) {
-                if (i == 0) appendBoundKey(lowerKey);
+                if (i == 0) appendBoundKey(lowerBoundKey);
                 else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]);
 
                 let ourXorSet = new Array(this.idSize).fill(0);
                 for (let bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) {
-                    for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr][j];
+                    for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr].id[j];
                 }
 
-                if (i === buckets - 1) appendBoundKey(upperKey);
+                if (i === buckets - 1) appendBoundKey(upperBoundKey);
                 else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]);
 
                 output.push(...encodeVarInt(0)); // mode = 0
-                output.push(...ourXorSet.id);
+                output.push(...ourXorSet);
             }
         }
     }
@@ -260,3 +275,7 @@ function lowerBound(arr, first, last, value, cmp) {
 function upperBound(arr, first, last, value, cmp) {
     return binarySearch(arr, first, last, (a) => cmp(value, a) >= 0);
 }
+
+
+
+module.exports = XorView;

test/xor.cpp

@@ -68,12 +68,9 @@ int main() {
             q = x2.reconcile(q, have, need);
 
             // q and have are returned to client
-            for (auto &id : have) {
+            for (auto &id : have) std::cout << "xor,2,HAVE," << to_hex(id) << "\n";
-                std::cout << "xor,2,HAVE," << to_hex(id) << "\n";
+            for (auto &id : need) std::cout << "xor,2,NEED," << to_hex(id) << "\n";
-            }
+
-            for (auto &id : need) {
-                std::cout << "xor,2,NEED," << to_hex(id) << "\n";
-            }
             std::cerr << "HAVE " << (have.size() * idSize) << " bytes "
                       << "NEED " << (need.size() * idSize) << " bytes " << std::endl;
         }
@@ -85,12 +82,8 @@ int main() {
             std::vector<std::string> have, need;
             q = x1.reconcile(q, have, need);
 
-            for (auto &id : need) {
+            for (auto &id : have) std::cout << "xor,1,HAVE," << to_hex(id) << "\n";
-                std::cout << "xor,1,NEED," << to_hex(id) << "\n";
+            for (auto &id : need) std::cout << "xor,1,NEED," << to_hex(id) << "\n";
-            }
-            for (auto &id : have) {
-                std::cout << "xor,1,HAVE," << to_hex(id) << "\n";
-            }
         }
     }
 

test/xorTest.pl

@@ -3,7 +3,8 @@
 use IPC::Open2;
 use Session::Token;
 
-my $idSize = 16;
+my $harnessCmd = shift;
+my $idSize = shift || 16;
 
 srand($ENV{SEED} || 0);
 my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2);
@@ -13,7 +14,7 @@ while(1) {
     my $ids1 = {};
     my $ids2 = {};
 
-    my $pid = open2(my $outfile, my $infile, './test/xor');
+    my $pid = open2(my $outfile, my $infile, $harnessCmd);
 
     my $num = rnd(10000) + 1;