diff --git a/js/harness.js b/js/harness.js new file mode 100644 index 0000000..2ccbe72 --- /dev/null +++ b/js/harness.js @@ -0,0 +1,76 @@ +const readline = require('readline'); +const XorView = require('./xor.js'); + +const idSize = 16; + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: false +}); + +let n = 0; +let x1 = new XorView(idSize); +let x2 = new XorView(idSize); + +rl.on('line', (line) => { + let items = line.split(','); + if (items.length !== 3) throw Error("too few items"); + + let mode = parseInt(items[0]); + let created = parseInt(items[1]); + let id = fromHexString(items[2]); + if (id.length !== idSize) throw Error("unexpected id size"); + + if (mode === 1) { + x1.addElem(created, id); + } else if (mode === 2) { + x2.addElem(created, id); + } else if (mode === 3) { + x1.addElem(created, id); + x2.addElem(created, id); + } else { + throw Error("unexpected mode"); + } + + n++; +}); + +rl.once('close', () => { + x1.finalise(); + x2.finalise(); + + let q = x1.initial(); + + while (q.length !== 0) { + { + let [newQ, haveIds, needIds] = x2.reconcile(q); + q = newQ; + + for (let id of haveIds) console.log(`xor,2,HAVE,${id}`); + for (let id of needIds) console.log(`xor,2,NEED,${id}`); + } + + if (q.length !== 0) { + let [newQ, haveIds, needIds] = x1.reconcile(q); + q = newQ; + + for (let id of haveIds) console.log(`xor,1,HAVE,${id}`); + for (let id of needIds) console.log(`xor,1,NEED,${id}`); + } + } +}); + + + + +// FIXME: copied from xor.js + +function fromHexString(hexString) { + if ((hexString.length % 2) !== 0) throw Error("uneven length of hex string"); + return hexString.match(/../g).map((byte) => parseInt(byte, 16)); +} + +function toHexString(buf) { + return buf.reduce((str, byte) => str + byte.toString(16).padStart(2, '0'), ''); +} diff --git a/js/xor.js b/js/xor.js index 4f86dad..1256eea 100644 --- a/js/xor.js +++ b/js/xor.js @@ -5,7 +5,7 @@ class XorView { } addElem(timestamp, id) { - elems.push({ timestamp, id, }); + this.elems.push({ timestamp, id, }); } finalise() { @@ -13,7 +13,18 @@ class XorView { this.ready = true; } + initial() { + if (!this.ready) throw Error("xor view not ready"); + + let output = []; + let lastTimestampOut = [0]; // wrapped in array so we can modify it by reference + this._splitRange(0, this.elems.length, { timestamp: 0, id: [], }, { timestamp: Number.MAX_VALUE, id: [], }, lastTimestampOut, output); + return toHexString(output); + } + reconcile(query) { + if (!this.ready) throw Error("xor view not ready"); + query = fromHexString(query); let output = []; let haveIds = [], needIds = []; @@ -69,20 +80,24 @@ class XorView { } } - if (!matches) this._splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output); + if (!matches) this._splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output); } else if (mode >= 8) { let theirElems = {}; - for (let i = 0; i < mode - 8; i++) theirElems[getBytes(query, this.idSize)] = false; + for (let i = 0; i < mode - 8; i++) { + let id = toHexString(getBytes(query, this.idSize)); + theirElems[id] = false; + } - for (let it = lower; it < upper; it++) { - let e = theirElems[this.elems[i]]; + for (let i = lower; i < upper; i++) { + let id = toHexString(this.elems[i].id); + let e = theirElems[id]; if (e === undefined) { // ID exists on our side, but not their side - haveIds.push(e.id); + haveIds.push(id); } else { // ID exists on both sides - theirElems[this.elems[i]] = true; + theirElems[id] = true; } } @@ -97,10 +112,10 @@ class XorView { } } - return [output, haveIds, needIds]; + return [toHexString(output), haveIds, needIds]; } - _splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output) { + _splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output) { let encodeTimestampOut = (timestamp) => { if (timestamp === Number.MAX_VALUE) { lastTimestampOut[0] = Number.MAX_VALUE; @@ -113,7 +128,7 @@ class XorView { return encodeVarInt(timestamp + 1); }; - let appendBoundKey = (key, output) => { + let appendBoundKey = (key) => { output.push(...encodeTimestampOut(key.timestamp)); output.push(...encodeVarInt(key.id.length)); output.push(...key.id); @@ -142,8 +157,8 @@ class XorView { let buckets = 16; if (numElems < buckets * 2) { - appendBoundKey(lowerKey); - appendBoundKey(upperKey); + appendBoundKey(lowerBoundKey); + appendBoundKey(upperBoundKey); output.push(...encodeVarInt(numElems + 8)); for (let it = lower; it < upper; ++it) output.push(...this.elems[it].id); @@ -153,19 +168,19 @@ class XorView { let curr = lower; for (let i = 0; i < buckets; i++) { - if (i == 0) appendBoundKey(lowerKey); + if (i == 0) appendBoundKey(lowerBoundKey); else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]); let ourXorSet = new Array(this.idSize).fill(0); for (let bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) { - for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr][j]; + for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr].id[j]; } - if (i === buckets - 1) appendBoundKey(upperKey); + if (i === buckets - 1) appendBoundKey(upperBoundKey); else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]); output.push(...encodeVarInt(0)); // mode = 0 - output.push(...ourXorSet.id); + output.push(...ourXorSet); } } } @@ -260,3 +275,7 @@ function lowerBound(arr, first, last, value, cmp) { function upperBound(arr, first, last, value, cmp) { return binarySearch(arr, first, last, (a) => cmp(value, a) >= 0); } + + + +module.exports = XorView; diff --git a/test/xor.cpp b/test/xor.cpp index 7d05e57..e0910e4 100644 --- a/test/xor.cpp +++ b/test/xor.cpp @@ -68,12 +68,9 @@ int main() { q = x2.reconcile(q, have, need); // q and have are returned to client - for (auto &id : have) { - std::cout << "xor,2,HAVE," << to_hex(id) << "\n"; - } - for (auto &id : need) { - std::cout << "xor,2,NEED," << to_hex(id) << "\n"; - } + for (auto &id : have) std::cout << "xor,2,HAVE," << to_hex(id) << "\n"; + for (auto &id : need) std::cout << "xor,2,NEED," << to_hex(id) << "\n"; + std::cerr << "HAVE " << (have.size() * idSize) << " bytes " << "NEED " << (need.size() * idSize) << " bytes " << std::endl; } @@ -85,12 +82,8 @@ int main() { std::vector have, need; q = x1.reconcile(q, have, need); - for (auto &id : need) { - std::cout << "xor,1,NEED," << to_hex(id) << "\n"; - } - for (auto &id : have) { - std::cout << "xor,1,HAVE," << to_hex(id) << "\n"; - } + for (auto &id : have) std::cout << "xor,1,HAVE," << to_hex(id) << "\n"; + for (auto &id : need) std::cout << "xor,1,NEED," << to_hex(id) << "\n"; } } diff --git a/test/xorTest.pl b/test/xorTest.pl index d5ec5ba..d03a920 100644 --- a/test/xorTest.pl +++ b/test/xorTest.pl @@ -3,7 +3,8 @@ use IPC::Open2; use Session::Token; -my $idSize = 16; +my $harnessCmd = shift; +my $idSize = shift || 16; srand($ENV{SEED} || 0); my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2); @@ -13,7 +14,7 @@ while(1) { my $ids1 = {}; my $ids2 = {}; - my $pid = open2(my $outfile, my $infile, './test/xor'); + my $pid = open2(my $outfile, my $infile, $harnessCmd); my $num = rnd(10000) + 1;