nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-16 08:18:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Doug Hoyte
2023-03-08 18:18:06 -05:00
parent d1a8383e64
commit 58c65dce09
4 changed files with 119 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
js/harness.js Normal file
View File

@ -0,0 +1,76 @@
const readline = require('readline');
const XorView = require('./xor.js');
const idSize = 16;
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false
});
let n = 0;
let x1 = new XorView(idSize);
let x2 = new XorView(idSize);
rl.on('line', (line) => {
let items = line.split(',');
if (items.length !== 3) throw Error("too few items");
let mode = parseInt(items[0]);
let created = parseInt(items[1]);
let id = fromHexString(items[2]);
if (id.length !== idSize) throw Error("unexpected id size");
if (mode === 1) {
x1.addElem(created, id);
} else if (mode === 2) {
x2.addElem(created, id);
} else if (mode === 3) {
x1.addElem(created, id);
x2.addElem(created, id);
} else {
throw Error("unexpected mode");
}
n++;
});
rl.once('close', () => {
x1.finalise();
x2.finalise();
let q = x1.initial();
while (q.length !== 0) {
{
let [newQ, haveIds, needIds] = x2.reconcile(q);
q = newQ;
for (let id of haveIds) console.log(`xor,2,HAVE,${id}`);
for (let id of needIds) console.log(`xor,2,NEED,${id}`);
}
if (q.length !== 0) {
let [newQ, haveIds, needIds] = x1.reconcile(q);
q = newQ;
for (let id of haveIds) console.log(`xor,1,HAVE,${id}`);
for (let id of needIds) console.log(`xor,1,NEED,${id}`);
}
}
});
// FIXME: copied from xor.js
function fromHexString(hexString) {
if ((hexString.length % 2) !== 0) throw Error("uneven length of hex string");
return hexString.match(/../g).map((byte) => parseInt(byte, 16));
}
function toHexString(buf) {
return buf.reduce((str, byte) => str + byte.toString(16).padStart(2, '0'), '');
}

51
js/xor.js
View File

@ -5,7 +5,7 @@ class XorView {
}
addElem(timestamp, id) {
elems.push({ timestamp, id, });
this.elems.push({ timestamp, id, });
}
finalise() {
@ -13,7 +13,18 @@ class XorView {
this.ready = true;
}
initial() {
if (!this.ready) throw Error("xor view not ready");
let output = [];
let lastTimestampOut = [0]; // wrapped in array so we can modify it by reference
this._splitRange(0, this.elems.length, { timestamp: 0, id: [], }, { timestamp: Number.MAX_VALUE, id: [], }, lastTimestampOut, output);
return toHexString(output);
}
reconcile(query) {
if (!this.ready) throw Error("xor view not ready");
query = fromHexString(query);
let output = [];
let haveIds = [], needIds = [];
@ -69,20 +80,24 @@ class XorView {
}
}
if (!matches) this._splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output);
if (!matches) this._splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output);
} else if (mode >= 8) {
let theirElems = {};
for (let i = 0; i < mode - 8; i++) theirElems[getBytes(query, this.idSize)] = false;
for (let i = 0; i < mode - 8; i++) {
let id = toHexString(getBytes(query, this.idSize));
theirElems[id] = false;
}
for (let it = lower; it < upper; it++) {
let e = theirElems[this.elems[i]];
for (let i = lower; i < upper; i++) {
let id = toHexString(this.elems[i].id);
let e = theirElems[id];
if (e === undefined) {
// ID exists on our side, but not their side
haveIds.push(e.id);
haveIds.push(id);
} else {
// ID exists on both sides
theirElems[this.elems[i]] = true;
theirElems[id] = true;
}
}
@ -97,10 +112,10 @@ class XorView {
}
}
return [output, haveIds, needIds];
return [toHexString(output), haveIds, needIds];
}
_splitRange(lower, upper, lowerKey, upperKey, lastTimestampOut, output) {
_splitRange(lower, upper, lowerBoundKey, upperBoundKey, lastTimestampOut, output) {
let encodeTimestampOut = (timestamp) => {
if (timestamp === Number.MAX_VALUE) {
lastTimestampOut[0] = Number.MAX_VALUE;
@ -113,7 +128,7 @@ class XorView {
return encodeVarInt(timestamp + 1);
};
let appendBoundKey = (key, output) => {
let appendBoundKey = (key) => {
output.push(...encodeTimestampOut(key.timestamp));
output.push(...encodeVarInt(key.id.length));
output.push(...key.id);
@ -142,8 +157,8 @@ class XorView {
let buckets = 16;
if (numElems < buckets * 2) {
appendBoundKey(lowerKey);
appendBoundKey(upperKey);
appendBoundKey(lowerBoundKey);
appendBoundKey(upperBoundKey);
output.push(...encodeVarInt(numElems + 8));
for (let it = lower; it < upper; ++it) output.push(...this.elems[it].id);
@ -153,19 +168,19 @@ class XorView {
let curr = lower;
for (let i = 0; i < buckets; i++) {
if (i == 0) appendBoundKey(lowerKey);
if (i == 0) appendBoundKey(lowerBoundKey);
else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]);
let ourXorSet = new Array(this.idSize).fill(0);
for (let bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) {
for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr][j];
for (let j = 0; j < this.idSize; j++) ourXorSet[j] ^= this.elems[curr].id[j];
}
if (i === buckets - 1) appendBoundKey(upperKey);
if (i === buckets - 1) appendBoundKey(upperBoundKey);
else appendMinimalBoundKey(this.elems[curr], this.elems[curr - 1]);
output.push(...encodeVarInt(0)); // mode = 0
output.push(...ourXorSet.id);
output.push(...ourXorSet);
}
}
}
@ -260,3 +275,7 @@ function lowerBound(arr, first, last, value, cmp) {
function upperBound(arr, first, last, value, cmp) {
return binarySearch(arr, first, last, (a) => cmp(value, a) >= 0);
}
module.exports = XorView;

17
test/xor.cpp
View File

@ -68,12 +68,9 @@ int main() {
q = x2.reconcile(q, have, need);
// q and have are returned to client
for (auto &id : have) {
std::cout << "xor,2,HAVE," << to_hex(id) << "\n";
}
for (auto &id : need) {
std::cout << "xor,2,NEED," << to_hex(id) << "\n";
}
for (auto &id : have) std::cout << "xor,2,HAVE," << to_hex(id) << "\n";
for (auto &id : need) std::cout << "xor,2,NEED," << to_hex(id) << "\n";
std::cerr << "HAVE " << (have.size() * idSize) << " bytes "
<< "NEED " << (need.size() * idSize) << " bytes " << std::endl;
}
@ -85,12 +82,8 @@ int main() {
std::vector<std::string> have, need;
q = x1.reconcile(q, have, need);
for (auto &id : need) {
std::cout << "xor,1,NEED," << to_hex(id) << "\n";
}
for (auto &id : have) {
std::cout << "xor,1,HAVE," << to_hex(id) << "\n";
}
for (auto &id : have) std::cout << "xor,1,HAVE," << to_hex(id) << "\n";
for (auto &id : need) std::cout << "xor,1,NEED," << to_hex(id) << "\n";
}
}

5
test/xorTest.pl
View File

@ -3,7 +3,8 @@
use IPC::Open2;
use Session::Token;
my $idSize = 16;
my $harnessCmd = shift;
my $idSize = shift || 16;
srand($ENV{SEED} || 0);
my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2);
@ -13,7 +14,7 @@ while(1) {
my $ids1 = {};
my $ids2 = {};
my $pid = open2(my $outfile, my $infile, './test/xor');
my $pid = open2(my $outfile, my $infile, $harnessCmd);
my $num = rnd(10000) + 1;