nostr/strfry
1
0
Fork 0
mirror of https://github.com/hoytech/strfry.git synced 2025-06-19 01:34:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Doug Hoyte
2023-03-05 16:35:43 -05:00
parent 0be5f1b5de
commit 2f59f8ffa3
3 changed files with 39 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/xor.h
View File

@ -73,7 +73,6 @@ struct XorView {
void addElem(uint64_t createdAt, std::string_view id) { void addElem(uint64_t createdAt, std::string_view id) {
elems.emplace_back(createdAt, id, idSize); elems.emplace_back(createdAt, id, idSize);
std::cerr << "ADDELEM " << to_hex(elems.back().getCompare(idSize)) << std::endl;
} }
void finalise() { void finalise() {
@ -82,8 +81,6 @@ struct XorView {
std::sort(elems.begin(), elems.end(), [&](const auto &a, const auto &b) { return a.getCompare(idSize) < b.getCompare(idSize); }); std::sort(elems.begin(), elems.end(), [&](const auto &a, const auto &b) { return a.getCompare(idSize) < b.getCompare(idSize); });
ready = true; ready = true;
for (auto &e : elems) std::cerr << "FIN " << to_hex(e.getCompare(idSize)) << std::endl;
} }
std::string initialQuery() { std::string initialQuery() {
@ -96,7 +93,6 @@ struct XorView {
// FIXME: try/catch everywhere that calls this // FIXME: try/catch everywhere that calls this
std::string handleQuery(std::string_view query, std::vector<std::string> &haveIds, std::vector<std::string> &needIds) { std::string handleQuery(std::string_view query, std::vector<std::string> &haveIds, std::vector<std::string> &needIds) {
std::cerr << "-------------------" << std::endl;
if (!ready) throw herr("xor view not ready"); if (!ready) throw herr("xor view not ready");
std::string output; std::string output;
@ -104,46 +100,36 @@ struct XorView {
auto cmp = [&](const auto &a, const auto &b){ return a.getCompare(idSize) < b.getCompare(idSize); }; auto cmp = [&](const auto &a, const auto &b){ return a.getCompare(idSize) < b.getCompare(idSize); };
while (query.size()) { while (query.size()) {
std::cerr << "=========" << std::endl;
uint64_t lowerLength = decodeVarInt(query); uint64_t lowerLength = decodeVarInt(query);
if (lowerLength > idSize + 5) throw herr("lower too long: ", lowerLength); if (lowerLength > idSize + 5) throw herr("lower too long: ", lowerLength);
auto lowerKeyRaw = getBytes(query, lowerLength); auto lowerKeyRaw = getBytes(query, lowerLength);
XorElem lowerKey(lowerKeyRaw); XorElem lowerKey(lowerKeyRaw);
std::cerr << "LWCMP: " << to_hex(lowerKey.getCompare(idSize)) << std::endl;
uint64_t upperLength = decodeVarInt(query); uint64_t upperLength = decodeVarInt(query);
if (upperLength > idSize + 5) throw herr("upper too long"); if (upperLength > idSize + 5) throw herr("upper too long");
auto upperKeyRaw = getBytes(query, upperLength); auto upperKeyRaw = getBytes(query, upperLength);
XorElem upperKey(upperKeyRaw); XorElem upperKey(upperKeyRaw);
std::cerr << "UPCMP: " << to_hex(upperKey.getCompare(idSize)) << std::endl;
auto lower = std::lower_bound(elems.begin(), elems.end(), lowerKey, cmp); // FIXME: start at prev upper? auto lower = std::lower_bound(elems.begin(), elems.end(), lowerKey, cmp); // FIXME: start at prev upper?
auto upper = std::upper_bound(elems.begin(), elems.end(), upperKey, cmp); // FIXME: start at lower? auto upper = std::upper_bound(elems.begin(), elems.end(), upperKey, cmp); // FIXME: start at lower?
std::cerr << "FOUND: " << size_t(upper-lower) << std::endl;
for (auto it = lower; it < upper; ++it) std::cerr << "MMM: " << to_hex(it->getId(idSize)) << std::endl;
uint64_t mode = decodeVarInt(query); // 0 = range, 8 and above = n-8 inline IDs uint64_t mode = decodeVarInt(query); // 0 = range, 8 and above = n-8 inline IDs
if (mode == 0) { if (mode == 0) {
std::cerr << "MODE 0" << std::endl;
XorElem theirXorSet(0, getBytes(query, idSize), idSize); XorElem theirXorSet(0, getBytes(query, idSize), idSize);
XorElem ourXorSet; XorElem ourXorSet;
for (auto i = lower; i < upper; ++i) ourXorSet.doXor(*i); for (auto i = lower; i < upper; ++i) ourXorSet.doXor(*i);
std::cerr << "XSETS " << to_hex(theirXorSet.getId(idSize)) << " / " << to_hex(ourXorSet.getId(idSize)) << std::endl;
if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, lowerKeyRaw, upperKeyRaw, output); if (theirXorSet.getId(idSize) != ourXorSet.getId(idSize)) splitRange(lower, upper, lowerKeyRaw, upperKeyRaw, output);
} else if (mode >= 8) { } else if (mode >= 8) {
std::cerr << "MODE " << (mode - 8) << std::endl;
flat_hash_map<XorElem, bool> theirElems; flat_hash_map<XorElem, bool> theirElems;
for (uint64_t i = 0; i < mode - 8; i++) { for (uint64_t i = 0; i < mode - 8; i++) {
auto bb = getBytes(query, idSize); auto bb = getBytes(query, idSize);
std::cerr << "INSERTED THEIR " << to_hex(bb) << std::endl;
theirElems.emplace(XorElem(0, bb, idSize), false); theirElems.emplace(XorElem(0, bb, idSize), false);
} }
for (auto it = lower; it < upper; ++it) { for (auto it = lower; it < upper; ++it) {
std::cerr << "SEARCHING " << to_hex(it->getId(idSize)) << std::endl;
auto e = theirElems.find(*it); auto e = theirElems.find(*it);
if (e == theirElems.end()) { if (e == theirElems.end()) {
@ -152,7 +138,6 @@ struct XorView {
} else { } else {
// ID exists on both sides // ID exists on both sides
e->second = true; e->second = true;
std::cerr << "ERMM " << to_hex(e->first.getId(idSize)) << std::endl;
} }
} }
@ -176,53 +161,37 @@ struct XorView {
const uint64_t buckets = 16; const uint64_t buckets = 16;
if (numElems < buckets * 2) { if (numElems < buckets * 2) {
std::cerr << "DUMPING IDs" << std::endl; appendBoundKey(lowerKey, output);
appendBoundKey(lowerKey /*getLowerKey(lower)*/, output); appendBoundKey(upperKey, output);
appendBoundKey(upperKey /*getUpperKey(upper)*/, output);
output += encodeVarInt(numElems + 8); output += encodeVarInt(numElems + 8);
for (auto it = lower; it < upper; ++it) output += it->getId(idSize); for (auto it = lower; it < upper; ++it) output += it->getId(idSize);
for (auto it = lower; it < upper; ++it) std::cerr << "DUMP ID: " << to_hex(it->getId(idSize)) << std::endl;
} else { } else {
std::cerr << "DOING SPLIT" << std::endl;
uint64_t elemsPerBucket = numElems / buckets; uint64_t elemsPerBucket = numElems / buckets;
uint64_t bucketsWithExtra = numElems % buckets; uint64_t bucketsWithExtra = numElems % buckets;
auto curr = lower; auto curr = lower;
for (uint64_t i = 0; i < buckets; i++) { for (uint64_t i = 0; i < buckets; i++) {
appendBoundKey(i == 0 ? lowerKey : getLowerKey(curr), output); appendBoundKey(i == 0 ? lowerKey : minimalKeyDiff(curr->getCompare(idSize), std::prev(curr)->getCompare(idSize)), output);
XorElem ourXorSet; XorElem ourXorSet;
for (auto bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) { for (auto bucketEnd = curr + elemsPerBucket + (i < bucketsWithExtra ? 1 : 0); curr != bucketEnd; curr++) {
std::cerr << "XORING IN " << to_hex(curr->getId(idSize)) << std::endl;
ourXorSet.doXor(*curr); ourXorSet.doXor(*curr);
} }
appendBoundKey(i == buckets - 1 ? upperKey : getUpperKey(curr), output); appendBoundKey(i == buckets - 1 ? upperKey : minimalKeyDiff(curr->getCompare(idSize), std::prev(curr)->getCompare(idSize)), output);
output += encodeVarInt(0); // mode = 0 output += encodeVarInt(0); // mode = 0
output += ourXorSet.getId(idSize); output += ourXorSet.getId(idSize);
std::cerr << "FULL XOR " << to_hex(ourXorSet.getId(idSize)) << std::endl;
} }
} }
} }
void appendBoundKey(std::string k, std::string &output) { void appendBoundKey(std::string k, std::string &output) {
std::cerr << "ABK: " << to_hex(k) << std::endl;
output += encodeVarInt(k.size()); output += encodeVarInt(k.size());
output += k; output += k;
} }
std::string getLowerKey(std::vector<XorElem>::iterator it) {
if (it == elems.begin()) return std::string(1, '\0');
return minimalKeyDiff(it->getCompare(idSize), std::prev(it)->getCompare(idSize));
}
std::string getUpperKey(std::vector<XorElem>::iterator it) {
if (it == elems.end()) return std::string(1, '\xFF');
return minimalKeyDiff(it->getCompare(idSize), std::prev(it)->getCompare(idSize));
}
std::string minimalKeyDiff(std::string_view key, std::string_view prevKey) { std::string minimalKeyDiff(std::string_view key, std::string_view prevKey) {
for (uint64_t i = 0; i < idSize + 5; i++) { for (uint64_t i = 0; i < idSize + 5; i++) {
if (key[i] != prevKey[i]) return std::string(key.substr(0, i + 1)); if (key[i] != prevKey[i]) return std::string(key.substr(0, i + 1));

31
test/xor.cpp
View File

@ -57,15 +57,8 @@ int main() {
} else { } else {
throw herr("unexpected mode"); throw herr("unexpected mode");
} }
if (mode == 1) std::cerr << "CLIENT-ONLY: " << to_hex(id) << std::endl;
if (mode == 2) std::cerr << "RELAY-ONLY : " << to_hex(id) << std::endl;
if (mode == 3) std::cerr << "BOTH : " << to_hex(id) << std::endl;
} }
std::cerr << "BEGIN RECONCILATION" << std::endl;
x1.finalise(); x1.finalise();
x2.finalise(); x2.finalise();
@ -76,35 +69,36 @@ int main() {
while (q.size()) { while (q.size()) {
round++; round++;
std::cerr << "ROUND A " << round << std::endl; std::cerr << "ROUND A " << round << std::endl;
std::cerr << "CLIENT -> RELAY" << std::endl; std::cerr << "CLIENT -> RELAY: " << q.size() << " bytes" << std::endl;
{ {
std::vector<std::string> have, need; std::vector<std::string> have, need;
q = x2.handleQuery(q, have, need); q = x2.handleQuery(q, have, need);
// q and have are returned to client // q and have are returned to client
for (auto &id : have) { for (auto &id : have) {
if (ids1.contains(id)) throw herr("redundant set");
ids1.insert(id); ids1.insert(id);
std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl;
} }
for (auto &id : need) { for (auto &id : need) {
if (ids2.contains(id)) throw herr("redundant set");
ids2.insert(id); ids2.insert(id);
std::cerr << "ADD RELAY: " << to_hex(id) << std::endl;
} }
} }
if (q.size()) { if (q.size()) {
std::cerr << "ROUND B " << round << std::endl; std::cerr << "ROUND B " << round << std::endl;
std::cerr << "RELAY -> CLIENT" << std::endl; std::cerr << "RELAY -> CLIENT: " << q.size() << " bytes" << std::endl;
std::vector<std::string> have, need; std::vector<std::string> have, need;
q = x1.handleQuery(q, have, need); q = x1.handleQuery(q, have, need);
for (auto &id : need) { for (auto &id : need) {
if (ids1.contains(id)) throw herr("redundant set");
ids1.insert(id); ids1.insert(id);
std::cerr << "ADD CLIENT: " << to_hex(id) << std::endl;
} }
for (auto &id : have) { for (auto &id : have) {
if (ids2.contains(id)) throw herr("redundant set");
ids2.insert(id); ids2.insert(id);
std::cerr << "ADD RELAY: " << to_hex(id) << std::endl;
} }
} }
} }
@ -115,14 +109,5 @@ int main() {
throw herr("mismatch"); throw herr("mismatch");
} }
/*
std::vector<std::string> have, need;
auto q2 = x2.handleQuery(q, have, need);
for (auto &s : have) std::cout << "HAVE: " << to_hex(s) << std::endl;
for (auto &s : need) std::cout << "NEED: " << to_hex(s) << std::endl;
std::cout << to_hex(q2) << std::endl;
*/
return 0; return 0;
} }

41
test/xorTest.pl
View File

@ -8,24 +8,37 @@ my $idSize = 16;
srand($ENV{SEED} || 0); srand($ENV{SEED} || 0);
my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2); my $stgen = Session::Token->new(seed => "\x00" x 1024, alphabet => '0123456789abcdef', length => $idSize * 2);
my $pid = open2(my $outfile, my $infile, './test/xor');
my $num = rnd(10000) + 1; while(1) {
my $pid = open2(my $outfile, my $infile, './test/xor');
for (1..$num) { my $num = rnd(10000) + 1;
my $mode = rnd(3) + 1;
my $created = 1677970534 + rnd($num); for (1..$num) {
my $id = $stgen->get; my $mode;
print $infile "$mode,$created,$id\n"; if (rand() < 0.001) {
$mode = rnd(2) + 1;
} else {
$mode = 3;
}
my $created = 1677970534 + rnd($num);
my $id = $stgen->get;
print $infile "$mode,$created,$id\n";
}
close($infile);
while (<$outfile>) {
print $_;
}
waitpid($pid, 0);
my $child_exit_status = $?;
die "failure" if $child_exit_status;
print "\n-----------OK-----------\n";
} }
close($infile);
while (<$outfile>) {
print $_;
}
sub rnd { sub rnd {
my $n = shift; my $n = shift;