don't store receivedAt and source metadata

golpe.yaml

@@ -22,18 +22,12 @@ tables:
   ## Primary key is auto-incremented, called "levId" for Local EVent ID
   Event:
     fields:
-      - name: receivedAt # microseconds
       - name: packed
         type: ubytes
-      - name: sourceType
-      - name: sourceInfo
-        type: ubytes
 
     indices:
       created_at:
         integer: true
-      receivedAt:
-        integer: true
       id:
         comparator: StringUint64
       pubkey:
@@ -57,7 +51,6 @@ tables:
         PackedEventView packed(v.packed());
         created_at = packed.created_at();
         uint64_t indexTime = *created_at;
-        receivedAt = v.receivedAt();
 
         id = makeKey_StringUint64(packed.id(), indexTime);
         pubkey = makeKey_StringUint64(packed.pubkey(), indexTime);

src/PluginEventSifter.h

@@ -59,7 +59,7 @@ struct PluginEventSifter {
 
     std::unique_ptr<RunningPlugin> running; 
 
-    PluginEventSifterResult acceptEvent(const std::string &pluginCmd, const tao::json::value &evJson, uint64_t receivedAtUs, EventSourceType sourceType, std::string_view sourceInfo, std::string &okMsg) {
+    PluginEventSifterResult acceptEvent(const std::string &pluginCmd, const tao::json::value &evJson, EventSourceType sourceType, std::string_view sourceInfo, std::string &okMsg) {
         if (pluginCmd.size() == 0) {
             running.reset();
             return PluginEventSifterResult::Accept;
@@ -85,7 +85,7 @@ struct PluginEventSifter {
             auto request = tao::json::value({
                 { "type", "new" },
                 { "event", evJson },
-                { "receivedAt", receivedAtUs / 1'000'000 },
+                { "receivedAt", ::time(nullptr) },
                 { "sourceType", eventSourceTypeToStr(sourceType) },
                 { "sourceInfo", sourceType == EventSourceType::IP4 || sourceType == EventSourceType::IP6 ? renderIP(sourceInfo) : sourceInfo },
             });

src/WriterPipeline.h

@@ -9,8 +9,6 @@
 
 struct WriterPipelineInput {
     tao::json::value eventJson;
-    EventSourceType sourceType;
-    std::string sourceInfo;
 };
 
 
@@ -79,7 +77,7 @@ struct WriterPipeline {
                         continue;
                     }
 
-                    writerInbox.push_move({ std::move(packedStr), std::move(jsonStr), hoytech::curr_time_us(), m.sourceType, std::move(m.sourceInfo) });
+                    writerInbox.push_move({ std::move(packedStr), std::move(jsonStr), });
                 }
             }
         });
@@ -187,7 +185,7 @@ struct WriterPipeline {
     }
 
     void flush() {
-        validatorInbox.push_move({ tao::json::null, EventSourceType::None, "" });
+        validatorInbox.push_move({ tao::json::null, });
         flushInbox.wait();
     }
 

src/apps/dbutils/cmd_import.cpp

@@ -55,7 +55,7 @@ void cmd_import(const std::vector<std::string> &subArgs) {
             continue;
         }
 
-        writer.write({ std::move(evJson), EventSourceType::Import, "" });
+        writer.write({ std::move(evJson), });
         writer.wait();
     }
 

src/apps/mesh/cmd_router.cpp

@@ -183,9 +183,9 @@ struct Router {
 
             std::string okMsg;
 
-            auto res = pluginDown.acceptEvent(pluginDownCmd, evJson, hoytech::curr_time_us(), EventSourceType::Stream, url, okMsg);
+            auto res = pluginDown.acceptEvent(pluginDownCmd, evJson, EventSourceType::Stream, url, okMsg);
             if (res == PluginEventSifterResult::Accept) {
-                router->writer.write({ std::move(evJson), EventSourceType::Stream, url });
+                router->writer.write({ std::move(evJson), });
             } else {
                 if (okMsg.size()) LI << groupName << " / " << url << " : pluginDown blocked event " << evJson.at("id").get_string() << ": " << okMsg;
             }
@@ -206,7 +206,7 @@ struct Router {
 
             std::string okMsg;
 
-            auto res = pluginUp.acceptEvent(pluginUpCmd, evJson, ev.receivedAt(), (EventSourceType)ev.sourceType(), ev.sourceInfo(), okMsg);
+            auto res = pluginUp.acceptEvent(pluginUpCmd, evJson, EventSourceType::Stored, "", okMsg);
             if (res == PluginEventSifterResult::Accept) {
                 for (auto &[url, c] : conns) {
                     if (c.ws) c.ws->send(responseStr.data(), responseStr.size(), uWS::OpCode::TEXT, nullptr, nullptr, true);

src/apps/mesh/cmd_stream.cpp

@@ -67,10 +67,10 @@ void cmd_stream(const std::vector<std::string> &subArgs) {
                     auto &evJson = origJson.at(2);
 
                     std::string okMsg;
-                    auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, hoytech::curr_time_us(), EventSourceType::Stream, ws.remoteAddr, okMsg);
+                    auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, EventSourceType::Stream, ws.remoteAddr, okMsg);
                     if (res == PluginEventSifterResult::Accept) {
                         downloadedIds.emplace(from_hex(evJson.at("id").get_string()));
-                        writer.write({ std::move(evJson), EventSourceType::Stream, url });
+                        writer.write({ std::move(evJson), });
                     } else {
                         if (okMsg.size()) LI << "[" << ws.remoteAddr << "] write policy blocked event " << evJson.at("id").get_string() << ": " << okMsg;
                     }

src/apps/mesh/cmd_sync.cpp

@@ -168,9 +168,9 @@ void cmd_sync(const std::vector<std::string> &subArgs) {
                 auto &evJson = msg.at(2);
 
                 std::string okMsg;
-                auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, hoytech::curr_time_us(), EventSourceType::Sync, ws.remoteAddr, okMsg);
+                auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, EventSourceType::Sync, ws.remoteAddr, okMsg);
                 if (res == PluginEventSifterResult::Accept) {
-                    writer.write({ std::move(evJson), EventSourceType::Sync, url });
+                    writer.write({ std::move(evJson), });
                 } else {
                     if (okMsg.size()) LI << "[" << ws.remoteAddr << "] write policy blocked event " << evJson.at("id").get_string() << ": " << okMsg;
                 }

src/apps/relay/RelayIngester.cpp

@@ -112,7 +112,7 @@ void RelayServer::ingesterProcessEvent(lmdb::txn &txn, uint64_t connId, std::str
         }
     }
 
-    output.emplace_back(MsgWriter{MsgWriter::AddEvent{connId, std::move(ipAddr), hoytech::curr_time_us(), std::move(packedStr), std::move(jsonStr)}});
+    output.emplace_back(MsgWriter{MsgWriter::AddEvent{connId, std::move(ipAddr), std::move(packedStr), std::move(jsonStr)}});
 }
 
 void RelayServer::ingesterProcessReq(lmdb::txn &txn, uint64_t connId, const tao::json::value &arr) {

src/apps/relay/RelayServer.h

@@ -65,7 +65,6 @@ struct MsgWriter : NonCopyable {
     struct AddEvent {
         uint64_t connId;
         std::string ipAddr;
-        uint64_t receivedAt;
         std::string packedStr;
         std::string jsonStr;
     };

src/apps/relay/RelayWriter.cpp

@@ -40,10 +40,10 @@ void RelayServer::runWriter(ThreadPool<MsgWriter>::Thread &thr) {
                 tao::json::value evJson = tao::json::from_string(msg->jsonStr);
                 EventSourceType sourceType = msg->ipAddr.size() == 4 ? EventSourceType::IP4 : EventSourceType::IP6;
                 std::string okMsg;
-                auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, msg->receivedAt, sourceType, msg->ipAddr, okMsg);
+                auto res = writePolicyPlugin.acceptEvent(cfg().relay__writePolicy__plugin, evJson, sourceType, msg->ipAddr, okMsg);
 
                 if (res == PluginEventSifterResult::Accept) {
-                    newEvents.emplace_back(std::move(msg->packedStr), std::move(msg->jsonStr), msg->receivedAt, sourceType, std::move(msg->ipAddr), msg);
+                    newEvents.emplace_back(std::move(msg->packedStr), std::move(msg->jsonStr), msg);
                 } else {
                     PackedEventView packed(msg->packedStr);
                     auto eventIdHex = to_hex(packed.id());

src/events.cpp

@@ -314,7 +314,7 @@ void writeEvents(lmdb::txn &txn, std::vector<EventToWrite> &evs, uint64_t logLev
         }
 
         if (ev.status == EventWriteStatus::Pending) {
-            ev.levId = env.insert_Event(txn, ev.receivedAt, ev.packedStr, (uint64_t)ev.sourceType, ev.sourceInfo);
+            ev.levId = env.insert_Event(txn, ev.packedStr);
 
             tmpBuf.clear();
             tmpBuf += '\x00';

src/events.h

@@ -63,6 +63,7 @@ enum class EventSourceType {
     Import = 3,
     Stream = 4,
     Sync = 5,
+    Stored = 6,
 };
 
 inline std::string eventSourceTypeToStr(EventSourceType t) {
@@ -88,16 +89,13 @@ enum class EventWriteStatus {
 struct EventToWrite {
     std::string packedStr;
     std::string jsonStr;
-    uint64_t receivedAt;
-    EventSourceType sourceType;
-    std::string sourceInfo;
     void *userData = nullptr;
     EventWriteStatus status = EventWriteStatus::Pending;
     uint64_t levId = 0;
 
     EventToWrite() {}
 
-    EventToWrite(std::string packedStr, std::string jsonStr, uint64_t receivedAt, EventSourceType sourceType, std::string sourceInfo, void *userData = nullptr) : packedStr(packedStr), jsonStr(jsonStr), receivedAt(receivedAt), sourceType(sourceType), sourceInfo(sourceInfo), userData(userData) {
+    EventToWrite(std::string packedStr, std::string jsonStr, void *userData = nullptr) : packedStr(packedStr), jsonStr(jsonStr), userData(userData) {
     }
 
     std::string_view id() {