GLOBAL FEED with issues:

1. We hardcoded wss://relay.damus.io/.  We need now mostly UI/UX setup for users to pick,
   possibly storage of relays to use for global feeds.
2. We are getting 2024-07-24T02:48:12.543807Z
   WARN gossip-lib/src/storage/mod.rs:1573: KINDS SCRAPE OF STORAGE
   so we should revisit if this matters (over the short limited events that we use)

gossip-bin/src/ui/feed/mod.rs

@@ -246,6 +246,18 @@ pub(super) fn update(app: &mut GossipUi, ctx: &Context, ui: &mut Ui) {
                 load_more,
             );
         }
+        FeedKind::Global => {
+            ui.add_space(10.0);
+            ui.horizontal(|ui| {
+                add_left_space(ui);
+                ui.heading("GLOBAL");
+                recompute_btn(ui);
+            });
+            ui.add_space(6.0);
+
+            let feed = GLOBALS.feed.get_feed_events();
+            render_a_feed(app, ctx, ui, feed, false, "global", load_more);
+        }
         FeedKind::DmChat(channel) => {
             if !GLOBALS.identity.is_unlocked() {
                 ui.add_space(10.0);

gossip-bin/src/ui/mod.rs

@@ -944,6 +944,7 @@ impl GossipUi {
                 ui.add_space(4.0);
 
                 self.add_feeds_submenu(ui, ctx);
+                self.add_global_feed(ui, ctx);
                 self.add_personal_notes(ui, ctx);
                 self.add_private_chats(ui, ctx);
                 self.add_search(ui, ctx);
@@ -1026,6 +1027,15 @@ impl GossipUi {
         self.after_openable_menu(ui, &cstate);
     }
 
+    fn add_global_feed(&mut self, ui: &mut Ui, ctx: &Context) {
+        if self
+            .add_selected_label(ui, self.page == Page::Feed(FeedKind::Global), "Global")
+            .clicked()
+        {
+            self.set_page(ctx, Page::Feed(FeedKind::Global));
+        }
+    }
+
     fn add_personal_notes(&mut self, ui: &mut Ui, ctx: &Context) {
         if let Some(pubkey) = GLOBALS.identity.public_key() {
             if self

gossip-lib/src/comms.rs

@@ -171,6 +171,9 @@ pub enum ToOverlordMessage {
     /// internal
     SetDmChannel(DmChannel),
 
+    /// internal
+    SetGlobalFeed(Unixtime),
+
     /// internal
     SetPersonFeed(PublicKey, Unixtime),
 
@@ -264,6 +267,7 @@ pub(crate) enum ToMinionPayloadDetail {
     SubscribeDiscover(Vec<PublicKey>),
     SubscribeGeneralFeed(Vec<PublicKey>, Unixtime),
     SubscribeGiftwraps(Unixtime),
+    SubscribeGlobalFeed(Unixtime),
     SubscribeInbox(Unixtime),
     SubscribePersonFeed(PublicKey, Unixtime),
     SubscribeReplies(IdHex),
@@ -274,6 +278,7 @@ pub(crate) enum ToMinionPayloadDetail {
     TempSubscribePersonFeedChunk { pubkey: PublicKey, anchor: Unixtime },
     TempSubscribeInboxFeedChunk(Unixtime),
     TempSubscribeMetadata(Vec<PublicKey>),
+    UnsubscribeGlobalFeed,
     UnsubscribePersonFeed,
     UnsubscribeReplies,
 }
@@ -300,6 +305,7 @@ pub enum RelayConnectionReason {
     PostNostrConnect,
     ReadThread,
     SubscribePerson,
+    SubscribeGlobal,
 }
 
 impl fmt::Display for RelayConnectionReason {
@@ -333,6 +339,7 @@ impl RelayConnectionReason {
             PostNostrConnect => "Posting nostrconnect",
             ReadThread => "Reading ancestors to build a thread",
             SubscribePerson => "Subscribe to the events of a person",
+            SubscribeGlobal => "Subscribe to the global feed on a relay",
         }
     }
 
@@ -359,6 +366,7 @@ impl RelayConnectionReason {
             PostNostrConnect => false,
             ReadThread => true,
             SubscribePerson => false,
+            SubscribeGlobal => false,
         }
     }
 }

gossip-lib/src/feed/feed_kind.rs

@@ -16,6 +16,7 @@ pub enum FeedKind {
     },
     Person(PublicKey),
     DmChat(DmChannel),
+    Global,
 }
 
 impl std::fmt::Display for FeedKind {
@@ -34,6 +35,7 @@ impl std::fmt::Display for FeedKind {
             } => write!(f, "Thread {}", crate::names::hex_id_short(&(*id).into())),
             FeedKind::Person(pk) => write!(f, "{}", crate::names::best_name_from_pubkey_lookup(pk)),
             FeedKind::DmChat(channel) => write!(f, "{}", channel.name()),
+            FeedKind::Global => write!(f, "Global"),
         }
     }
 }
@@ -48,6 +50,7 @@ impl FeedKind {
             Self::Thread { .. } => "thread".to_owned(),
             Self::Person(pubkey) => format!("person{}", pubkey.as_hex_string()),
             Self::DmChat(_) => "dmchat".to_owned(),
+            Self::Global => "global".to_owned(),
         }
     }
 
@@ -59,6 +62,7 @@ impl FeedKind {
             Self::Thread { .. } => false, // always full
             Self::Person(_) => true,
             Self::DmChat(_) => false, // always full
+            Self::Global => true,
         }
     }
 }

gossip-lib/src/feed/mod.rs

@@ -109,6 +109,18 @@ impl Feed {
                 },
             });
         }
+
+        // If not in the Global feed
+        if !matches!(feed_kind, FeedKind::Global) {
+            // Stop listening to Global events
+            let _ = GLOBALS.to_minions.send(ToMinionMessage {
+                target: "all".to_string(),
+                payload: ToMinionPayload {
+                    job_id: 0,
+                    detail: ToMinionPayloadDetail::UnsubscribeGlobalFeed,
+                },
+            });
+        }
     }
 
     pub fn switch_feed(&self, feed_kind: FeedKind) {
@@ -182,6 +194,11 @@ impl Feed {
                     .to_overlord
                     .send(ToOverlordMessage::SetDmChannel(dm_channel.clone()));
             }
+            FeedKind::Global => {
+                let _ = GLOBALS
+                    .to_overlord
+                    .send(ToOverlordMessage::SetGlobalFeed(anchor));
+            }
             _ => (),
         }
     }
@@ -370,6 +387,15 @@ impl Feed {
                 let ids = GLOBALS.storage.dm_events(&channel)?;
                 *self.current_feed_events.write() = ids;
             }
+            FeedKind::Global => {
+                let filter = {
+                    let mut filter = Filter::new();
+                    filter.kinds = feed_displayable_event_kinds(false);
+                    filter
+                };
+                let events = Self::load_event_range(anchor, filter, true, false, |_| true).await?;
+                *self.current_feed_events.write() = events;
+            }
         }
 
         *self.last_computed.write() = Some(Instant::now());

gossip-lib/src/minion/filter_fns.rs

@@ -143,6 +143,22 @@ pub fn person_feed(pubkey: PublicKey, range: FeedRange) -> Vec<Filter> {
     }]
 }
 
+pub fn global_feed(range: FeedRange) -> Vec<Filter> {
+    // Allow all feed related event kinds (excluding DMs)
+    // Do not load feed related or the limit will be wrong
+    let event_kinds = crate::feed::feed_displayable_event_kinds(false);
+
+    let (since, until, limit) = range.since_until_limit();
+
+    vec![Filter {
+        kinds: event_kinds,
+        since,
+        until,
+        limit,
+        ..Default::default()
+    }]
+}
+
 pub fn augments(ids: &[IdHex]) -> Vec<Filter> {
     let event_kinds = crate::feed::feed_augment_event_kinds();
 

gossip-lib/src/minion/mod.rs

@@ -643,6 +643,9 @@ impl Minion {
             ToMinionPayloadDetail::SubscribeGiftwraps(anchor) => {
                 self.subscribe_giftwraps(message.job_id, anchor).await?;
             }
+            ToMinionPayloadDetail::SubscribeGlobalFeed(anchor) => {
+                self.subscribe_global_feed(message.job_id, anchor).await?;
+            }
             ToMinionPayloadDetail::SubscribePersonFeed(pubkey, anchor) => {
                 self.subscribe_person_feed(message.job_id, pubkey, anchor)
                     .await?;
@@ -675,6 +678,9 @@ impl Minion {
                 self.temp_subscribe_metadata(message.job_id, pubkeys)
                     .await?;
             }
+            ToMinionPayloadDetail::UnsubscribeGlobalFeed => {
+                self.unsubscribe("global_feed").await?;
+            }
             ToMinionPayloadDetail::UnsubscribePersonFeed => {
                 self.unsubscribe("person_feed").await?;
             }
@@ -879,6 +885,30 @@ impl Minion {
         Ok(())
     }
 
+    async fn subscribe_global_feed(&mut self, job_id: u64, anchor: Unixtime) -> Result<(), Error> {
+        // NOTE we do not unsubscribe to the general feed
+
+        let limit = GLOBALS.storage.read_setting_load_more_count() as usize;
+        let mut filters = filter_fns::global_feed(FeedRange::After { since: anchor });
+        let filters2 = filter_fns::global_feed(FeedRange::ChunkBefore {
+            until: anchor,
+            limit,
+        });
+        filters.extend(filters2);
+
+        if filters.is_empty() {
+            self.unsubscribe_global_feed().await?;
+            self.to_overlord.send(ToOverlordMessage::MinionJobComplete(
+                self.url.clone(),
+                job_id,
+            ))?;
+        } else {
+            self.subscribe(filters, "global_feed", job_id).await?;
+        }
+
+        Ok(())
+    }
+
     async fn temp_subscribe_person_feed_chunk(
         &mut self,
         job_id: u64,
@@ -947,6 +977,17 @@ impl Minion {
         Ok(())
     }
 
+    async fn unsubscribe_global_feed(&mut self) -> Result<(), Error> {
+        // Unsubscribe global_feed and all person feed chunks
+        let handles = self
+            .subscription_map
+            .get_all_handles_matching("global_feed");
+        for handle in handles {
+            self.unsubscribe(&handle).await?;
+        }
+        Ok(())
+    }
+
     async fn unsubscribe_person_feed(&mut self) -> Result<(), Error> {
         // Unsubscribe person_feed and all person feed chunks
         let handles = self

gossip-lib/src/overlord.rs

@@ -711,6 +711,9 @@ impl Overlord {
             ToOverlordMessage::SetDmChannel(dmchannel) => {
                 self.set_dm_channel(dmchannel).await?;
             }
+            ToOverlordMessage::SetGlobalFeed(anchor) => {
+                self.set_global_feed(anchor).await?;
+            }
             ToOverlordMessage::SetPersonFeed(pubkey, anchor) => {
                 self.set_person_feed(pubkey, anchor).await?;
             }
@@ -2242,6 +2245,25 @@ impl Overlord {
         Ok(())
     }
 
+    async fn set_global_feed(&mut self, anchor: Unixtime) -> Result<(), Error> {
+        let relay_url = RelayUrl::try_from_str("wss://relay.damus.io/")?;
+        if let Some(relay) = GLOBALS.storage.read_relay(&relay_url, None)? {
+            // Subscribe
+            manager::engage_minion(
+                relay.url.clone(),
+                vec![RelayJob {
+                    reason: RelayConnectionReason::SubscribeGlobal,
+                    payload: ToMinionPayload {
+                        job_id: rand::random::<u64>(),
+                        detail: ToMinionPayloadDetail::SubscribeGlobalFeed(anchor),
+                    },
+                }],
+            );
+        }
+
+        Ok(())
+    }
+
     async fn set_person_feed(&mut self, pubkey: PublicKey, anchor: Unixtime) -> Result<(), Error> {
         let relays: Vec<RelayUrl> = relay::get_some_pubkey_outboxes(pubkey)?;
         manager::run_jobs_on_all_relays(