From e87faa6ac650fe00388159ce149ec95777a11608 Mon Sep 17 00:00:00 2001
From: Adam Soltys <adam@coinos.io>
Date: Wed, 11 Nov 2026 22:11:05 -0700
Subject: Post search shows a bot fleet's newsletter once, not forty-six times.
 "searching "fiatjaf" surfaced this same post over and over i guess from
 different people all boosting it" (Adam, 2026-11-11). Not boosts: about a
 dozen keys each post the NostrMag "Quiet Coup" issue every few hours,
 reworded a little each time (an emoji, "BREAKING", the picture moved), so the
 exact-repeat spam check never matches. In a search feed a post whose words
 and links overlap half or more (Jaccard) with one already shown is dropped;
 posts under 8 words are left alone. Measured on 120 live hits for "fiatjaf"
 from the three search relays: newsletter copies overlap 0.53 and up, distinct
 posts 0.45 at most; 46 copies → 1, nothing else dropped. post-search-test
 adds three reworded copies from three keys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VMan6UKGJyuGyZFxddYRxS
---
 src/features/messages.js | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

(limited to 'src/features')

diff --git a/src/features/messages.js b/src/features/messages.js
index 3ed1fa7..a42396c 100644
--- a/src/features/messages.js
+++ b/src/features/messages.js
@@ -7504,6 +7504,35 @@ export function messagesFeature(ctx) {
   // Posts at the door: filtered in synchronously, so the same note from a
   // second relay is dropped while the first copy is still warming.
   const feedStaged = new WeakMap();
+  // A word search turns up bot fleets posting one newsletter dozens of times
+  // from a dozen keys, each copy reworded a little (an emoji, "BREAKING", the
+  // picture moved) so it isn't the same text. Words and links as a set: a
+  // post sharing half of them with one already in the search feed is that
+  // post again, and only the first is shown.
+  const textSigs = new WeakMap(); // event -> Set of words
+  function textSig(e) {
+    let sig = textSigs.get(e);
+    if (!sig) {
+      const words = String(e.content || '').toLowerCase().match(/https?:\/\/\S+|[\p{L}\p{N}]{3,}/gu) || [];
+      textSigs.set(e, sig = new Set(words));
+    }
+    return sig;
+  }
+  const NEAR_COPY = 0.5; // measured 2026-11-11: copies 0.53+, distinct posts at most 0.45
+  const nearCopies = new WeakMap(); // feed state -> sigs admitted
+  function nearCopy(c, held, e) {
+    const sig = textSig(e);
+    if (sig.size < 8) return false; // short posts ("gm fiatjaf") are allowed to agree
+    let seen = nearCopies.get(c);
+    if (!seen) nearCopies.set(c, seen = held.map(textSig).filter((x) => x.size >= 8));
+    for (const o of seen) {
+      let both = 0;
+      for (const w of sig) if (o.has(w)) both++;
+      if (both / (sig.size + o.size - both) >= NEAR_COPY) return true;
+    }
+    seen.push(sig);
+    return false;
+  }
   async function mergeFeed(evs, opts = {}, c = feedNow()) {
     let staged = feedStaged.get(c);
     if (!staged) feedStaged.set(c, staged = new Set());
@@ -7525,6 +7554,7 @@ export function messagesFeature(ctx) {
     const def = feedDef(c.id);
     let add = (evs || []).filter((e) => (e.kind === REPOST ? !(def && def.replies) && !!repostInner(e) && !hidden(repostInner(e)) && !reboosted(e) : feedKeeps(def, e)) && !hidden(e)
       && fresh(e) && !known.has(e.id) && !staged.has(e.id) && addrNew(e) && known.add(e.id) && staged.add(e.id) && staged.add(noteOf(e).id));
+    if (def && def.search) add = add.filter((e) => !nearCopy(c, held, e));
     if (!add.length) return false;
     if (def && def.curated) {
       const kept = curateFeed(add);
-- 
cgit v1.2.3