fix: drop stale-category rows and prune orphan Qdrant points

Two follow-up cleanups for issues surfaced by the URL-normalization fix:

1. Stale category orphans. The URL classifier's keyword list has grown
   over time — a URL stored as 'contacts' months ago may classify as
   'team' today, but the system never deleted the old row when
   reclassification happened. New SiteDatabase.cleanupStaleCategoryRows
   walks every row for a site, computes the expected content_type
   (custom_urls override the classifier), and DELETEs rows whose stored
   type doesn't match. Wired into WebScraper.scrape after the existing
   cleanups. On artdent this caught 3 rows from 2026-04-09 that have
   been orphaned since the classifier was updated.

2. Qdrant orphan points. When a site_content row is deleted (URL form
   change, category reclass, etc.), the Qdrant points whose ID hashed
   from that URL stay around — wasting storage and returning duplicate
   results in semantic search. New QdrantService.pruneOrphanPoints
   scrolls a collection and deletes any point whose `url` payload isn't
   in the supplied valid-URL set. QdrantCleanupService.performCleanup
   now runs this for every Qdrant-enabled site (one valid-URL set per
   category, built from site_content), so the scheduled 24h cleanup
   collects orphans automatically. The existing
   `POST /api/qdrant/cleanup` endpoint also exercises this code path
   for manual triggers.

src/database/Database.ts

@@ -1331,6 +1331,76 @@ export class SiteDatabase {
     return removedUrls;
   }
 
+  /**
+   * Remove rows whose stored content_type no longer matches what classifyFn
+   * returns for the URL today. Caused by the classifier's keyword list growing
+   * over time — a URL stored as 'contacts' in April may classify as 'team' now,
+   * leaving a stale orphan row in the old category.
+   *
+   * Custom URLs (which have a user-specified content_type via the custom_urls
+   * table) take precedence over the classifier output. They define the
+   * "expected" type for their URL.
+   *
+   * Returns the deleted (url, content_type) tuples so the caller can clean up
+   * matching Qdrant points.
+   */
+  cleanupStaleCategoryRows(
+    siteId: string,
+    classifyFn: (url: string) => ContentType
+  ): Array<{ url: string; content_type: ContentType }> {
+    const rows = this.db.prepare(`
+      SELECT id, url, content_type FROM site_content WHERE site_id = ?
+    `).all(siteId) as { id: string; url: string; content_type: ContentType }[];
+
+    const customRows = this.db.prepare(`
+      SELECT url, content_type FROM custom_urls WHERE site_id = ? AND enabled = 1
+    `).all(siteId) as { url: string; content_type: string }[];
+    const customMap = new Map<string, ContentType>();
+    for (const cu of customRows) {
+      const norm = normalizeUrl(cu.url);
+      if (norm) customMap.set(norm, cu.content_type as ContentType);
+    }
+
+    const idsToDelete: string[] = [];
+    const removed: Array<{ url: string; content_type: ContentType }> = [];
+    for (const row of rows) {
+      const expected = customMap.get(row.url) ?? classifyFn(row.url);
+      if (row.content_type !== expected) {
+        idsToDelete.push(row.id);
+        removed.push({ url: row.url, content_type: row.content_type });
+      }
+    }
+
+    if (idsToDelete.length > 0) {
+      const placeholders = idsToDelete.map(() => '?').join(',');
+      this.db.prepare(`DELETE FROM site_content WHERE id IN (${placeholders})`).run(...idsToDelete);
+      const preview = removed.slice(0, 5).map(r => `${r.url}[${r.content_type}]`).join(', ');
+      logger.warn(`Removed ${idsToDelete.length} stale category rows for site ${siteId}: ${preview}${removed.length > 5 ? `, +${removed.length - 5} more` : ''}`);
+    }
+    return removed;
+  }
+
+  /**
+   * Return every URL the DB currently knows about for this site, grouped by
+   * content_type. Used by the Qdrant orphan-prune pass to figure out which
+   * points are still backed by a row.
+   */
+  getContentUrlsByCategory(siteId: string): Map<ContentType, Set<string>> {
+    const rows = this.db.prepare(`
+      SELECT url, content_type FROM site_content WHERE site_id = ?
+    `).all(siteId) as { url: string; content_type: ContentType }[];
+    const out = new Map<ContentType, Set<string>>();
+    for (const r of rows) {
+      let set = out.get(r.content_type);
+      if (!set) {
+        set = new Set();
+        out.set(r.content_type, set);
+      }
+      set.add(r.url);
+    }
+    return out;
+  }
+
   /**
    * Clean up duplicate content entries by content_hash
    * When the same content appears in multiple categories, keep only the first occurrence

src/scraper/WebScraper.ts

@@ -272,6 +272,10 @@ export class WebScraper {
         // Collapse non-canonical URL variants (e.g. /araink + /araink/) into one
         // row per content_hash. Cheap idempotent pass — on a clean DB it's a no-op.
         this.db.cleanupTrailingSlashDuplicates(siteId);
+
+        // Drop rows whose stored content_type no longer matches the classifier's
+        // current output for that URL (classifier keywords have grown over time).
+        this.db.cleanupStaleCategoryRows(siteId, SitemapParser.classifyUrl.bind(SitemapParser));
       }
 
       const result: ScraperResult = {

src/services/QdrantCleanupService.ts

@@ -9,6 +9,7 @@ export interface CleanupStats {
   errorsResolved: number;
   deletionQueueProcessed: number;
   mcpLogsArchived: number;
+  orphanPointsPruned: number;
 }
 
 export class QdrantCleanupService {
@@ -32,6 +33,7 @@ export class QdrantCleanupService {
       errorsResolved: 0,
       deletionQueueProcessed: 0,
       mcpLogsArchived: 0,
+      orphanPointsPruned: 0,
     };
 
     if (!this.config.qdrant.enabled) {
@@ -53,6 +55,10 @@ export class QdrantCleanupService {
       // Archive old MCP logs (90+ days old)
       stats.mcpLogsArchived = await this.archiveMcpLogs();
 
+      // Prune points whose URL no longer exists in site_content (orphans from
+      // URL-form changes, category reclassification, manual DB edits, etc.)
+      stats.orphanPointsPruned = await this.pruneOrphanPointsForAllSites();
+
       logger.info('Qdrant cleanup completed', stats);
       return stats;
 
@@ -62,6 +68,45 @@ export class QdrantCleanupService {
     }
   }
 
+  /**
+   * For every Qdrant-enabled site, walk its collections and delete points
+   * whose `url` payload doesn't match a row in site_content. Each collection
+   * is one category (`{custom_id}-{category}`); the valid URL set for that
+   * category comes from the DB.
+   */
+  private async pruneOrphanPointsForAllSites(): Promise<number> {
+    let totalPruned = 0;
+    const sites = this.database.getQdrantEnabledSites();
+
+    for (const site of sites) {
+      if (!site.custom_id) continue;
+      let collections: string[];
+      try {
+        collections = await this.qdrantService.getCollectionsForShop(site.custom_id);
+      } catch (error) {
+        logger.warn(`Failed to list collections for site ${site.id}:`, error);
+        continue;
+      }
+
+      const validByCategory = this.database.getContentUrlsByCategory(site.id);
+      for (const collectionName of collections) {
+        const parsed = this.qdrantService.parseCollectionName(collectionName);
+        if (!parsed) continue;
+        const validUrls = validByCategory.get(parsed.category as any) ?? new Set<string>();
+        try {
+          totalPruned += await this.qdrantService.pruneOrphanPoints(collectionName, validUrls);
+        } catch (error) {
+          logger.warn(`Orphan prune failed for ${collectionName}:`, error);
+        }
+      }
+    }
+
+    if (totalPruned > 0) {
+      logger.info(`Pruned ${totalPruned} orphan Qdrant points across ${sites.length} sites`);
+    }
+    return totalPruned;
+  }
+
   /**
    * Process items in the deletion queue that are past their delay period
    */

src/services/QdrantService.ts

@@ -370,6 +370,63 @@ export class QdrantService {
    * Find and remove duplicate points in a collection (keep only latest by content_hash)
    * Returns the number of duplicates removed
    */
+  /**
+   * Delete points whose `url` payload is not present in `validUrls`. Used to
+   * remove orphans left behind when DB rows are deleted (URL form changes,
+   * category reclassification, etc.). Returns the number of points removed.
+   */
+  async pruneOrphanPoints(collectionName: string, validUrls: Set<string>): Promise<number> {
+    if (!this.enabled) {
+      logger.warn('Attempted to prune orphans while Qdrant is disabled');
+      return 0;
+    }
+
+    try {
+      const orphanIds: string[] = [];
+      let offset: string | undefined = undefined;
+
+      while (true) {
+        const response = await this.client.scroll(collectionName, {
+          limit: 256,
+          offset,
+          with_payload: true,
+          with_vector: false,
+        });
+
+        if (!response.points || response.points.length === 0) break;
+
+        for (const point of response.points) {
+          const url = (point.payload as any)?.url;
+          if (typeof url === 'string' && !validUrls.has(url)) {
+            orphanIds.push(String(point.id));
+          }
+        }
+
+        offset = response.next_page_offset as string | undefined;
+        if (!offset) break;
+      }
+
+      if (orphanIds.length > 0) {
+        const batchSize = 256;
+        for (let i = 0; i < orphanIds.length; i += batchSize) {
+          await this.client.delete(collectionName, {
+            wait: true,
+            points: orphanIds.slice(i, i + batchSize),
+          });
+        }
+        logger.info(`Pruned ${orphanIds.length} orphan points from collection ${collectionName}`);
+      }
+      return orphanIds.length;
+    } catch (error: any) {
+      if (error.status === 404) {
+        logger.debug(`Collection ${collectionName} does not exist, skipping orphan prune`);
+        return 0;
+      }
+      logger.error(`Failed to prune orphans in collection ${collectionName}:`, error);
+      throw error;
+    }
+  }
+
   async deduplicateCollection(collectionName: string): Promise<number> {
     if (!this.enabled) {
       logger.warn('Attempted to deduplicate while Qdrant is disabled');