feat: per-site cron schedules editable from the Schedules UI

Adds a `scrape_cron` column on sites (TEXT, nullable) that overrides the
sitemap-derived changefreq when set. Cron expressions are validated with
cron-parser (added as a dep — node-cron's getNextRun returned nonsense
for some day-of-week expressions, so it's not reliable for scheduling).

Backend:
- `PATCH /api/sites/:id/schedule` now accepts an optional `cron` field
  (string | null). Empty/null clears the override and falls back to
  sitemap default. Invalid expressions return 400.
- Pending scheduled_jobs row gets its next_run_at recomputed on save so
  a frequency change takes effect immediately, not after the next scrape.
- ScrapeScheduler.scheduleNextRun consults `sites.scrape_cron` first and
  only falls back to the changefreq map when it's null.

Frontend:
- SchedulesPage adds a cron text input per site row, four preset buttons
  (Hourly / Daily / Weekly / Monthly), and a Save button. Shows the
  current cron expression in the card header (or "sitemap default" when
  unset).
- ApiService.updateSiteSchedule now takes `{ enabled?, cron? }` so the
  same endpoint handles toggle and cron updates.

package-lock.json

@@ -17,6 +17,7 @@
         "better-sqlite3": "^12.4.1",
         "cheerio": "^1.0.0-rc.12",
         "compression": "^1.7.4",
+        "cron-parser": "^5.5.0",
         "express": "^4.18.2",
         "node-cron": "^4.2.1",
         "openai": "^6.9.1",
@@ -1006,6 +1007,17 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cron-parser": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.5.0.tgz",
+      "integrity": "sha512-oML4lKUXxizYswqmxuOCpgFS8BNUJpIu6k/2HVHyaL8Ynnf3wdf9tkns0yRdJLSIjkJ+b0DXHMZEHGpMwjnPww==",
+      "dependencies": {
+        "luxon": "^3.7.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -1673,6 +1685,14 @@
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
       "license": "MIT"
     },
+    "node_modules/luxon": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz",
+      "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/make-error": {
       "version": "1.3.6",
       "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",

package.json

@@ -31,6 +31,7 @@
     "better-sqlite3": "^12.4.1",
     "cheerio": "^1.0.0-rc.12",
     "compression": "^1.7.4",
+    "cron-parser": "^5.5.0",
     "express": "^4.18.2",
     "node-cron": "^4.2.1",
     "openai": "^6.9.1",

src/api/components/SitesEndpoint.ts

@@ -254,9 +254,12 @@ export class SitesEndpoint extends BaseEndpointComponent {
                 enabled: {
                   type: 'boolean',
                   description: 'Whether to enable or disable scheduling'
+                },
+                cron: {
+                  type: ['string', 'null'],
+                  description: 'Standard 5-field cron expression overriding the sitemap-derived cadence. Pass null to clear and fall back to sitemap defaults.'
                 }
-              },
-              required: ['enabled']
+              }
             }
           },
           responses: {
@@ -267,13 +270,14 @@ export class SitesEndpoint extends BaseEndpointComponent {
                 properties: {
                   site_id: { type: 'string' },
                   schedule_enabled: { type: 'boolean' },
+                  scrape_cron: { type: ['string', 'null'] },
+                  next_run_at: { type: ['string', 'null'] },
                   message: { type: 'string' }
                 }
               }
             },
-            '404': {
-              description: 'Site not found'
-            }
+            '400': { description: 'Invalid cron expression' },
+            '404': { description: 'Site not found' }
           },
           requiresAuth: true
         }
@@ -732,26 +736,55 @@ export class SitesEndpoint extends BaseEndpointComponent {
       }
 
       const { id } = req.params;
-      const { enabled } = req.body;
+      const { enabled, cron } = req.body;
 
-      if (typeof enabled !== 'boolean') {
-        res.status(400).json({ error: 'enabled field must be a boolean' });
+      // At least one of the two must be present.
+      if (typeof enabled !== 'boolean' && cron === undefined) {
+        res.status(400).json({ error: 'either `enabled` (boolean) or `cron` (string|null) must be provided' });
+        return;
+      }
+      if (enabled !== undefined && typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled must be a boolean' });
         return;
       }
 
-      // Get site by any ID (try custom_id first, then internal ID)
       const site = this.db.getSiteByAnyId(id);
       if (!site) {
         res.status(404).json({ error: 'Site not found' });
         return;
       }
 
-      this.db.setScheduleEnabled(site.id, enabled);
+      let nextRunAt: string | null = null;
+      if (cron !== undefined) {
+        if (cron !== null && typeof cron !== 'string') {
+          res.status(400).json({ error: 'cron must be a string or null' });
+          return;
+        }
+        if (cron !== null && cron.trim() !== '') {
+          const { isValidCron, getNextCronRun } = await import('../../utils/cron');
+          if (!isValidCron(cron)) {
+            res.status(400).json({ error: `invalid cron expression: ${cron}` });
+            return;
+          }
+          const next = getNextCronRun(cron);
+          nextRunAt = next ? next.toISOString() : null;
+        }
+        // null or empty string both clear the override
+        const value = cron && cron.trim() !== '' ? cron.trim() : null;
+        this.db.setSiteScrapeCron(site.id, value, nextRunAt);
+      }
+
+      if (typeof enabled === 'boolean') {
+        this.db.setScheduleEnabled(site.id, enabled);
+      }
 
+      const updated = this.db.getSiteByAnyId(site.id);
       res.json({
         site_id: site.id,
-        schedule_enabled: enabled,
-        message: `Schedule ${enabled ? 'enabled' : 'disabled'} successfully`
+        schedule_enabled: typeof enabled === 'boolean' ? enabled : undefined,
+        scrape_cron: updated?.scrape_cron ?? null,
+        next_run_at: nextRunAt,
+        message: 'Schedule updated successfully'
       });
     } catch (error) {
       logger.error('Error updating schedule status', error);

src/database/Database.ts

@@ -13,6 +13,7 @@ export interface Site {
   site_type: string;
   site_platform: string;
   custom_sitemap_url: string | null;
+  scrape_cron?: string | null;
   max_crawl_depth: number;
   max_pages: number;
   qdrant_enabled: boolean;
@@ -293,6 +294,16 @@ export class SiteDatabase {
       }
     }
 
+    // Add scrape_cron column for per-site custom scrape schedules
+    try {
+      this.db.exec(`ALTER TABLE sites ADD COLUMN scrape_cron TEXT`);
+      logger.info('Added scrape_cron column to sites table');
+    } catch (error: any) {
+      if (!error.message.includes('duplicate column name')) {
+        logger.error('Error adding scrape_cron column:', error);
+      }
+    }
+
     // Site analytics table
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS site_analytics (
@@ -682,6 +693,7 @@ export class SiteDatabase {
       custom_sitemap_url: row.custom_sitemap_url,
       max_crawl_depth: row.max_crawl_depth,
       max_pages: row.max_pages,
+      scrape_cron: row.scrape_cron,
       qdrant_enabled: Boolean(row.qdrant_enabled),
       deleted_at: row.deleted_at,
       created_at: row.created_at,
@@ -1508,6 +1520,28 @@ export class SiteDatabase {
     logger.info(`${enabled ? 'Enabled' : 'Disabled'} schedule for site ${site.id}`);
   }
 
+  /**
+   * Set or clear the user-defined cron expression for a site. When set, the
+   * ScrapeScheduler uses this instead of the sitemap-derived changefreq when
+   * scheduling the next run. Also re-times any pending scheduled_jobs row so
+   * the new cadence takes effect immediately.
+   */
+  setSiteScrapeCron(siteId: string, cron: string | null, nextRunAt: string | null): void {
+    const now = new Date().toISOString();
+    this.db.prepare(`UPDATE sites SET scrape_cron = ?, updated_at = ? WHERE id = ?`)
+      .run(cron, now, siteId);
+    if (nextRunAt) {
+      // Update any pending queued job so the new frequency takes effect now.
+      this.db.prepare(`
+        UPDATE scheduled_jobs
+        SET next_run_at = ?
+        WHERE site_id = ? AND status = 'queued'
+      `).run(nextRunAt, siteId);
+      this.setNextScrapeTime(siteId, nextRunAt);
+    }
+    logger.info(`Updated scrape_cron for site ${siteId}: ${cron ?? '(cleared)'}`);
+  }
+
   // Webhook operations
   createWebhook(siteIdentifier: string, url: string, secret?: string): string {
     const site = this.getSiteByAnyId(siteIdentifier);

src/scheduler/ScrapeScheduler.ts

@@ -4,6 +4,7 @@ import { JobQueue } from '../queue/JobQueue';
 import { logger } from '../utils/logger';
 import { v4 as uuidv4 } from 'uuid';
 import { WebhookManager } from '../webhooks/WebhookManager';
+import { getNextCronRun } from '../utils/cron';
 
 export class ScrapeScheduler {
   private db: SiteDatabase;
@@ -101,23 +102,38 @@ export class ScrapeScheduler {
     const now = new Date();
     let nextRunAt: Date;
 
-    // Map sitemap changefreq to actual intervals
-    const frequencyMap: { [key: string]: number } = {
-      'always': 60 * 60 * 1000, // 1 hour
-      'hourly': 60 * 60 * 1000, // 1 hour
-      'daily': 24 * 60 * 60 * 1000, // 1 day
-      'weekly': 7 * 24 * 60 * 60 * 1000, // 1 week
-      'monthly': 30 * 24 * 60 * 60 * 1000, // 30 days
-      'yearly': 365 * 24 * 60 * 60 * 1000, // 1 year
-      'never': 365 * 24 * 60 * 60 * 1000 // 1 year (still check once a year)
-    };
-
-    const frequency = scheduledJob.frequency?.toLowerCase();
-    const interval = frequency && frequencyMap[frequency]
-      ? frequencyMap[frequency]
-      : 7 * 24 * 60 * 60 * 1000; // Default to weekly
-
-    nextRunAt = new Date(now.getTime() + interval);
+    // User-supplied cron overrides the sitemap-derived cadence.
+    const site = this.db.getSiteByAnyId(scheduledJob.site_id);
+    const userCron = site?.scrape_cron;
+    if (userCron) {
+      const next = getNextCronRun(userCron, now);
+      if (next) {
+        nextRunAt = next;
+        logger.info(`Using user cron "${userCron}" for site ${scheduledJob.site_id} -> ${nextRunAt.toISOString()}`);
+      } else {
+        // Invalid cron in DB (shouldn't happen since we validate on save); fall through.
+        logger.warn(`Site ${scheduledJob.site_id} has invalid scrape_cron "${userCron}", falling back to weekly`);
+        nextRunAt = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000);
+      }
+    } else {
+      // Map sitemap changefreq to actual intervals
+      const frequencyMap: { [key: string]: number } = {
+        'always': 60 * 60 * 1000, // 1 hour
+        'hourly': 60 * 60 * 1000, // 1 hour
+        'daily': 24 * 60 * 60 * 1000, // 1 day
+        'weekly': 7 * 24 * 60 * 60 * 1000, // 1 week
+        'monthly': 30 * 24 * 60 * 60 * 1000, // 30 days
+        'yearly': 365 * 24 * 60 * 60 * 1000, // 1 year
+        'never': 365 * 24 * 60 * 60 * 1000 // 1 year (still check once a year)
+      };
+
+      const frequency = scheduledJob.frequency?.toLowerCase();
+      const interval = frequency && frequencyMap[frequency]
+        ? frequencyMap[frequency]
+        : 7 * 24 * 60 * 60 * 1000; // Default to weekly
+
+      nextRunAt = new Date(now.getTime() + interval);
+    }
 
     // Create a new scheduled job for the next run
     this.db.createScheduledJob(

src/utils/cron.ts

@@ -0,0 +1,26 @@
+import { CronExpressionParser } from 'cron-parser';
+
+/**
+ * Cron-expression helpers. We use cron-parser (not node-cron) for next-run
+ * computation because node-cron's getNextRun returns nonsense for some
+ * day-of-week expressions like `0 0 * * 0`. cron-parser is purpose-built
+ * for parsing/iterating cron strings and is more accurate.
+ */
+
+export function isValidCron(expression: string): boolean {
+  try {
+    CronExpressionParser.parse(expression);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function getNextCronRun(expression: string, from: Date = new Date()): Date | null {
+  try {
+    const it = CronExpressionParser.parse(expression, { currentDate: from });
+    return it.next().toDate();
+  } catch {
+    return null;
+  }
+}

web/package-lock.json

@@ -821,7 +821,6 @@
       "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "undici-types": "~6.21.0"
       }
@@ -948,7 +947,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "baseline-browser-mapping": "^2.8.25",
         "caniuse-lite": "^1.0.30001754",
@@ -1311,7 +1309,6 @@
       "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "jiti": "bin/jiti.js"
       }
@@ -1511,7 +1508,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "nanoid": "^3.3.11",
         "picocolors": "^1.1.1",
@@ -1945,7 +1941,6 @@
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=12"
       },

web/src/components/pages/SchedulesPage.ts

@@ -206,6 +206,9 @@ export class SchedulesPage {
     // Find next scheduled run - for enabled, use active; for disabled, find most recent queued
     const nextRun = hasEnabledSchedule ? activeSchedule : siteSchedules.find(s => s.status === 'queued');
 
+    const currentCron = site.scrape_cron || '';
+    const cronLabel = currentCron ? `Cron: <code class="text-xs">${currentCron}</code>` : 'Cron: <span class="text-xs italic">sitemap default</span>';
+
     return `
       <div class="p-4 border border-gray-200 dark:border-gray-600 rounded-lg hover:shadow-md transition-shadow">
         <div class="flex items-center justify-between">
@@ -284,6 +287,28 @@ export class SchedulesPage {
           </div>
         </div>
 
+        <!-- Cron Editor -->
+        <div class="mt-4 pt-4 border-t border-gray-200 dark:border-gray-600">
+          <div class="flex items-center justify-between mb-2">
+            <h5 class="text-sm font-medium text-gray-900 dark:text-white">Crawl frequency</h5>
+            <span class="text-xs text-gray-500 dark:text-gray-400">${cronLabel}</span>
+          </div>
+          <div class="flex items-center gap-2 flex-wrap">
+            <input
+              type="text"
+              class="cron-input flex-1 min-w-[200px] px-2 py-1 text-xs font-mono border rounded dark:bg-gray-700 dark:border-gray-600 dark:text-white"
+              placeholder="e.g. 0 3 * * * (daily at 03:00) — leave blank for sitemap default"
+              value="${currentCron}"
+              data-site-id="${site.id}"
+            />
+            <button type="button" class="btn btn-secondary btn-xs cron-preset" data-cron="0 * * * *" data-site-id="${site.id}" title="Every hour">Hourly</button>
+            <button type="button" class="btn btn-secondary btn-xs cron-preset" data-cron="0 3 * * *" data-site-id="${site.id}" title="Every day at 03:00">Daily</button>
+            <button type="button" class="btn btn-secondary btn-xs cron-preset" data-cron="0 3 * * 0" data-site-id="${site.id}" title="Every Sunday at 03:00">Weekly</button>
+            <button type="button" class="btn btn-secondary btn-xs cron-preset" data-cron="0 3 1 * *" data-site-id="${site.id}" title="1st of month at 03:00">Monthly</button>
+            <button type="button" class="btn btn-primary btn-xs cron-save" data-site-id="${site.id}">Save</button>
+          </div>
+        </div>
+
         <!-- Schedule Details -->
         ${siteSchedules.length > 0 ? `
           <div class="mt-4 pt-4 border-t border-gray-200 dark:border-gray-600">
@@ -354,6 +379,52 @@ export class SchedulesPage {
   private bindToggleEvents(): void {
     if (!this.element) return;
 
+    // Preset buttons fill the input for that site
+    this.element.querySelectorAll('.cron-preset').forEach(btn => {
+      btn.addEventListener('click', (e) => {
+        const target = e.currentTarget as HTMLButtonElement;
+        const siteId = target.getAttribute('data-site-id');
+        const cron = target.getAttribute('data-cron') || '';
+        const input = this.element?.querySelector(`input.cron-input[data-site-id="${siteId}"]`) as HTMLInputElement | null;
+        if (input) input.value = cron;
+      });
+    });
+
+    // Save button persists the cron expression
+    this.element.querySelectorAll('.cron-save').forEach(btn => {
+      btn.addEventListener('click', async (e) => {
+        const target = e.currentTarget as HTMLButtonElement;
+        const siteId = target.getAttribute('data-site-id');
+        if (!siteId) return;
+        const input = this.element?.querySelector(`input.cron-input[data-site-id="${siteId}"]`) as HTMLInputElement | null;
+        const raw = input?.value.trim() ?? '';
+        const cron = raw === '' ? null : raw;
+
+        target.disabled = true;
+        try {
+          const response = await this.apiService.updateSiteSchedule(siteId, { cron });
+          if (response.success) {
+            const next = response.data?.next_run_at;
+            this.toastService.success(
+              'Schedule Updated',
+              cron
+                ? `Cron set to "${cron}"${next ? ` — next run ${new Date(next).toLocaleString()}` : ''}`
+                : 'Cron cleared — using sitemap default'
+            );
+            await this.loadScheduleDetails();
+            this.renderStats();
+            this.renderSchedulesList();
+          } else {
+            this.toastService.error('Update Failed', response.error || 'Failed to update cron');
+          }
+        } catch (error) {
+          this.toastService.error('Error', 'Failed to update cron');
+        } finally {
+          target.disabled = false;
+        }
+      });
+    });
+
     this.element.querySelectorAll('.schedule-toggle').forEach(toggle => {
       toggle.addEventListener('change', async (e) => {
         const checkbox = e.target as HTMLInputElement;
@@ -367,7 +438,7 @@ export class SchedulesPage {
         checkbox.disabled = true;
 
         try {
-          const response = await this.apiService.updateSiteSchedule(siteId, enabled);
+          const response = await this.apiService.updateSiteSchedule(siteId, { enabled });
 
           if (response.success) {
             this.toastService.success(

web/src/services/ApiService.ts

@@ -179,11 +179,11 @@ export class ApiService {
   // Schedule Management
   async updateSiteSchedule(
     siteId: string,
-    enabled: boolean
-  ): Promise<ApiResponse<{ message: string }>> {
+    payload: { enabled?: boolean; cron?: string | null }
+  ): Promise<ApiResponse<{ message: string; scrape_cron?: string | null; next_run_at?: string | null }>> {
     return this.request(`/api/sites/${siteId}/schedule`, {
       method: 'PATCH',
-      body: JSON.stringify({ enabled }),
+      body: JSON.stringify(payload),
     });
   }
 

web/src/types/index.ts

@@ -9,6 +9,7 @@ export interface Site {
   custom_sitemap_url?: string | null;
   max_crawl_depth?: number;
   max_pages?: number;
+  scrape_cron?: string | null;
   qdrant_enabled: boolean;
   deleted_at: string | null;
   created_at: string;