feat: enhance HTTP access logging with status codes and response times

Logging Improvements:
- Add HTTP status codes to all access logs with emoji indicators
- Track and log response time for every request (in milliseconds)
- Include content-length in responses
- Add optional verbose mode with User-Agent logging

New Logger Features:
- New logger.http() method for structured HTTP logging
- Color-coded status indicators (✓ 2xx, ↪️ 3xx, ⚠️ 4xx, ❌ 5xx)
- LOG_HTTP_VERBOSE environment variable for detailed logs
- Automatic detection of systemd vs development mode

Log Format Examples:
Standard: ✓ 200 GET /api/shops ::1 45ms
Verbose:  ✓ 200 GET /api/shops ::1 45ms 2456b UA:"curl/7.81.0"

Benefits:
- Easy identification of slow requests (response time tracking)
- Quick visual scanning with status code emojis
- Better debugging with optional verbose User-Agent info
- Performance monitoring and metrics collection ready
- Compatible with existing systemd/journalctl workflow

Updated README with:
- Logging configuration options
- Log format examples
- Status code indicator reference
- journalctl filtering examples

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

README.md

@@ -43,6 +43,9 @@ API_KEY=your-secret-key
 PORT=3000
 MAX_CONCURRENT_JOBS=3
 
+# Logging (Optional)
+LOG_HTTP_VERBOSE=true  # Enable detailed HTTP logs with User-Agent and content-length
+
 # Qdrant Vector Search (Optional)
 QDRANT_API_URL=http://localhost:6333
 QDRANT_API_KEY=your-qdrant-key
@@ -160,9 +163,45 @@ Data is stored in `data/shops.db` (SQLite). The database includes:
 
 ## Logging
 
-View logs using journalctl:
+The system includes enhanced HTTP access logging with status codes, response times, and optional verbose mode.
+
+### Log Format
+
+**Standard HTTP logs:**
+```
+[INFO] [WebshopScraper] ✓ 200 GET /api/shops ::1 45ms
+[INFO] [WebshopScraper] ⚠️ 404 GET /api/shops/invalid ::1 12ms
+[INFO] [WebshopScraper] ❌ 500 POST /api/jobs ::1 234ms
+```
+
+**Verbose HTTP logs** (with `LOG_HTTP_VERBOSE=true`):
+```
+[INFO] [WebshopScraper] ✓ 200 GET /api/shops ::1 45ms 2456b UA:"curl/7.81.0"
+```
+
+### Status Code Indicators
+- ✓ (200-299) - Success
+- ↪️ (300-399) - Redirect
+- ⚠️ (400-499) - Client error
+- ❌ (500-599) - Server error
+
+### View Logs
+
+Using journalctl (systemd):
 ```bash
+# Follow logs in real-time
 sudo journalctl -u webshop-scraper -f
+
+# View logs with timestamps
+sudo journalctl -u webshop-scraper --no-pager
+
+# Filter by priority
+sudo journalctl -u webshop-scraper -p err  # Only errors
+```
+
+Using npm (development):
+```bash
+npm run dev  # Includes timestamps in output
 ```
 
 ## License

src/api/server.ts

@@ -33,10 +33,36 @@ export function createServer(apiKey: string, jobQueue: JobQueue, db?: ShopDataba
 
   app.use(express.json());
 
-  // Logging middleware
+  // Enhanced HTTP access logging middleware with status codes and timing
   app.use((req, res, next) => {
+    const startTime = Date.now();
     const clientIp = req.ip || req.socket.remoteAddress || 'unknown';
-    logger.info(`${req.method} ${req.path} from ${clientIp}`);
+    const userAgent = req.headers['user-agent'] || '-';
+
+    // Capture the original res.end to log response
+    const originalEnd = res.end.bind(res);
+
+    // Override res.end to log when response is sent
+    res.end = function(chunk?: any, encoding?: any, callback?: any): any {
+      const responseTime = Date.now() - startTime;
+      const statusCode = res.statusCode;
+      const contentLength = res.get('content-length');
+
+      // Use structured HTTP logging
+      logger.http({
+        method: req.method,
+        path: req.path,
+        statusCode,
+        ip: clientIp,
+        responseTime,
+        contentLength,
+        userAgent
+      });
+
+      // Call original end with proper arguments
+      return originalEnd(chunk, encoding, callback);
+    };
+
     next();
   });
 

src/utils/logger.ts

@@ -8,12 +8,15 @@
 export class Logger {
   private prefix: string;
   private isSystemd: boolean;
+  private verboseHttp: boolean;
 
   constructor(prefix: string = 'WebshopScraper') {
     this.prefix = prefix;
     // Detect if running under systemd by checking for INVOCATION_ID
     // This environment variable is set by systemd for each service invocation
     this.isSystemd = !!process.env.INVOCATION_ID;
+    // Enable verbose HTTP logging if LOG_HTTP_VERBOSE is set
+    this.verboseHttp = process.env.LOG_HTTP_VERBOSE === 'true';
   }
 
   private formatMessage(level: string, message: string, ...args: any[]): string {
@@ -44,6 +47,46 @@ export class Logger {
   debug(message: string, ...args: any[]): void {
     console.log(this.formatMessage('DEBUG', message, ...args));
   }
+
+  /**
+   * Log HTTP access in a structured format
+   * Can be used for metrics and monitoring
+   */
+  http(data: {
+    method: string;
+    path: string;
+    statusCode: number;
+    ip: string;
+    responseTime: number;
+    contentLength?: string | number;
+    userAgent?: string;
+  }): void {
+    const { method, path, statusCode, ip, responseTime, contentLength, userAgent } = data;
+
+    // Color code based on status
+    const statusEmoji =
+      statusCode >= 500 ? '❌' :
+      statusCode >= 400 ? '⚠️' :
+      statusCode >= 300 ? '↪️' :
+      statusCode >= 200 ? '✓' : '→';
+
+    // Basic format (always logged)
+    const basicLog = `${statusEmoji} ${statusCode} ${method} ${path} ${ip} ${responseTime}ms`;
+
+    // Verbose format (includes user-agent and content-length)
+    const verboseLog = this.verboseHttp
+      ? ` ${contentLength || '-'}b UA:"${userAgent || '-'}"`
+      : '';
+
+    this.info(`${basicLog}${verboseLog}`);
+  }
+
+  /**
+   * Check if verbose HTTP logging is enabled
+   */
+  isVerboseHttp(): boolean {
+    return this.verboseHttp;
+  }
 }
 
 export const logger = new Logger();