feat: implement versioning with git hash and update crawler user agent

Implement proper version tracking using build time and git commit hash.
Update crawler user agent to identify as ShopCallCrawler with version.

Changes:
- Add scripts/generate-version.js to generate version info at build time
- Create src/version.ts (auto-generated) with VERSION_INFO and helper functions
- Update package.json to run version generation before build (prebuild hook)
- Update ContentExtractor.ts to use getUserAgent() function
- Update SitemapParser.ts to use getUserAgent() function
- Add src/version.ts to .gitignore (auto-generated file)

Version format: {package_version}+{git_hash}
Example: 1.0.0+602aeb5

User agent format: ShopCallCrawler/{version} (+https://shopcall.ai/crawler)
Example: ShopCallCrawler/1.0.0+602aeb5 (+https://shopcall.ai/crawler)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

.gitignore

@@ -32,6 +32,9 @@ node_modules
 dist/
 *.tsbuildinfo
 
+# Auto-generated version file
+src/version.ts
+
 # Environment variables
 .env
 

package.json

@@ -4,6 +4,8 @@
   "description": "A web scraper for extracting information from webshops (ShopRenter, WooCommerce, Shopify)",
   "main": "dist/index.js",
   "scripts": {
+    "generate:version": "node scripts/generate-version.js",
+    "prebuild": "npm run generate:version",
     "build": "tsc && npm run build:web",
     "build:web": "cd web && npm install && npm run build",
     "start": "node dist/index.js",

scripts/generate-version.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+
+/**
+ * Generate version.ts file with build information
+ * This script creates a TypeScript file containing version info from git and build time
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+function getGitHash() {
+  try {
+    return execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
+  } catch (error) {
+    console.warn('Warning: Could not get git hash, using "unknown"');
+    return 'unknown';
+  }
+}
+
+function getGitBranch() {
+  try {
+    return execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim();
+  } catch (error) {
+    return 'unknown';
+  }
+}
+
+function getBuildTime() {
+  return new Date().toISOString();
+}
+
+function getPackageVersion() {
+  try {
+    const packageJson = JSON.parse(
+      fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8')
+    );
+    return packageJson.version;
+  } catch (error) {
+    return '1.0.0';
+  }
+}
+
+const gitHash = getGitHash();
+const gitBranch = getGitBranch();
+const buildTime = getBuildTime();
+const packageVersion = getPackageVersion();
+
+const versionContent = `/**
+ * Auto-generated version information
+ * DO NOT EDIT THIS FILE MANUALLY
+ * Generated at build time by scripts/generate-version.js
+ */
+
+export const VERSION_INFO = {
+  version: '${packageVersion}',
+  gitHash: '${gitHash}',
+  gitBranch: '${gitBranch}',
+  buildTime: '${buildTime}',
+  fullVersion: '${packageVersion}+${gitHash}',
+} as const;
+
+export function getVersion(): string {
+  return VERSION_INFO.fullVersion;
+}
+
+export function getUserAgent(): string {
+  return \`ShopCallCrawler/\${VERSION_INFO.fullVersion} (+https://shopcall.ai/crawler)\`;
+}
+`;
+
+const outputPath = path.join(__dirname, '../src/version.ts');
+
+try {
+  fs.writeFileSync(outputPath, versionContent, 'utf-8');
+  console.log(`✓ Generated version file: ${outputPath}`);
+  console.log(`  Version: ${packageVersion}+${gitHash}`);
+  console.log(`  Branch: ${gitBranch}`);
+  console.log(`  Build time: ${buildTime}`);
+} catch (error) {
+  console.error('Error writing version file:', error);
+  process.exit(1);
+}

src/scraper/ContentExtractor.ts

@@ -2,6 +2,7 @@ import axios from 'axios';
 import * as cheerio from 'cheerio';
 import TurndownService from 'turndown';
 import { logger } from '../utils/logger';
+import { getUserAgent } from '../version';
 
 export class ContentExtractor {
   private turndownService: TurndownService;
@@ -23,7 +24,7 @@ export class ContentExtractor {
       const response = await axios.get(url, {
         timeout: 30000,
         headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; WebshopScraper/1.0)'
+          'User-Agent': getUserAgent()
         }
       });
 

src/scraper/SitemapParser.ts

@@ -3,6 +3,7 @@ import { parseString } from 'xml2js';
 import { promisify } from 'util';
 import { SitemapUrl, WebshopType } from '../types';
 import { logger } from '../utils/logger';
+import { getUserAgent } from '../version';
 
 const parseXml = promisify(parseString);
 
@@ -47,7 +48,7 @@ export class SitemapParser {
       const response = await axios.get(sitemapUrl, {
         timeout: 30000,
         headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; WebshopScraper/1.0)'
+          'User-Agent': getUserAgent()
         }
       });