|
|
@@ -0,0 +1,327 @@
|
|
|
+import { logger } from '../utils/logger';
|
|
|
+import { QdrantService } from './QdrantService';
|
|
|
+import { ShopDatabase } from '../database/Database';
|
|
|
+import type { AppConfig } from '../config';
|
|
|
+
|
|
|
+export interface MigrationResult {
|
|
|
+ success: boolean;
|
|
|
+ collectionsProcessed: number;
|
|
|
+ pointsMigrated: number;
|
|
|
+ errors: string[];
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Service to migrate from old collection structure (one per URL) to new structure (one per category)
|
|
|
+ */
|
|
|
+export class QdrantMigrationService {
|
|
|
+ private qdrantService: QdrantService;
|
|
|
+
|
|
|
+ constructor(
|
|
|
+ private config: AppConfig,
|
|
|
+ private database: ShopDatabase
|
|
|
+ ) {
|
|
|
+ this.qdrantService = new QdrantService(config);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Migrate a shop's collections from old structure to new structure
|
|
|
+ * Old: {shop_custom_id}-{category}_{url_hash} (many collections per category)
|
|
|
+ * New: {shop_custom_id}-{category} (one collection per category)
|
|
|
+ */
|
|
|
+ async migrateShop(shopCustomId: string): Promise<MigrationResult> {
|
|
|
+ const result: MigrationResult = {
|
|
|
+ success: true,
|
|
|
+ collectionsProcessed: 0,
|
|
|
+ pointsMigrated: 0,
|
|
|
+ errors: []
|
|
|
+ };
|
|
|
+
|
|
|
+ try {
|
|
|
+ logger.info(`Starting migration for shop: ${shopCustomId}`);
|
|
|
+
|
|
|
+ // Get all collections for this shop
|
|
|
+ const collections = await this.qdrantService.getCollectionsForShop(shopCustomId);
|
|
|
+
|
|
|
+ if (collections.length === 0) {
|
|
|
+ logger.info(`No collections found for shop ${shopCustomId}`);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Group collections by category
|
|
|
+ const collectionsByCategory = new Map<string, string[]>();
|
|
|
+
|
|
|
+ for (const collectionName of collections) {
|
|
|
+ const parsed = this.qdrantService.parseCollectionName(collectionName);
|
|
|
+ if (!parsed) {
|
|
|
+ result.errors.push(`Failed to parse collection name: ${collectionName}`);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Skip if already in new format (no hash or iterator)
|
|
|
+ if (!parsed.urlHash && parsed.iterator === undefined) {
|
|
|
+ logger.debug(`Collection ${collectionName} already in new format, skipping`);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Group by category
|
|
|
+ if (!collectionsByCategory.has(parsed.category)) {
|
|
|
+ collectionsByCategory.set(parsed.category, []);
|
|
|
+ }
|
|
|
+ collectionsByCategory.get(parsed.category)!.push(collectionName);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Process each category
|
|
|
+ for (const [category, oldCollections] of collectionsByCategory) {
|
|
|
+ try {
|
|
|
+ const migratedPoints = await this.migrateCategory(
|
|
|
+ shopCustomId,
|
|
|
+ category,
|
|
|
+ oldCollections
|
|
|
+ );
|
|
|
+
|
|
|
+ result.collectionsProcessed += oldCollections.length;
|
|
|
+ result.pointsMigrated += migratedPoints;
|
|
|
+
|
|
|
+ logger.info(`Migrated category ${category}: ${migratedPoints} points from ${oldCollections.length} collections`);
|
|
|
+ } catch (error: any) {
|
|
|
+ const errorMsg = `Failed to migrate category ${category}: ${error.message}`;
|
|
|
+ logger.error(errorMsg, error);
|
|
|
+ result.errors.push(errorMsg);
|
|
|
+ result.success = false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ logger.info(`Migration completed for shop ${shopCustomId}: ${result.pointsMigrated} points from ${result.collectionsProcessed} collections`);
|
|
|
+
|
|
|
+ } catch (error: any) {
|
|
|
+ logger.error(`Migration failed for shop ${shopCustomId}:`, error);
|
|
|
+ result.success = false;
|
|
|
+ result.errors.push(error.message || 'Unknown error');
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Migrate a single category's collections to the new format
|
|
|
+ */
|
|
|
+ private async migrateCategory(
|
|
|
+ shopCustomId: string,
|
|
|
+ category: string,
|
|
|
+ oldCollections: string[]
|
|
|
+ ): Promise<number> {
|
|
|
+ let pointsMigrated = 0;
|
|
|
+
|
|
|
+ // Generate new collection name
|
|
|
+ const newCollectionName = this.qdrantService.generateCollectionName(shopCustomId, category);
|
|
|
+
|
|
|
+ // Create new collection if it doesn't exist
|
|
|
+ if (!(await this.qdrantService.collectionExists(newCollectionName))) {
|
|
|
+ await this.qdrantService.createCollection(newCollectionName);
|
|
|
+ logger.info(`Created new collection: ${newCollectionName}`);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Copy points from old collections to new collection
|
|
|
+ for (const oldCollectionName of oldCollections) {
|
|
|
+ try {
|
|
|
+ const points = await this.getAllPointsFromCollection(oldCollectionName);
|
|
|
+
|
|
|
+ if (points.length === 0) {
|
|
|
+ logger.debug(`No points found in collection ${oldCollectionName}`);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Upsert points to new collection
|
|
|
+ for (const point of points) {
|
|
|
+ try {
|
|
|
+ await this.qdrantService.upsertPoint(newCollectionName, point);
|
|
|
+ pointsMigrated++;
|
|
|
+ } catch (error: any) {
|
|
|
+ logger.warn(`Failed to migrate point ${point.id} from ${oldCollectionName}:`, error);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ logger.info(`Migrated ${points.length} points from ${oldCollectionName} to ${newCollectionName}`);
|
|
|
+
|
|
|
+ } catch (error: any) {
|
|
|
+ logger.error(`Failed to process collection ${oldCollectionName}:`, error);
|
|
|
+ throw error;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return pointsMigrated;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Get all points from a collection
|
|
|
+ * Uses scroll API to retrieve all points in batches
|
|
|
+ */
|
|
|
+ private async getAllPointsFromCollection(collectionName: string): Promise<any[]> {
|
|
|
+ const points: any[] = [];
|
|
|
+ const batchSize = 100;
|
|
|
+ let offset: string | number | undefined = undefined;
|
|
|
+
|
|
|
+ try {
|
|
|
+ // Use Qdrant's scroll API to get all points
|
|
|
+ while (true) {
|
|
|
+ const response: any = await (this.qdrantService as any).client.scroll(collectionName, {
|
|
|
+ limit: batchSize,
|
|
|
+ offset,
|
|
|
+ with_payload: true,
|
|
|
+ with_vector: true
|
|
|
+ });
|
|
|
+
|
|
|
+ if (!response.points || response.points.length === 0) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const point of response.points) {
|
|
|
+ points.push({
|
|
|
+ id: String(point.id),
|
|
|
+ vector: point.vector,
|
|
|
+ payload: point.payload
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ // Check if there are more points
|
|
|
+ if (!response.next_page_offset) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ offset = response.next_page_offset;
|
|
|
+ }
|
|
|
+
|
|
|
+ return points;
|
|
|
+
|
|
|
+ } catch (error: any) {
|
|
|
+ if (error.status === 404) {
|
|
|
+ logger.debug(`Collection ${collectionName} not found`);
|
|
|
+ return [];
|
|
|
+ }
|
|
|
+ throw error;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Delete old collections after successful migration
|
|
|
+ * WARNING: This will permanently delete the old collections!
|
|
|
+ */
|
|
|
+ async cleanupOldCollections(shopCustomId: string, dryRun: boolean = true): Promise<{
|
|
|
+ deleted: string[];
|
|
|
+ kept: string[];
|
|
|
+ errors: string[];
|
|
|
+ }> {
|
|
|
+ const result = {
|
|
|
+ deleted: [] as string[],
|
|
|
+ kept: [] as string[],
|
|
|
+ errors: [] as string[]
|
|
|
+ };
|
|
|
+
|
|
|
+ try {
|
|
|
+ const collections = await this.qdrantService.getCollectionsForShop(shopCustomId);
|
|
|
+
|
|
|
+ for (const collectionName of collections) {
|
|
|
+ const parsed = this.qdrantService.parseCollectionName(collectionName);
|
|
|
+
|
|
|
+ if (!parsed) {
|
|
|
+ result.errors.push(`Failed to parse collection name: ${collectionName}`);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Only delete collections in old format (with hash or iterator)
|
|
|
+ if (parsed.urlHash || parsed.iterator !== undefined) {
|
|
|
+ if (dryRun) {
|
|
|
+ logger.info(`[DRY RUN] Would delete collection: ${collectionName}`);
|
|
|
+ result.deleted.push(collectionName);
|
|
|
+ } else {
|
|
|
+ try {
|
|
|
+ await this.qdrantService.deleteCollection(collectionName);
|
|
|
+ logger.info(`Deleted old collection: ${collectionName}`);
|
|
|
+ result.deleted.push(collectionName);
|
|
|
+ } catch (error: any) {
|
|
|
+ const errorMsg = `Failed to delete collection ${collectionName}: ${error.message}`;
|
|
|
+ logger.error(errorMsg);
|
|
|
+ result.errors.push(errorMsg);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ result.kept.push(collectionName);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ } catch (error: any) {
|
|
|
+ logger.error(`Failed to cleanup old collections for shop ${shopCustomId}:`, error);
|
|
|
+ result.errors.push(error.message || 'Unknown error');
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Get migration status for a shop
|
|
|
+ */
|
|
|
+ async getMigrationStatus(shopCustomId: string): Promise<{
|
|
|
+ needsMigration: boolean;
|
|
|
+ oldCollections: number;
|
|
|
+ newCollections: number;
|
|
|
+ categories: {
|
|
|
+ category: string;
|
|
|
+ oldCollections: number;
|
|
|
+ newCollectionExists: boolean;
|
|
|
+ }[];
|
|
|
+ }> {
|
|
|
+ const collections = await this.qdrantService.getCollectionsForShop(shopCustomId);
|
|
|
+
|
|
|
+ const categoriesMap = new Map<string, {
|
|
|
+ oldCollections: number;
|
|
|
+ newCollectionExists: boolean;
|
|
|
+ }>();
|
|
|
+
|
|
|
+ let oldCollectionCount = 0;
|
|
|
+ let newCollectionCount = 0;
|
|
|
+
|
|
|
+ for (const collectionName of collections) {
|
|
|
+ const parsed = this.qdrantService.parseCollectionName(collectionName);
|
|
|
+ if (!parsed) continue;
|
|
|
+
|
|
|
+ const isOldFormat = parsed.urlHash !== undefined || parsed.iterator !== undefined;
|
|
|
+
|
|
|
+ if (isOldFormat) {
|
|
|
+ oldCollectionCount++;
|
|
|
+
|
|
|
+ if (!categoriesMap.has(parsed.category)) {
|
|
|
+ categoriesMap.set(parsed.category, {
|
|
|
+ oldCollections: 0,
|
|
|
+ newCollectionExists: false
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ const categoryData = categoriesMap.get(parsed.category)!;
|
|
|
+ categoryData.oldCollections++;
|
|
|
+
|
|
|
+ } else {
|
|
|
+ newCollectionCount++;
|
|
|
+
|
|
|
+ if (!categoriesMap.has(parsed.category)) {
|
|
|
+ categoriesMap.set(parsed.category, {
|
|
|
+ oldCollections: 0,
|
|
|
+ newCollectionExists: true
|
|
|
+ });
|
|
|
+ } else {
|
|
|
+ categoriesMap.get(parsed.category)!.newCollectionExists = true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return {
|
|
|
+ needsMigration: oldCollectionCount > 0,
|
|
|
+ oldCollections: oldCollectionCount,
|
|
|
+ newCollections: newCollectionCount,
|
|
|
+ categories: Array.from(categoriesMap.entries()).map(([category, data]) => ({
|
|
|
+ category,
|
|
|
+ oldCollections: data.oldCollections,
|
|
|
+ newCollectionExists: data.newCollectionExists
|
|
|
+ }))
|
|
|
+ };
|
|
|
+ }
|
|
|
+}
|
Fszontagh