diff --git a/svc/queries/product_performance.go b/svc/queries/product_performance.go index 918d5b0..bba4215 100644 --- a/svc/queries/product_performance.go +++ b/svc/queries/product_performance.go @@ -883,6 +883,11 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( total := 0 save := func(reportKey string, rows any, started time.Time) error { + rowCount := productPerformanceSnapshotPayloadLen(rows) + log.Printf("[ProductPerformanceRefresh] snapshot save start key=%s rows=%d", reportKey, rowCount) + defer func() { + log.Printf("[ProductPerformanceRefresh] snapshot save done key=%s rows=%d elapsed=%s", reportKey, rowCount, time.Since(started).Round(time.Second)) + }() switch v := rows.(type) { case []models.ProductPerformanceSummary: total += len(v) @@ -923,24 +928,28 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( } started := time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("summary")) if summary, err := GetProductPerformanceSummary(ctx, pg); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("summary"), []models.ProductPerformanceSummary{summary}, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("products")) if rows, _, err := ListProductPerformance(ctx, pg, ProductPerformanceFilters{Limit: 50000, Page: 1, SortBy: "performance_score", Descending: true}); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("products"), rows, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("general")) if rows, err := ListProductPerformanceGeneral(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("general"), rows, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("orders")) if rows, err := ListProductPerformanceOrderAnalysis(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("orders"), rows, started); err != nil { @@ -948,6 +957,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( } for _, mode := range []string{"market", "customer"} { started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-groups", mode)) rows, err := ListProductPerformanceOrderGroups(ctx, pg, mode, 50000) if err != nil { return total, err @@ -957,24 +967,28 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( } } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-product-customers")) if rows, err := ListProductPerformanceOrderProductCustomers(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("order-product-customers"), rows, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-market-details")) if rows, err := ListProductPerformanceOrderMarketDetails(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("order-market-details"), rows, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("markets")) if rows, err := ListProductPerformanceMarkets(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("markets"), rows, started); err != nil { return total, err } started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("countries")) if rows, err := ListProductPerformanceCountries(ctx, pg, 50000); err != nil { return total, err } else if err := save(productPerformanceSnapshotKey("countries"), rows, started); err != nil { @@ -982,6 +996,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( } for _, mode := range []string{"market_customer", "country_customer", "market_country_customer"} { started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("customers", mode)) rows, err := ListProductPerformanceCustomers(ctx, pg, mode, 50000) if err != nil { return total, err @@ -992,6 +1007,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( } for _, mode := range []string{"color_yaka_market_customer", "product_country_segment_market_customer", "market_customer_product", "country_segment_market_customer_product"} { started = time.Now() + log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("sales-breakdown", mode)) rows, err := ListProductPerformanceSalesBreakdown(ctx, pg, mode, 50000) if err != nil { return total, err @@ -1000,14 +1016,45 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) ( return total, err } } + log.Printf("[ProductPerformanceRefresh] grouped snapshots rebuild start") groupedRows, err := RebuildProductPerformanceGroupedSnapshots(snapshotReadCtx, pg) if err != nil { return total, err } + log.Printf("[ProductPerformanceRefresh] grouped snapshots rebuild done rows=%d", groupedRows) total += groupedRows return total, nil } +func productPerformanceSnapshotPayloadLen(rows any) int { + switch v := rows.(type) { + case []models.ProductPerformanceSummary: + return len(v) + case []models.ProductPerformanceRow: + return len(v) + case []models.ProductPerformanceGeneralRow: + return len(v) + case []models.ProductPerformanceOrderAnalysisRow: + return len(v) + case []models.ProductPerformanceOrderGroupRow: + return len(v) + case []models.ProductPerformanceOrderProductCustomerRow: + return len(v) + case []models.ProductPerformanceOrderMarketDetailRow: + return len(v) + case []models.ProductPerformanceMarketRow: + return len(v) + case []models.ProductPerformanceCountryRow: + return len(v) + case []models.ProductPerformanceCustomerRow: + return len(v) + case []models.ProductPerformanceSalesBreakdownRow: + return len(v) + default: + return 0 + } +} + type productPerformanceGroupedSnapshotDefinition struct { Mode string Levels []string @@ -1022,25 +1069,29 @@ func RebuildProductPerformanceGroupedSnapshots(ctx context.Context, pg *sql.DB) if err != nil { return 0, err } + log.Printf("[ProductPerformanceRefresh] grouped snapshot definitions ready count=%d", len(defs)) total := 0 - for _, def := range defs { + for i, def := range defs { started := time.Now() levels := sanitizeProductPerformanceGroupLevels(def.Levels) if len(levels) == 0 { levels = defaultProductPerformanceGroupLevels(def.Mode) } + reportKey := productPerformanceGroupedSnapshotReportKey(def.Mode, levels, def.MainGroup) + log.Printf("[ProductPerformanceRefresh] grouped snapshot start %d/%d key=%s mode=%s main_group=%s levels=%s", i+1, len(defs), reportKey, def.Mode, def.MainGroup, productPerformanceGroupedLevelsKey(levels)) sourceRows, err := productPerformanceGroupedRawSnapshotSourceRows(ctx, pg, def.Mode, 50000) if err != nil { return total, err } + log.Printf("[ProductPerformanceRefresh] grouped snapshot source loaded key=%s source_rows=%d elapsed=%s", reportKey, len(sourceRows), time.Since(started).Round(time.Second)) req := ProductPerformanceGroupedRequest{ Mode: def.Mode, MainGroup: def.MainGroup, } sourceRows = filterProductPerformanceGroupedRows(sourceRows, productPerformanceGroupedEffectiveFilters(req)) - rows := make([]map[string]any, 0, len(sourceRows)) - appendProductPerformanceGroupedRows(&rows, sourceRows, levels, 0, 0, []string{"tab:" + def.Mode}, nil, len(levels)-1) - reportKey := productPerformanceGroupedSnapshotReportKey(def.Mode, levels, def.MainGroup) + groupStarted := time.Now() + rows := buildProductPerformanceGroupedSnapshotRows(sourceRows, levels, def.Mode) + log.Printf("[ProductPerformanceRefresh] grouped snapshot tree built key=%s rows=%d elapsed=%s total_elapsed=%s", reportKey, len(rows), time.Since(groupStarted).Round(time.Second), time.Since(started).Round(time.Second)) if err := saveProductPerformanceGroupedSnapshotRows(ctx, pg, reportKey, productPerformanceGroupedSnapshotDefinition{ Mode: def.Mode, Levels: levels, @@ -1049,6 +1100,7 @@ func RebuildProductPerformanceGroupedSnapshots(ctx context.Context, pg *sql.DB) return total, err } total += len(rows) + log.Printf("[ProductPerformanceRefresh] grouped snapshot done key=%s rows=%d cumulative_rows=%d elapsed=%s", reportKey, len(rows), total, time.Since(started).Round(time.Second)) } return total, nil } @@ -3736,9 +3788,6 @@ func ListProductPerformanceGrouped(ctx context.Context, pg *sql.DB, req ProductP if out, ok, err := loadProductPerformancePreparedGroupedRows(ctx, pg, req, levels); ok || err != nil { return out, err } - if !productPerformanceLiveFallbackEnabled() { - return []map[string]any{}, nil - } sourceRows, err := productPerformanceGroupedSourceRows(ctx, pg, req.Mode, req.Limit) if err != nil { @@ -4766,6 +4815,208 @@ func productPerformanceIdleSourceRows(rows []map[string]any) []map[string]any { return out } +type productPerformanceGroupedSnapshotAvgState struct { + Weighted float64 + Weight float64 + Sum float64 + Count float64 +} + +type productPerformanceGroupedSnapshotNode struct { + Key string + Level int + Field string + Value string + Row map[string]any + Count int + Children map[string]*productPerformanceGroupedSnapshotNode + ChildOrder []string + StockSeen map[string]bool + IdleSeen map[string]bool + Avg map[string]*productPerformanceGroupedSnapshotAvgState + BucketCounts map[string]int + Image map[string]any +} + +func buildProductPerformanceGroupedSnapshotRows(sourceRows []map[string]any, levels []string, mode string) []map[string]any { + roots := map[string]*productPerformanceGroupedSnapshotNode{} + rootOrder := make([]string, 0) + for _, row := range sourceRows { + parentKeys := []string{"tab:" + mode} + parentChildren := roots + parentOrder := &rootOrder + visualLevel := 0 + for _, field := range levels { + value := normalizeProductPerformanceGroupValue(mapGroupValue(row, field)) + if shouldSkipProductPerformanceGroupValue(field, value) { + continue + } + keyPart := field + ":" + value + key := strings.Join(append(parentKeys, keyPart), "|") + node := parentChildren[key] + if node == nil { + node = &productPerformanceGroupedSnapshotNode{ + Key: key, + Level: visualLevel, + Field: field, + Value: value, + Row: map[string]any{}, + Children: map[string]*productPerformanceGroupedSnapshotNode{}, + } + parentChildren[key] = node + *parentOrder = append(*parentOrder, key) + } + node.add(row) + parentKeys = append(parentKeys, keyPart) + parentChildren = node.Children + parentOrder = &node.ChildOrder + visualLevel++ + } + } + + out := make([]map[string]any, 0, len(sourceRows)) + appendProductPerformanceGroupedSnapshotNodes(&out, roots, rootOrder) + return out +} + +func (n *productPerformanceGroupedSnapshotNode) add(row map[string]any) { + n.Count++ + if n.Image == nil && stringFromMap(row, "product_code") != "" { + n.Image = row + } + if bucket := stringFromMap(row, "performance_bucket"); bucket != "" { + if n.BucketCounts == nil { + n.BucketCounts = map[string]int{} + } + n.BucketCounts[bucket]++ + } + for key, value := range row { + if key == "row_key" || key == "key" || isProductPerformanceMarginField(key) { + continue + } + switch { + case key == "stock_qty": + variantKey := productPerformanceMapVariantKey(row) + if variantKey == "" { + n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value) + continue + } + if n.StockSeen == nil { + n.StockSeen = map[string]bool{} + } + if !n.StockSeen[variantKey] { + n.StockSeen[variantKey] = true + n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value) + } + case key == "idle_cost_usd": + variantKey := productPerformanceMapVariantKey(row) + if variantKey == "" { + n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value) + continue + } + if n.IdleSeen == nil { + n.IdleSeen = map[string]bool{} + } + if !n.IdleSeen[variantKey] { + n.IdleSeen[variantKey] = true + n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value) + } + case shouldSumProductPerformanceField(key): + n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value) + case shouldAverageProductPerformanceField(key): + if n.Avg == nil { + n.Avg = map[string]*productPerformanceGroupedSnapshotAvgState{} + } + state := n.Avg[key] + if state == nil { + state = &productPerformanceGroupedSnapshotAvgState{} + n.Avg[key] = state + } + number := floatFromAny(value) + weight := floatFromMap(row, productPerformanceMetricWeightField(key)) + if weight > 0 { + state.Weighted += number * weight + state.Weight += weight + } + state.Sum += number + state.Count++ + default: + if _, ok := n.Row[key]; !ok { + n.Row[key] = value + } + } + } +} + +func appendProductPerformanceGroupedSnapshotNodes(out *[]map[string]any, nodes map[string]*productPerformanceGroupedSnapshotNode, order []string) { + sort.SliceStable(order, func(i, j int) bool { + left := nodes[order[i]] + right := nodes[order[j]] + if left == nil || right == nil { + return order[i] < order[j] + } + return strings.Compare(left.Value, right.Value) < 0 + }) + for _, key := range order { + node := nodes[key] + if node == nil { + continue + } + *out = append(*out, node.snapshotRow()) + if len(node.Children) > 0 { + appendProductPerformanceGroupedSnapshotNodes(out, node.Children, node.ChildOrder) + } + } +} + +func (n *productPerformanceGroupedSnapshotNode) snapshotRow() map[string]any { + row := cloneMap(n.Row) + for key, state := range n.Avg { + if state == nil { + continue + } + if state.Weight > 0 { + row[key] = state.Weighted / state.Weight + } else if state.Count > 0 { + row[key] = state.Sum / state.Count + } + } + deriveProductPerformanceGroupMetrics(row, n.Field) + if bucket := n.dominantBucket(); bucket != "" { + row["performance_bucket"] = bucket + } + clearProductPerformanceGroupDimensions(row, n.Field, n.Value) + row["__group"] = true + row["row_key"] = "group|" + n.Key + row["key"] = n.Key + row["level"] = n.Level + row["group_field"] = n.Field + row["group_value"] = n.Value + row["label"] = n.Value + row["count"] = n.Count + row["recommendation"] = fmt.Sprintf("%d satir", n.Count) + image := n.Image + if image == nil { + image = row + } + row["image_product_code"] = stringFromMap(image, "product_code") + row["image_color_code"] = stringFromMap(image, "color_code") + row["image_yaka_kodu"] = stringFromMap(image, "yaka_kodu") + return row +} + +func (n *productPerformanceGroupedSnapshotNode) dominantBucket() string { + best := "" + bestCount := 0 + for bucket, count := range n.BucketCounts { + if count > bestCount || (count == bestCount && bucket < best) { + best = bucket + bestCount = count + } + } + return best +} + func appendProductPerformanceGroupedRows(out *[]map[string]any, sourceRows []map[string]any, levels []string, level int, visualLevel int, parentKeys []string, expandedKeys map[string]bool, expandThroughLevel int) { if level >= len(levels) { return