Fix product performance grouped JSON build

svc/queries/product_performance.go

@@ -883,6 +883,11 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 	total := 0
 
 	save := func(reportKey string, rows any, started time.Time) error {
+		rowCount := productPerformanceSnapshotPayloadLen(rows)
+		log.Printf("[ProductPerformanceRefresh] snapshot save start key=%s rows=%d", reportKey, rowCount)
+		defer func() {
+			log.Printf("[ProductPerformanceRefresh] snapshot save done key=%s rows=%d elapsed=%s", reportKey, rowCount, time.Since(started).Round(time.Second))
+		}()
 		switch v := rows.(type) {
 		case []models.ProductPerformanceSummary:
 			total += len(v)
@@ -923,24 +928,28 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 	}
 
 	started := time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("summary"))
 	if summary, err := GetProductPerformanceSummary(ctx, pg); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("summary"), []models.ProductPerformanceSummary{summary}, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("products"))
 	if rows, _, err := ListProductPerformance(ctx, pg, ProductPerformanceFilters{Limit: 50000, Page: 1, SortBy: "performance_score", Descending: true}); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("products"), rows, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("general"))
 	if rows, err := ListProductPerformanceGeneral(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("general"), rows, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("orders"))
 	if rows, err := ListProductPerformanceOrderAnalysis(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("orders"), rows, started); err != nil {
@@ -948,6 +957,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 	}
 	for _, mode := range []string{"market", "customer"} {
 		started = time.Now()
+		log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-groups", mode))
 		rows, err := ListProductPerformanceOrderGroups(ctx, pg, mode, 50000)
 		if err != nil {
 			return total, err
@@ -957,24 +967,28 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 		}
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-product-customers"))
 	if rows, err := ListProductPerformanceOrderProductCustomers(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("order-product-customers"), rows, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("order-market-details"))
 	if rows, err := ListProductPerformanceOrderMarketDetails(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("order-market-details"), rows, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("markets"))
 	if rows, err := ListProductPerformanceMarkets(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("markets"), rows, started); err != nil {
 		return total, err
 	}
 	started = time.Now()
+	log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("countries"))
 	if rows, err := ListProductPerformanceCountries(ctx, pg, 50000); err != nil {
 		return total, err
 	} else if err := save(productPerformanceSnapshotKey("countries"), rows, started); err != nil {
@@ -982,6 +996,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 	}
 	for _, mode := range []string{"market_customer", "country_customer", "market_country_customer"} {
 		started = time.Now()
+		log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("customers", mode))
 		rows, err := ListProductPerformanceCustomers(ctx, pg, mode, 50000)
 		if err != nil {
 			return total, err
@@ -992,6 +1007,7 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 	}
 	for _, mode := range []string{"color_yaka_market_customer", "product_country_segment_market_customer", "market_customer_product", "country_segment_market_customer_product"} {
 		started = time.Now()
+		log.Printf("[ProductPerformanceRefresh] snapshot build start key=%s", productPerformanceSnapshotKey("sales-breakdown", mode))
 		rows, err := ListProductPerformanceSalesBreakdown(ctx, pg, mode, 50000)
 		if err != nil {
 			return total, err
@@ -1000,14 +1016,45 @@ func RebuildProductPerformanceReportSnapshots(ctx context.Context, pg *sql.DB) (
 			return total, err
 		}
 	}
+	log.Printf("[ProductPerformanceRefresh] grouped snapshots rebuild start")
 	groupedRows, err := RebuildProductPerformanceGroupedSnapshots(snapshotReadCtx, pg)
 	if err != nil {
 		return total, err
 	}
+	log.Printf("[ProductPerformanceRefresh] grouped snapshots rebuild done rows=%d", groupedRows)
 	total += groupedRows
 	return total, nil
 }
 
+func productPerformanceSnapshotPayloadLen(rows any) int {
+	switch v := rows.(type) {
+	case []models.ProductPerformanceSummary:
+		return len(v)
+	case []models.ProductPerformanceRow:
+		return len(v)
+	case []models.ProductPerformanceGeneralRow:
+		return len(v)
+	case []models.ProductPerformanceOrderAnalysisRow:
+		return len(v)
+	case []models.ProductPerformanceOrderGroupRow:
+		return len(v)
+	case []models.ProductPerformanceOrderProductCustomerRow:
+		return len(v)
+	case []models.ProductPerformanceOrderMarketDetailRow:
+		return len(v)
+	case []models.ProductPerformanceMarketRow:
+		return len(v)
+	case []models.ProductPerformanceCountryRow:
+		return len(v)
+	case []models.ProductPerformanceCustomerRow:
+		return len(v)
+	case []models.ProductPerformanceSalesBreakdownRow:
+		return len(v)
+	default:
+		return 0
+	}
+}
+
 type productPerformanceGroupedSnapshotDefinition struct {
 	Mode      string
 	Levels    []string
@@ -1022,25 +1069,29 @@ func RebuildProductPerformanceGroupedSnapshots(ctx context.Context, pg *sql.DB)
 	if err != nil {
 		return 0, err
 	}
+	log.Printf("[ProductPerformanceRefresh] grouped snapshot definitions ready count=%d", len(defs))
 	total := 0
-	for _, def := range defs {
+	for i, def := range defs {
 		started := time.Now()
 		levels := sanitizeProductPerformanceGroupLevels(def.Levels)
 		if len(levels) == 0 {
 			levels = defaultProductPerformanceGroupLevels(def.Mode)
 		}
+		reportKey := productPerformanceGroupedSnapshotReportKey(def.Mode, levels, def.MainGroup)
+		log.Printf("[ProductPerformanceRefresh] grouped snapshot start %d/%d key=%s mode=%s main_group=%s levels=%s", i+1, len(defs), reportKey, def.Mode, def.MainGroup, productPerformanceGroupedLevelsKey(levels))
 		sourceRows, err := productPerformanceGroupedRawSnapshotSourceRows(ctx, pg, def.Mode, 50000)
 		if err != nil {
 			return total, err
 		}
+		log.Printf("[ProductPerformanceRefresh] grouped snapshot source loaded key=%s source_rows=%d elapsed=%s", reportKey, len(sourceRows), time.Since(started).Round(time.Second))
 		req := ProductPerformanceGroupedRequest{
 			Mode:      def.Mode,
 			MainGroup: def.MainGroup,
 		}
 		sourceRows = filterProductPerformanceGroupedRows(sourceRows, productPerformanceGroupedEffectiveFilters(req))
-		rows := make([]map[string]any, 0, len(sourceRows))
+		groupStarted := time.Now()
-		appendProductPerformanceGroupedRows(&rows, sourceRows, levels, 0, 0, []string{"tab:" + def.Mode}, nil, len(levels)-1)
+		rows := buildProductPerformanceGroupedSnapshotRows(sourceRows, levels, def.Mode)
-		reportKey := productPerformanceGroupedSnapshotReportKey(def.Mode, levels, def.MainGroup)
+		log.Printf("[ProductPerformanceRefresh] grouped snapshot tree built key=%s rows=%d elapsed=%s total_elapsed=%s", reportKey, len(rows), time.Since(groupStarted).Round(time.Second), time.Since(started).Round(time.Second))
 		if err := saveProductPerformanceGroupedSnapshotRows(ctx, pg, reportKey, productPerformanceGroupedSnapshotDefinition{
 			Mode:      def.Mode,
 			Levels:    levels,
@@ -1049,6 +1100,7 @@ func RebuildProductPerformanceGroupedSnapshots(ctx context.Context, pg *sql.DB)
 			return total, err
 		}
 		total += len(rows)
+		log.Printf("[ProductPerformanceRefresh] grouped snapshot done key=%s rows=%d cumulative_rows=%d elapsed=%s", reportKey, len(rows), total, time.Since(started).Round(time.Second))
 	}
 	return total, nil
 }
@@ -3736,9 +3788,6 @@ func ListProductPerformanceGrouped(ctx context.Context, pg *sql.DB, req ProductP
 	if out, ok, err := loadProductPerformancePreparedGroupedRows(ctx, pg, req, levels); ok || err != nil {
 		return out, err
 	}
-	if !productPerformanceLiveFallbackEnabled() {
-		return []map[string]any{}, nil
-	}
 
 	sourceRows, err := productPerformanceGroupedSourceRows(ctx, pg, req.Mode, req.Limit)
 	if err != nil {
@@ -4766,6 +4815,208 @@ func productPerformanceIdleSourceRows(rows []map[string]any) []map[string]any {
 	return out
 }
 
+type productPerformanceGroupedSnapshotAvgState struct {
+	Weighted float64
+	Weight   float64
+	Sum      float64
+	Count    float64
+}
+
+type productPerformanceGroupedSnapshotNode struct {
+	Key          string
+	Level        int
+	Field        string
+	Value        string
+	Row          map[string]any
+	Count        int
+	Children     map[string]*productPerformanceGroupedSnapshotNode
+	ChildOrder   []string
+	StockSeen    map[string]bool
+	IdleSeen     map[string]bool
+	Avg          map[string]*productPerformanceGroupedSnapshotAvgState
+	BucketCounts map[string]int
+	Image        map[string]any
+}
+
+func buildProductPerformanceGroupedSnapshotRows(sourceRows []map[string]any, levels []string, mode string) []map[string]any {
+	roots := map[string]*productPerformanceGroupedSnapshotNode{}
+	rootOrder := make([]string, 0)
+	for _, row := range sourceRows {
+		parentKeys := []string{"tab:" + mode}
+		parentChildren := roots
+		parentOrder := &rootOrder
+		visualLevel := 0
+		for _, field := range levels {
+			value := normalizeProductPerformanceGroupValue(mapGroupValue(row, field))
+			if shouldSkipProductPerformanceGroupValue(field, value) {
+				continue
+			}
+			keyPart := field + ":" + value
+			key := strings.Join(append(parentKeys, keyPart), "|")
+			node := parentChildren[key]
+			if node == nil {
+				node = &productPerformanceGroupedSnapshotNode{
+					Key:      key,
+					Level:    visualLevel,
+					Field:    field,
+					Value:    value,
+					Row:      map[string]any{},
+					Children: map[string]*productPerformanceGroupedSnapshotNode{},
+				}
+				parentChildren[key] = node
+				*parentOrder = append(*parentOrder, key)
+			}
+			node.add(row)
+			parentKeys = append(parentKeys, keyPart)
+			parentChildren = node.Children
+			parentOrder = &node.ChildOrder
+			visualLevel++
+		}
+	}
+
+	out := make([]map[string]any, 0, len(sourceRows))
+	appendProductPerformanceGroupedSnapshotNodes(&out, roots, rootOrder)
+	return out
+}
+
+func (n *productPerformanceGroupedSnapshotNode) add(row map[string]any) {
+	n.Count++
+	if n.Image == nil && stringFromMap(row, "product_code") != "" {
+		n.Image = row
+	}
+	if bucket := stringFromMap(row, "performance_bucket"); bucket != "" {
+		if n.BucketCounts == nil {
+			n.BucketCounts = map[string]int{}
+		}
+		n.BucketCounts[bucket]++
+	}
+	for key, value := range row {
+		if key == "row_key" || key == "key" || isProductPerformanceMarginField(key) {
+			continue
+		}
+		switch {
+		case key == "stock_qty":
+			variantKey := productPerformanceMapVariantKey(row)
+			if variantKey == "" {
+				n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value)
+				continue
+			}
+			if n.StockSeen == nil {
+				n.StockSeen = map[string]bool{}
+			}
+			if !n.StockSeen[variantKey] {
+				n.StockSeen[variantKey] = true
+				n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value)
+			}
+		case key == "idle_cost_usd":
+			variantKey := productPerformanceMapVariantKey(row)
+			if variantKey == "" {
+				n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value)
+				continue
+			}
+			if n.IdleSeen == nil {
+				n.IdleSeen = map[string]bool{}
+			}
+			if !n.IdleSeen[variantKey] {
+				n.IdleSeen[variantKey] = true
+				n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value)
+			}
+		case shouldSumProductPerformanceField(key):
+			n.Row[key] = floatFromAny(n.Row[key]) + floatFromAny(value)
+		case shouldAverageProductPerformanceField(key):
+			if n.Avg == nil {
+				n.Avg = map[string]*productPerformanceGroupedSnapshotAvgState{}
+			}
+			state := n.Avg[key]
+			if state == nil {
+				state = &productPerformanceGroupedSnapshotAvgState{}
+				n.Avg[key] = state
+			}
+			number := floatFromAny(value)
+			weight := floatFromMap(row, productPerformanceMetricWeightField(key))
+			if weight > 0 {
+				state.Weighted += number * weight
+				state.Weight += weight
+			}
+			state.Sum += number
+			state.Count++
+		default:
+			if _, ok := n.Row[key]; !ok {
+				n.Row[key] = value
+			}
+		}
+	}
+}
+
+func appendProductPerformanceGroupedSnapshotNodes(out *[]map[string]any, nodes map[string]*productPerformanceGroupedSnapshotNode, order []string) {
+	sort.SliceStable(order, func(i, j int) bool {
+		left := nodes[order[i]]
+		right := nodes[order[j]]
+		if left == nil || right == nil {
+			return order[i] < order[j]
+		}
+		return strings.Compare(left.Value, right.Value) < 0
+	})
+	for _, key := range order {
+		node := nodes[key]
+		if node == nil {
+			continue
+		}
+		*out = append(*out, node.snapshotRow())
+		if len(node.Children) > 0 {
+			appendProductPerformanceGroupedSnapshotNodes(out, node.Children, node.ChildOrder)
+		}
+	}
+}
+
+func (n *productPerformanceGroupedSnapshotNode) snapshotRow() map[string]any {
+	row := cloneMap(n.Row)
+	for key, state := range n.Avg {
+		if state == nil {
+			continue
+		}
+		if state.Weight > 0 {
+			row[key] = state.Weighted / state.Weight
+		} else if state.Count > 0 {
+			row[key] = state.Sum / state.Count
+		}
+	}
+	deriveProductPerformanceGroupMetrics(row, n.Field)
+	if bucket := n.dominantBucket(); bucket != "" {
+		row["performance_bucket"] = bucket
+	}
+	clearProductPerformanceGroupDimensions(row, n.Field, n.Value)
+	row["__group"] = true
+	row["row_key"] = "group|" + n.Key
+	row["key"] = n.Key
+	row["level"] = n.Level
+	row["group_field"] = n.Field
+	row["group_value"] = n.Value
+	row["label"] = n.Value
+	row["count"] = n.Count
+	row["recommendation"] = fmt.Sprintf("%d satir", n.Count)
+	image := n.Image
+	if image == nil {
+		image = row
+	}
+	row["image_product_code"] = stringFromMap(image, "product_code")
+	row["image_color_code"] = stringFromMap(image, "color_code")
+	row["image_yaka_kodu"] = stringFromMap(image, "yaka_kodu")
+	return row
+}
+
+func (n *productPerformanceGroupedSnapshotNode) dominantBucket() string {
+	best := ""
+	bestCount := 0
+	for bucket, count := range n.BucketCounts {
+		if count > bestCount || (count == bestCount && bucket < best) {
+			best = bucket
+			bestCount = count
+		}
+	}
+	return best
+}
+
 func appendProductPerformanceGroupedRows(out *[]map[string]any, sourceRows []map[string]any, levels []string, level int, visualLevel int, parentKeys []string, expandedKeys map[string]bool, expandThroughLevel int) {
 	if level >= len(levels) {
 		return