change values merge sequence (#2934)

* change values merge sequence
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix image preview error
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* add custom workflow deploy logic
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix value merge error of workflows
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix values preview error
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* add debug log
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* add debug log
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix image preview error
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix image parse error
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* fix image parse error
Signed-off-by: Nallenshen <shendongdong@koderover.com>

* remove debug log
Signed-off-by: Nallenshen <shendongdong@koderover.com>

---------
Signed-off-by: Nallenshen <shendongdong@koderover.com>

pkg/microservice/aslan/core/common/service/kube/apply.go

@@ -530,7 +530,7 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Rende
 	productService := applyParam.ProductInfo.GetServiceMap()[applyParam.ServiceName]
 	if productService == nil {
 		if !applyParam.UpdateServiceRevision {
-			return nil, nil, nil, fmt.Errorf("First time online service %s needs to check the update service configuration", applyParam.ServiceName)
+			return nil, nil, nil, fmt.Errorf("first time online service %s needs to check the update service configuration", applyParam.ServiceName)
 		}
 
 		productService = &commonmodels.ProductService{

pkg/microservice/aslan/core/common/service/kube/render.go

@@ -38,7 +38,6 @@ import (
 	commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb"
 	templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository"
-	commomtemplate "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template"
 	commontypes "github.com/koderover/zadig/pkg/microservice/aslan/core/common/types"
 	commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util"
 	"github.com/koderover/zadig/pkg/setting"
@@ -266,7 +265,7 @@ func buildContainerMap(cs []*models.Container) map[string]*models.Container {
 	return containerMap
 }
 
-// calculateContainer calculates containers to be applied into environments for helm and k8s projects
+// CalculateContainer calculates containers to be applied into environments for helm and k8s projects
 // if image has no change since last deploy, containers in latest service will be used
 // if image hse been change since lase deploy (eg. workflow), current values will be remained
 func CalculateContainer(productSvc *commonmodels.ProductService, curUsedSvc *commonmodels.Service, latestContainers []*models.Container, productInfo *commonmodels.Product) []*models.Container {
@@ -313,60 +312,29 @@ func mergeContainers(curContainers []*commonmodels.Container, newContainers ...[
 	return containers
 }
 
-func FetchCurrentServiceVariable(option *GeneSvcYamlOption) ([]*commonmodels.VariableKV, error) {
-	_, err := templaterepo.NewProductColl().Find(option.ProductName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find template product %s", option.ProductName)
-	}
-
-	productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{
-		EnvName: option.EnvName,
-		Name:    option.ProductName,
-	})
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find product %s", option.ProductName)
-	}
-
-	curProductSvc := productInfo.GetServiceMap()[option.ServiceName]
-
-	productSvcRevision := int64(0)
-	if curProductSvc != nil {
-		productSvcRevision = curProductSvc.Revision
-	}
-
-	prodSvcTemplate, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{
-		ProductName: option.ProductName,
-		ServiceName: option.ServiceName,
-		Revision:    productSvcRevision,
-	}, productInfo.Production)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find service %s with revision %d", option.ServiceName, productSvcRevision)
+func MergeImages(curContainers []*models.Container, images []string) []string {
+	ret := make([]string, 0)
+	containerMap := make(map[string]string)
+	for _, container := range curContainers {
+		containerMap[container.ImageName] = container.Image
 	}
 
-	var usedRenderset *commonmodels.RenderSet
-	usedRenderset, err = commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{
-		ProductTmpl: productInfo.ProductName,
-		EnvName:     productInfo.EnvName,
-		IsDefault:   false,
-		Revision:    productInfo.Render.Revision,
-		Name:        productInfo.Render.Name,
-	})
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find renderset for %s/%s", productInfo.ProductName, productInfo.EnvName)
+	imageMap := make(map[string]string)
+	for _, image := range images {
+		imageMap[util.ExtractImageName(image)] = image
 	}
 
-	serviceVariableYaml := ""
-	serviceRender := usedRenderset.GetServiceRenderMap()[option.ServiceName]
-	if serviceRender != nil && serviceRender.OverrideYaml != nil {
-		serviceVariableYaml = serviceRender.OverrideYaml.YamlContent
+	for imageName, image := range imageMap {
+		if len(imageName) == 0 {
+			continue
+		}
+		containerMap[imageName] = image
 	}
 
-	variableYaml, _, err := commomtemplate.SafeMergeVariableYaml(prodSvcTemplate.VariableYaml, serviceVariableYaml)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to merge variable yaml for %s/%s", option.ProductName, option.ServiceName)
+	for _, image := range containerMap {
+		ret = append(ret, image)
 	}
-
-	return GeneKVFromYaml(variableYaml)
+	return ret
 }
 
 // FetchCurrentAppliedYaml generates full yaml of some service currently applied in Zadig

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go

@@ -33,6 +33,7 @@ import (
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template"
 	commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube"
+	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository"
 	"github.com/koderover/zadig/pkg/setting"
 	"github.com/koderover/zadig/pkg/tool/log"
 )
@@ -108,6 +109,13 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) {
 		Timeout:               c.timeout(),
 	}
 
+	curProdSvcRevision := func() int64 {
+		if productInfo.GetServiceMap()[c.jobTaskSpec.ServiceName] != nil {
+			return productInfo.GetServiceMap()[c.jobTaskSpec.ServiceName].Revision
+		}
+		return 0
+	}()
+
 	renderSet, productService, svcTemplate, err := kube.PrepareHelmServiceData(param)
 	if err != nil {
 		msg := fmt.Sprintf("prepare helm service data error: %v", err)
@@ -115,6 +123,33 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) {
 		return
 	}
 
+	if updateServiceRevision && curProdSvcRevision > int64(0) {
+		svcFindOption := &commonrepo.ServiceFindOption{
+			ProductName: productInfo.ProductName,
+			ServiceName: c.jobTaskSpec.ServiceName,
+		}
+		latestSvc, err := repository.QueryTemplateService(svcFindOption, productInfo.Production)
+		if err != nil {
+			msg := fmt.Sprintf("failed to find service %s/%d in product %s, error: %v", productInfo.ProductName, svcFindOption.Revision, productInfo.ProductName, err)
+			logError(c.job, msg, c.logger)
+			return
+		}
+
+		curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{
+			ProductName: productInfo.ProductName,
+			ServiceName: c.jobTaskSpec.ServiceName,
+			Revision:    curProdSvcRevision,
+		}, productInfo.Production)
+		if err != nil {
+			msg := fmt.Sprintf("failed to find service %s/%d in product %s, error: %v", productInfo.ProductName, productService.Revision, productInfo.ProductName, err)
+			logError(c.job, msg, c.logger)
+			return
+		}
+
+		calculatedContainers := kube.CalculateContainer(productService, curUsedSvc, latestSvc.Containers, productInfo)
+		param.Images = kube.MergeImages(calculatedContainers, param.Images)
+	}
+
 	chartInfo, ok := renderSet.GetChartRenderMap()[param.ServiceName]
 	if !ok {
 		msg := fmt.Sprintf("failed to find chart info in render")

pkg/microservice/aslan/core/environment/service/environment.go

@@ -2881,24 +2881,6 @@ func buildInstallParam(defaultValues string, productInfo *commonmodels.Product,
 	return ret, nil
 }
 
-func buildChartInstallParam(namespace, envName, defaultValues string, renderChart *templatemodels.ServiceRender, serviceObj *commonmodels.Service, productSvc *commonmodels.ProductService) (*kube.ReleaseInstallParam, error) {
-	mergedValues, err := helmtool.MergeOverrideValues("", defaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to merge release %s: override yaml %s and values %s, err: %s", renderChart.ReleaseName, renderChart.GetOverrideYaml(), renderChart.OverrideValues, err)
-	}
-	ret := &kube.ReleaseInstallParam{
-		ProductName:    productSvc.ProductName,
-		Namespace:      namespace,
-		ReleaseName:    renderChart.ReleaseName,
-		MergedValues:   mergedValues,
-		RenderChart:    renderChart,
-		ServiceObj:     serviceObj,
-		Production:     true,
-		IsChartInstall: true,
-	}
-	return ret, nil
-}
-
 func installProductHelmCharts(user, requestID string, args *commonmodels.Product, renderset *commonmodels.RenderSet, eventStart int64, helmClient *helmtool.HelmClient,
 	kclient client.Client, istioClient versionedclient.Interface, log *zap.SugaredLogger) {
 	var (

pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go

@@ -28,8 +28,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/koderover/zadig/pkg/shared/client/user"
-	internalhandler "github.com/koderover/zadig/pkg/shared/handler"
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -60,6 +58,7 @@ import (
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube"
 	larkservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/lark"
+	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository"
 	commomtemplate "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template"
 	"github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook"
 	commontypes "github.com/koderover/zadig/pkg/microservice/aslan/core/common/types"
@@ -67,6 +66,8 @@ import (
 	jobctl "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow/job"
 	"github.com/koderover/zadig/pkg/microservice/picket/client/opa"
 	"github.com/koderover/zadig/pkg/setting"
+	"github.com/koderover/zadig/pkg/shared/client/user"
+	internalhandler "github.com/koderover/zadig/pkg/shared/handler"
 	kubeclient "github.com/koderover/zadig/pkg/shared/kube/client"
 	e "github.com/koderover/zadig/pkg/tool/errors"
 	helmtool "github.com/koderover/zadig/pkg/tool/helmclient"
@@ -2122,12 +2123,43 @@ func CompareHelmServiceYamlInEnv(serviceName, variableYaml, envName, projectName
 		Timeout:               setting.DeployTimeout,
 	}
 
+	curProdSvcRevision := func() int64 {
+		if prod.GetServiceMap()[serviceName] != nil {
+			return prod.GetServiceMap()[serviceName].Revision
+		}
+		return 0
+	}()
+
 	renderSet, productService, _, err := kube.PrepareHelmServiceData(param)
 	if err != nil {
 		log.Errorf("prepare helm service data error: %v", err)
 		return nil, err
 	}
 
+	if updateServiceRevision && curProdSvcRevision > int64(0) {
+		svcFindOption := &commonrepo.ServiceFindOption{
+			ProductName: prod.ProductName,
+			ServiceName: serviceName,
+		}
+		latestSvc, err := repository.QueryTemplateService(svcFindOption, prod.Production)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to find service %s/%d in product %s", serviceName, svcFindOption.Revision, prod.ProductName)
+		}
+
+		curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{
+			ProductName: prod.ProductName,
+			ServiceName: serviceName,
+			Revision:    curProdSvcRevision,
+		}, prod.Production)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to find service %s/%d in product %s", serviceName, svcFindOption.Revision, prod.ProductName)
+		}
+
+		containers := kube.CalculateContainer(productService, curUsedSvc, latestSvc.Containers, prod)
+		images = kube.MergeImages(containers, images)
+		param.Images = images
+	}
+
 	chartInfo, ok := renderSet.GetChartRenderMap()[param.ServiceName]
 	if !ok {
 		log.Errorf("failed to find chart info in render")

pkg/tool/helmclient/helmclient.go

@@ -177,8 +177,24 @@ type KV struct {
 func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues string, imageKvs []*KV) (string, error) {
 
 	// merge files for helm -f option
-	// precedence from low to high: valuesYaml defaultValues overrideYaml
-	valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{[]byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)})
+	// precedence from low to high: images valuesYaml defaultValues overrideYaml
+
+	var imageRelatedValues []byte
+	if len(imageKvs) > 0 {
+		imageValuesMap := make(map[string]interface{})
+		imageKvStr := make([]string, 0)
+		// image related values
+		for _, imageKv := range imageKvs {
+			imageKvStr = append(imageKvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value))
+		}
+		err := strvals.ParseInto(strings.Join(imageKvStr, ","), imageValuesMap)
+		if err != nil {
+			return "", err
+		}
+		imageRelatedValues, err = yaml.Marshal(imageValuesMap)
+	}
+
+	valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{imageRelatedValues, []byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)})
 	if err != nil {
 		return "", err
 	}
@@ -196,10 +212,11 @@ func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues
 		}
 	}
 
-	// image related values
-	for _, imageKv := range imageKvs {
-		kvStr = append(kvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value))
-	}
+	//// image related values
+	//for _, imageKv := range imageKvs {
+	//	kvStr = append(kvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value))
+	//}
+	//
 
 	// override values for --set option
 	if len(kvStr) > 0 {