From f3b8f4533f2a60dd0835af332d1e7890c89b25d5 Mon Sep 17 00:00:00 2001 From: shendongdong Date: Wed, 9 Aug 2023 18:05:41 +0800 Subject: [PATCH] change values merge sequence (#2934) * change values merge sequence Signed-off-by: allenshen * fix image preview error Signed-off-by: allenshen * add custom workflow deploy logic Signed-off-by: allenshen * fix value merge error of workflows Signed-off-by: allenshen * fix values preview error Signed-off-by: allenshen * add debug log Signed-off-by: allenshen * add debug log Signed-off-by: allenshen * fix image preview error Signed-off-by: allenshen * fix image parse error Signed-off-by: allenshen * fix image parse error Signed-off-by: allenshen * remove debug log Signed-off-by: allenshen --------- Signed-off-by: allenshen --- .../aslan/core/common/service/kube/apply.go | 2 +- .../aslan/core/common/service/kube/render.go | 66 +++++-------------- .../jobcontroller/job_helm_deploy.go | 35 ++++++++++ .../core/environment/service/environment.go | 18 ----- .../workflow/service/workflow/workflow_v4.go | 36 +++++++++- pkg/tool/helmclient/helmclient.go | 29 ++++++-- 6 files changed, 110 insertions(+), 76 deletions(-) diff --git a/pkg/microservice/aslan/core/common/service/kube/apply.go b/pkg/microservice/aslan/core/common/service/kube/apply.go index 3cb0d9ed0..f7e226b55 100644 --- a/pkg/microservice/aslan/core/common/service/kube/apply.go +++ b/pkg/microservice/aslan/core/common/service/kube/apply.go @@ -530,7 +530,7 @@ func PrepareHelmServiceData(applyParam *ResourceApplyParam) (*commonmodels.Rende productService := applyParam.ProductInfo.GetServiceMap()[applyParam.ServiceName] if productService == nil { if !applyParam.UpdateServiceRevision { - return nil, nil, nil, fmt.Errorf("First time online service %s needs to check the update service configuration", applyParam.ServiceName) + return nil, nil, nil, fmt.Errorf("first time online service %s needs to check the update service configuration", applyParam.ServiceName) } productService = &commonmodels.ProductService{ diff --git a/pkg/microservice/aslan/core/common/service/kube/render.go b/pkg/microservice/aslan/core/common/service/kube/render.go index a6e250c81..ef27ccebf 100644 --- a/pkg/microservice/aslan/core/common/service/kube/render.go +++ b/pkg/microservice/aslan/core/common/service/kube/render.go @@ -38,7 +38,6 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository" - commomtemplate "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" commontypes "github.com/koderover/zadig/pkg/microservice/aslan/core/common/types" commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" "github.com/koderover/zadig/pkg/setting" @@ -266,7 +265,7 @@ func buildContainerMap(cs []*models.Container) map[string]*models.Container { return containerMap } -// calculateContainer calculates containers to be applied into environments for helm and k8s projects +// CalculateContainer calculates containers to be applied into environments for helm and k8s projects // if image has no change since last deploy, containers in latest service will be used // if image hse been change since lase deploy (eg. workflow), current values will be remained func CalculateContainer(productSvc *commonmodels.ProductService, curUsedSvc *commonmodels.Service, latestContainers []*models.Container, productInfo *commonmodels.Product) []*models.Container { @@ -313,60 +312,29 @@ func mergeContainers(curContainers []*commonmodels.Container, newContainers ...[ return containers } -func FetchCurrentServiceVariable(option *GeneSvcYamlOption) ([]*commonmodels.VariableKV, error) { - _, err := templaterepo.NewProductColl().Find(option.ProductName) - if err != nil { - return nil, errors.Wrapf(err, "failed to find template product %s", option.ProductName) - } - - productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ - EnvName: option.EnvName, - Name: option.ProductName, - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to find product %s", option.ProductName) - } - - curProductSvc := productInfo.GetServiceMap()[option.ServiceName] - - productSvcRevision := int64(0) - if curProductSvc != nil { - productSvcRevision = curProductSvc.Revision - } - - prodSvcTemplate, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{ - ProductName: option.ProductName, - ServiceName: option.ServiceName, - Revision: productSvcRevision, - }, productInfo.Production) - if err != nil { - return nil, errors.Wrapf(err, "failed to find service %s with revision %d", option.ServiceName, productSvcRevision) +func MergeImages(curContainers []*models.Container, images []string) []string { + ret := make([]string, 0) + containerMap := make(map[string]string) + for _, container := range curContainers { + containerMap[container.ImageName] = container.Image } - var usedRenderset *commonmodels.RenderSet - usedRenderset, err = commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ - ProductTmpl: productInfo.ProductName, - EnvName: productInfo.EnvName, - IsDefault: false, - Revision: productInfo.Render.Revision, - Name: productInfo.Render.Name, - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to find renderset for %s/%s", productInfo.ProductName, productInfo.EnvName) + imageMap := make(map[string]string) + for _, image := range images { + imageMap[util.ExtractImageName(image)] = image } - serviceVariableYaml := "" - serviceRender := usedRenderset.GetServiceRenderMap()[option.ServiceName] - if serviceRender != nil && serviceRender.OverrideYaml != nil { - serviceVariableYaml = serviceRender.OverrideYaml.YamlContent + for imageName, image := range imageMap { + if len(imageName) == 0 { + continue + } + containerMap[imageName] = image } - variableYaml, _, err := commomtemplate.SafeMergeVariableYaml(prodSvcTemplate.VariableYaml, serviceVariableYaml) - if err != nil { - return nil, errors.Wrapf(err, "failed to merge variable yaml for %s/%s", option.ProductName, option.ServiceName) + for _, image := range containerMap { + ret = append(ret, image) } - - return GeneKVFromYaml(variableYaml) + return ret } // FetchCurrentAppliedYaml generates full yaml of some service currently applied in Zadig diff --git a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go index b7d189f36..fbef7da98 100644 --- a/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go +++ b/pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_helm_deploy.go @@ -33,6 +33,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) @@ -108,6 +109,13 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) { Timeout: c.timeout(), } + curProdSvcRevision := func() int64 { + if productInfo.GetServiceMap()[c.jobTaskSpec.ServiceName] != nil { + return productInfo.GetServiceMap()[c.jobTaskSpec.ServiceName].Revision + } + return 0 + }() + renderSet, productService, svcTemplate, err := kube.PrepareHelmServiceData(param) if err != nil { msg := fmt.Sprintf("prepare helm service data error: %v", err) @@ -115,6 +123,33 @@ func (c *HelmDeployJobCtl) Run(ctx context.Context) { return } + if updateServiceRevision && curProdSvcRevision > int64(0) { + svcFindOption := &commonrepo.ServiceFindOption{ + ProductName: productInfo.ProductName, + ServiceName: c.jobTaskSpec.ServiceName, + } + latestSvc, err := repository.QueryTemplateService(svcFindOption, productInfo.Production) + if err != nil { + msg := fmt.Sprintf("failed to find service %s/%d in product %s, error: %v", productInfo.ProductName, svcFindOption.Revision, productInfo.ProductName, err) + logError(c.job, msg, c.logger) + return + } + + curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{ + ProductName: productInfo.ProductName, + ServiceName: c.jobTaskSpec.ServiceName, + Revision: curProdSvcRevision, + }, productInfo.Production) + if err != nil { + msg := fmt.Sprintf("failed to find service %s/%d in product %s, error: %v", productInfo.ProductName, productService.Revision, productInfo.ProductName, err) + logError(c.job, msg, c.logger) + return + } + + calculatedContainers := kube.CalculateContainer(productService, curUsedSvc, latestSvc.Containers, productInfo) + param.Images = kube.MergeImages(calculatedContainers, param.Images) + } + chartInfo, ok := renderSet.GetChartRenderMap()[param.ServiceName] if !ok { msg := fmt.Sprintf("failed to find chart info in render") diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index fc71abefe..e2fef7792 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -2881,24 +2881,6 @@ func buildInstallParam(defaultValues string, productInfo *commonmodels.Product, return ret, nil } -func buildChartInstallParam(namespace, envName, defaultValues string, renderChart *templatemodels.ServiceRender, serviceObj *commonmodels.Service, productSvc *commonmodels.ProductService) (*kube.ReleaseInstallParam, error) { - mergedValues, err := helmtool.MergeOverrideValues("", defaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues, nil) - if err != nil { - return nil, fmt.Errorf("failed to merge release %s: override yaml %s and values %s, err: %s", renderChart.ReleaseName, renderChart.GetOverrideYaml(), renderChart.OverrideValues, err) - } - ret := &kube.ReleaseInstallParam{ - ProductName: productSvc.ProductName, - Namespace: namespace, - ReleaseName: renderChart.ReleaseName, - MergedValues: mergedValues, - RenderChart: renderChart, - ServiceObj: serviceObj, - Production: true, - IsChartInstall: true, - } - return ret, nil -} - func installProductHelmCharts(user, requestID string, args *commonmodels.Product, renderset *commonmodels.RenderSet, eventStart int64, helmClient *helmtool.HelmClient, kclient client.Client, istioClient versionedclient.Interface, log *zap.SugaredLogger) { var ( diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go index 194fd41f4..46a553ed7 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_v4.go @@ -28,8 +28,6 @@ import ( "sync" "time" - "github.com/koderover/zadig/pkg/shared/client/user" - internalhandler "github.com/koderover/zadig/pkg/shared/handler" "github.com/pkg/errors" "github.com/samber/lo" "go.mongodb.org/mongo-driver/bson/primitive" @@ -60,6 +58,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/collaboration" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" larkservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/lark" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/repository" commomtemplate "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" commontypes "github.com/koderover/zadig/pkg/microservice/aslan/core/common/types" @@ -67,6 +66,8 @@ import ( jobctl "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow/job" "github.com/koderover/zadig/pkg/microservice/picket/client/opa" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/user" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" @@ -2122,12 +2123,43 @@ func CompareHelmServiceYamlInEnv(serviceName, variableYaml, envName, projectName Timeout: setting.DeployTimeout, } + curProdSvcRevision := func() int64 { + if prod.GetServiceMap()[serviceName] != nil { + return prod.GetServiceMap()[serviceName].Revision + } + return 0 + }() + renderSet, productService, _, err := kube.PrepareHelmServiceData(param) if err != nil { log.Errorf("prepare helm service data error: %v", err) return nil, err } + if updateServiceRevision && curProdSvcRevision > int64(0) { + svcFindOption := &commonrepo.ServiceFindOption{ + ProductName: prod.ProductName, + ServiceName: serviceName, + } + latestSvc, err := repository.QueryTemplateService(svcFindOption, prod.Production) + if err != nil { + return nil, errors.Wrapf(err, "failed to find service %s/%d in product %s", serviceName, svcFindOption.Revision, prod.ProductName) + } + + curUsedSvc, err := repository.QueryTemplateService(&commonrepo.ServiceFindOption{ + ProductName: prod.ProductName, + ServiceName: serviceName, + Revision: curProdSvcRevision, + }, prod.Production) + if err != nil { + return nil, errors.Wrapf(err, "failed to find service %s/%d in product %s", serviceName, svcFindOption.Revision, prod.ProductName) + } + + containers := kube.CalculateContainer(productService, curUsedSvc, latestSvc.Containers, prod) + images = kube.MergeImages(containers, images) + param.Images = images + } + chartInfo, ok := renderSet.GetChartRenderMap()[param.ServiceName] if !ok { log.Errorf("failed to find chart info in render") diff --git a/pkg/tool/helmclient/helmclient.go b/pkg/tool/helmclient/helmclient.go index 45f982702..e781eb508 100644 --- a/pkg/tool/helmclient/helmclient.go +++ b/pkg/tool/helmclient/helmclient.go @@ -177,8 +177,24 @@ type KV struct { func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues string, imageKvs []*KV) (string, error) { // merge files for helm -f option - // precedence from low to high: valuesYaml defaultValues overrideYaml - valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{[]byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)}) + // precedence from low to high: images valuesYaml defaultValues overrideYaml + + var imageRelatedValues []byte + if len(imageKvs) > 0 { + imageValuesMap := make(map[string]interface{}) + imageKvStr := make([]string, 0) + // image related values + for _, imageKv := range imageKvs { + imageKvStr = append(imageKvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value)) + } + err := strvals.ParseInto(strings.Join(imageKvStr, ","), imageValuesMap) + if err != nil { + return "", err + } + imageRelatedValues, err = yaml.Marshal(imageValuesMap) + } + + valuesMap, err := yamlutil.MergeAndUnmarshal([][]byte{imageRelatedValues, []byte(valuesYaml), []byte(defaultValues), []byte(overrideYaml)}) if err != nil { return "", err } @@ -196,10 +212,11 @@ func MergeOverrideValues(valuesYaml, defaultValues, overrideYaml, overrideValues } } - // image related values - for _, imageKv := range imageKvs { - kvStr = append(kvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value)) - } + //// image related values + //for _, imageKv := range imageKvs { + // kvStr = append(kvStr, fmt.Sprintf("%s=%v", imageKv.Key, imageKv.Value)) + //} + // // override values for --set option if len(kvStr) > 0 { -- GitLab