remove unused field for db's sake

pkg/cli/upgradeassistant/cmd/migrate/300.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2023 The KodeRover Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrate
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/koderover/zadig/v2/pkg/cli/upgradeassistant/internal/upgradepath"
+	commonmodels "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/models"
+	"github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb"
+	templaterepo "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/common/repository/mongodb/template"
+	jobctl "github.com/koderover/zadig/v2/pkg/microservice/aslan/core/workflow/service/workflow/job"
+	"github.com/koderover/zadig/v2/pkg/tool/log"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+func init() {
+	upgradepath.RegisterHandler("2.3.0", "3.0.0", V230ToV300)
+	upgradepath.RegisterHandler("3.0.0", "2.3.0", V230ToV220)
+}
+
+func V230ToV300() error {
+	log.Infof("-------- start migrate workflow task data --------")
+	err := removeWorkflowTaskExtraData()
+	if err != nil {
+		log.Errorf("migrate workflow task data error: %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func V300ToV230() error {
+	return nil
+}
+
+// remove 100 of the workflow task's extra option field to make data clean
+func removeWorkflowTaskExtraData() error {
+	// find all projects to do the migration
+	projects, err := templaterepo.NewProductColl().List()
+	if err != nil {
+		log.Errorf("failed to list project list to do workflow migrations, error: %s", err)
+		return fmt.Errorf("failed to list project list to do workflow migrations")
+	}
+
+	for _, project := range projects {
+		// list all workflows
+		workflows, _, err := mongodb.NewWorkflowV4Coll().List(&mongodb.ListWorkflowV4Option{
+			ProjectName: project.ProductName,
+		}, 0, 0)
+
+		if err != nil {
+			log.Errorf("failed to list workflow for project: %s, error: %s", project.ProductName, err)
+			return fmt.Errorf("failed to list workflow for project: %s, error: %s", project.ProductName, err)
+		}
+
+		for _, workflow := range workflows {
+			// list first 100 task to remove all unnecessary fields
+			tasks, _, err := mongodb.NewworkflowTaskv4Coll().List(&mongodb.ListWorkflowTaskV4Option{
+				WorkflowName: "",
+				ProjectName:  "",
+				Limit:        100,
+				Skip:         0,
+			})
+
+			if err != nil && (err != mongo.ErrNilDocument && err != mongo.ErrNoDocuments) {
+				log.Errorf("failed to list workflow task for project: %s, workflowName: %s, error: %s", project.ProductName, workflow.Name, err)
+				return fmt.Errorf("failed to list workflow task for project: %s, workflowName: %s, error: %s", project.ProductName, workflow.Name, err)
+			}
+
+			if tasks != nil && len(tasks) != 0 {
+				for _, task := range tasks {
+					err := cleanTask(task, workflow)
+					if err != nil {
+						log.Errorf("failed to clean the task's data, taskID: %d, workflowName: %s, project: %s, error: %s", task.TaskID, workflow.Name, project.ProductName, err)
+						return fmt.Errorf("failed to clean the task's data, taskID: %d, workflowName: %s, project: %s, error: %s", task.TaskID, workflow.Name, project.ProductName, err)
+					}
+
+					err = mongodb.NewworkflowTaskv4Coll().Update(task.ID.Hex(), task)
+					if err != nil {
+						log.Errorf("failed to update the task's data, taskID: %d, workflowName: %s, project: %s, error: %s", task.TaskID, workflow.Name, project.ProductName, err)
+						return fmt.Errorf("failed to update the task's data, taskID: %d, workflowName: %s, project: %s, error: %s", task.TaskID, workflow.Name, project.ProductName, err)
+					}
+				}
+			}
+		}
+
+		// after a project is done, trigger a manual gc to save memory
+		runtime.GC()
+	}
+
+	return nil
+}
+
+func cleanTask(task *commonmodels.WorkflowTask, workflow *commonmodels.WorkflowV4) error {
+	for _, stage := range task.OriginWorkflowArgs.Stages {
+		for _, job := range stage.Jobs {
+			err := jobctl.ClearOptions(job, workflow)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/microservice/aslan/core/workflow/service/workflow/job/job.go

@@ -53,6 +53,8 @@ type JobCtl interface {
 	SetPreset() error
 	// SetOptions sets all the possible options for the workflow
 	SetOptions() error
+	// ClearOptions clears the option field to save space for db and memory
+	ClearOptions() error
 	ClearSelectionField() error
 	ToJobs(taskID int64) ([]*commonmodels.JobTask, error)
 	// MergeArgs merge the current workflow with the user input: args
@@ -182,6 +184,14 @@ func SetOptions(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) error
 	return jobCtl.SetOptions()
 }
 
+func ClearOptions(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) error {
+	jobCtl, err := InitJobCtl(job, workflow)
+	if err != nil {
+		return warpJobError(job.Name, err)
+	}
+	return jobCtl.ClearOptions()
+}
+
 func UpdateWithLatestSetting(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) error {
 	jobCtl, err := InitJobCtl(job, workflow)
 	if err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_apollo.go

@@ -125,6 +125,17 @@ func (j *ApolloJob) SetOptions() error {
 	return nil
 }
 
+func (j *ApolloJob) ClearOptions() error {
+	j.spec = &commonmodels.ApolloJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.NamespaceListOption = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *ApolloJob) ClearSelectionField() error {
 	j.spec = &commonmodels.ApolloJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_blue_green_deploy_v2.go

@@ -168,6 +168,17 @@ func (j *BlueGreenDeployV2Job) SetOptions() error {
 	return nil
 }
 
+func (j *BlueGreenDeployV2Job) ClearOptions() error {
+	j.spec = &commonmodels.BlueGreenDeployV2JobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.EnvOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *BlueGreenDeployV2Job) ClearSelectionField() error {
 	j.spec = &commonmodels.BlueGreenDeployV2JobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_blue_green_release_v2.go

@@ -55,6 +55,10 @@ func (j *BlueGreenReleaseV2Job) SetOptions() error {
 	return nil
 }
 
+func (j *BlueGreenReleaseV2Job) ClearOptions() error {
+	return nil
+}
+
 func (j *BlueGreenReleaseV2Job) ClearSelectionField() error {
 	return nil
 }

pkg/microservice/aslan/core/workflow/service/workflow/job/job_blueking.go

@@ -54,6 +54,10 @@ func (j *BlueKingJob) SetOptions() error {
 	return nil
 }
 
+func (j *BlueKingJob) ClearOptions() error {
+	return nil
+}
+
 func (j *BlueKingJob) ClearSelectionField() error {
 	j.spec = &commonmodels.BlueKingJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_build.go

@@ -237,6 +237,17 @@ func (j *BuildJob) SetOptions() error {
 	return nil
 }
 
+func (j *BuildJob) ClearOptions() error {
+	j.spec = &commonmodels.ZadigBuildJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.ServiceAndBuildsOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *BuildJob) GetRepos() ([]*types.Repository, error) {
 	resp := []*types.Repository{}
 	j.spec = &commonmodels.ZadigBuildJobSpec{}

pkg/microservice/aslan/core/workflow/service/workflow/job/job_canary_deploy.go

@@ -98,6 +98,17 @@ func (j *CanaryDeployJob) SetOptions() error {
 	return nil
 }
 
+func (j *CanaryDeployJob) ClearOptions() error {
+	j.spec = &commonmodels.CanaryDeployJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.TargetOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *CanaryDeployJob) ClearSelectionField() error {
 	j.spec = &commonmodels.CanaryDeployJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_canary_release.go

@@ -54,6 +54,10 @@ func (j *CanaryReleaseJob) SetOptions() error {
 	return nil
 }
 
+func (j *CanaryReleaseJob) ClearOptions() error {
+	return nil
+}
+
 func (j *CanaryReleaseJob) ClearSelectionField() error {
 	return nil
 }

pkg/microservice/aslan/core/workflow/service/workflow/job/job_custom_deploy.go

@@ -89,6 +89,17 @@ func (j *CustomDeployJob) SetOptions() error {
 	return nil
 }
 
+func (j *CustomDeployJob) ClearOptions() error {
+	j.spec = &commonmodels.CustomDeployJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.TargetOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *CustomDeployJob) ClearSelectionField() error {
 	j.spec = &commonmodels.CustomDeployJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_deploy.go

@@ -379,6 +379,17 @@ func (j *DeployJob) SetOptions() error {
 	return nil
 }
 
+func (j *DeployJob) ClearOptions() error {
+	j.spec = &commonmodels.ZadigDeployJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.EnvOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *DeployJob) ClearSelectionField() error {
 	j.spec = &commonmodels.ZadigDeployJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_distribute_image.go

@@ -133,6 +133,17 @@ func (j *ImageDistributeJob) SetOptions() error {
 	return nil
 }
 
+func (j *ImageDistributeJob) ClearOptions() error {
+	j.spec = &commonmodels.ZadigDistributeImageJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.TargetOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *ImageDistributeJob) ClearSelectionField() error {
 	j.spec = &commonmodels.ZadigDistributeImageJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_freestyle.go

@@ -110,6 +110,10 @@ func (j *FreeStyleJob) SetOptions() error {
 	return nil
 }
 
+func (j *FreeStyleJob) ClearOptions() error {
+	return nil
+}
+
 func (j *FreeStyleJob) ClearSelectionField() error {
 	j.spec = &commonmodels.FreestyleJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_grafana.go

@@ -57,6 +57,10 @@ func (j *GrafanaJob) SetOptions() error {
 	return nil
 }
 
+func (j *GrafanaJob) ClearOptions() error {
+	return nil
+}
+
 func (j *GrafanaJob) ClearSelectionField() error {
 	return nil
 }

pkg/microservice/aslan/core/workflow/service/workflow/job/job_gray_release.go

@@ -90,6 +90,17 @@ func (j *GrayReleaseJob) SetOptions() error {
 	return nil
 }
 
+func (j *GrayReleaseJob) ClearOptions() error {
+	j.spec = &commonmodels.GrayReleaseJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.TargetOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *GrayReleaseJob) ClearSelectionField() error {
 	j.spec = &commonmodels.GrayReleaseJobSpec{}
 	if err := commonmodels.IToiYaml(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_gray_rollback.go

@@ -132,6 +132,17 @@ func (j *GrayRollbackJob) SetOptions() error {
 	return nil
 }
 
+func (j *GrayRollbackJob) ClearOptions() error {
+	j.spec = &commonmodels.GrayRollbackJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.Targets = make([]*commonmodels.GrayRollbackTarget, 0)
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *GrayRollbackJob) ClearSelectionField() error {
 	j.spec = &commonmodels.GrayRollbackJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {

pkg/microservice/aslan/core/workflow/service/workflow/job/job_guanceyun.go

@@ -54,6 +54,10 @@ func (j *GuanceyunCheckJob) SetOptions() error {
 	return nil
 }
 
+func (j *GuanceyunCheckJob) ClearOptions() error {
+	return nil
+}
+
 func (j *GuanceyunCheckJob) ClearSelectionField() error {
 	return nil
 }

pkg/microservice/aslan/core/workflow/service/workflow/job/job_helm_chart_deploy.go

@@ -149,6 +149,17 @@ func (j *HelmChartDeployJob) SetOptions() error {
 	return nil
 }
 
+func (j *HelmChartDeployJob) ClearOptions() error {
+	j.spec = &commonmodels.ZadigHelmChartDeployJobSpec{}
+	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {
+		return err
+	}
+
+	j.spec.EnvOptions = nil
+	j.job.Spec = j.spec
+	return nil
+}
+
 func (j *HelmChartDeployJob) ClearSelectionField() error {
 	j.spec = &commonmodels.ZadigHelmChartDeployJobSpec{}
 	if err := commonmodels.IToi(j.job.Spec, j.spec); err != nil {