[Optimization] Calculate global parameter and local parameter at master. (#10704)

* Global parameter and local parameter calculation external expansion.

* k8s task ut fix.

* TimePlaceholderUtils import DateUtils fix

* follow the review comments to fix.

* follow the review comments to fix.

* e2e rerun

Author: WangJPLeo

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java

@@ -51,7 +51,7 @@
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.Flag;
-import org.apache.dolphinscheduler.common.expand.CuringGlobalParamsService;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.common.graph.DAG;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
@@ -155,7 +155,7 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
     private ScheduleMapper scheduleMapper;
 
     @Autowired
-    private CuringGlobalParamsService curingGlobalParamsService;
+    private CuringParamsService curingGlobalParamsService;
 
     /**
      * return top n SUCCESS process instance order by running time which started between startTime and endTime

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java

@@ -36,8 +36,7 @@
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.expand.CuringGlobalParamsService;
-import org.apache.dolphinscheduler.common.expand.DolphinSchedulerCuringGlobalParams;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.common.graph.DAG;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
@@ -72,7 +71,6 @@
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
-import org.springframework.beans.factory.annotation.Autowired;
 
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_DELETE;
 import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_UPDATE;
@@ -131,7 +129,7 @@ public class ProcessInstanceServiceTest {
     ScheduleMapper scheduleMapper;
 
     @Mock
-    CuringGlobalParamsService curingGlobalParamsService;
+    CuringParamsService curingGlobalParamsService;
 
 
     private String shellJson = "[{\"name\":\"\",\"preTaskCode\":0,\"preTaskVersion\":0,\"postTaskCode\":123456789,"

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@@ -831,4 +831,6 @@ private Constants() {
     public static final int USER_PASSWORD_MAX_LENGTH = 20;
 
     public static final int USER_PASSWORD_MIN_LENGTH = 2;
+
+    public static final String FUNCTION_START_WITH = "$";
 }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/expand/DolphinSchedulerCuringGlobalParams.java

@@ -512,7 +512,7 @@ public static Map.Entry<Date, String> calcMinutes(String expression, Date date)
 
             if (Character.isDigit(expression.charAt(index + 1))) {
                 String addMinuteExpr = expression.substring(index + 1);
-                Date targetDate = org.apache.commons.lang.time.DateUtils
+                Date targetDate = org.apache.commons.lang3.time.DateUtils
                     .addMinutes(date, calcMinutes(addMinuteExpr));
                 String dateFormat = expression.substring(0, index);
 

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java

@@ -20,27 +20,16 @@
 import static org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils.replacePlaceholders;
 
 import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.expand.TimePlaceholderResolverExpandService;
-import org.apache.dolphinscheduler.common.expand.TimePlaceholderResolverExpandServiceImpl;
 import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
-import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
-import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
-import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 
 import java.text.ParseException;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,12 +38,6 @@
 public class ParameterUtilsTest {
     public static final Logger logger = LoggerFactory.getLogger(ParameterUtilsTest.class);
 
-    @Mock
-    private TimePlaceholderResolverExpandService timePlaceholderResolverExpandService;
-
-    @InjectMocks
-    private TimePlaceholderResolverExpandServiceImpl timePlaceholderResolverExpandServiceImpl;
-
     /**
      * Test convertParameterPlaceholders
      */

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ParameterUtilsTest.java

@@ -29,7 +29,12 @@
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
+
+import java.util.Map;
 
 /**
  *  TaskExecutionContext builder
@@ -131,6 +136,28 @@ public TaskExecutionContextBuilder buildK8sTaskRelatedInfo(K8sTaskExecutionConte
         taskExecutionContext.setK8sTaskExecutionContext(k8sTaskExecutionContext);
         return this;
     }
+
+    /**
+     * build global and local params
+     * @param propertyMap
+     * @return
+     */
+    public TaskExecutionContextBuilder buildParamInfo(Map<String, Property> propertyMap) {
+        taskExecutionContext.setPrepareParamsMap(propertyMap);
+        return this;
+    }
+
+    /**
+     * build business params
+     * @param businessParamsMap
+     * @return
+     */
+    public TaskExecutionContextBuilder buildBusinessParamsMap(Map<String, Property> businessParamsMap) {
+        taskExecutionContext.setParamsMap(businessParamsMap);
+        return this;
+    }
+
+
     /**
      * create
      *

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java

@@ -19,7 +19,7 @@
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.SlotCheckState;
-import org.apache.dolphinscheduler.common.expand.CuringGlobalParamsService;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
@@ -95,7 +95,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
     private StateWheelExecuteThread stateWheelExecuteThread;
 
     @Autowired
-    private CuringGlobalParamsService curingGlobalParamsService;
+    private CuringParamsService curingGlobalParamsService;
 
     private String masterAddress;
 

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java

@@ -40,7 +40,7 @@
 import org.apache.dolphinscheduler.common.enums.TaskDependType;
 import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
 import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
-import org.apache.dolphinscheduler.common.expand.CuringGlobalParamsService;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.common.graph.DAG;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
@@ -49,7 +49,6 @@
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
 import org.apache.dolphinscheduler.dao.entity.Environment;
 import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@@ -239,7 +238,7 @@ public class WorkflowExecuteRunnable implements Runnable {
     /**
      * curing global params service
      */
-    private final CuringGlobalParamsService curingGlobalParamsService;
+    private final CuringParamsService curingParamsService;
 
     /**
      * @param processInstance         processInstance
@@ -255,14 +254,14 @@ public WorkflowExecuteRunnable(ProcessInstance processInstance
             , ProcessAlertManager processAlertManager
             , MasterConfig masterConfig
             , StateWheelExecuteThread stateWheelExecuteThread
-            , CuringGlobalParamsService curingGlobalParamsService) {
+            , CuringParamsService curingParamsService) {
         this.processService = processService;
         this.processInstance = processInstance;
         this.masterConfig = masterConfig;
         this.nettyExecutorManager = nettyExecutorManager;
         this.processAlertManager = processAlertManager;
         this.stateWheelExecuteThread = stateWheelExecuteThread;
-        this.curingGlobalParamsService = curingGlobalParamsService;
+        this.curingParamsService = curingParamsService;
         TaskMetrics.registerTaskPrepared(readyToSubmitTaskQueue::size);
     }
 
@@ -1008,7 +1007,7 @@ private void initTaskQueue() {
 
                     if (!complementListDate.isEmpty() && Flag.NO == processInstance.getIsSubProcess()) {
                         processInstance.setScheduleTime(complementListDate.get(0));
-                        String globalParams = curingGlobalParamsService.curingGlobalParams(processInstance.getId(),
+                        String globalParams = curingParamsService.curingGlobalParams(processInstance.getId(),
                                 processDefinition.getGlobalParamMap(),
                                 processDefinition.getGlobalParamList(),
                                 CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime(), cmdParam.get(Constants.SCHEDULE_TIMEZONE));

dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java

@@ -59,6 +59,7 @@
 import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ConnectorType;
 import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType;
 import org.apache.dolphinscheduler.plugin.task.api.model.JdbcInfo;
+import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
@@ -73,6 +74,7 @@
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 import org.apache.dolphinscheduler.spi.enums.DbType;
@@ -123,13 +125,16 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
 
     protected TaskPluginManager taskPluginManager;
 
+    protected CuringParamsService curingParamsService;
+
     protected String threadLoggerInfoName;
 
     @Override
     public void init(@NonNull TaskInstance taskInstance, @NonNull ProcessInstance processInstance) {
         processService = SpringApplicationContext.getBean(ProcessService.class);
         masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
         taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class);
+        curingParamsService = SpringApplicationContext.getBean(CuringParamsService.class);
         this.taskInstance = taskInstance;
         this.processInstance = processInstance;
         this.maxRetryTimes = masterConfig.getTaskCommitRetryTimes();
@@ -301,6 +306,10 @@ protected TaskExecutionContext getTaskExecutionContext(TaskInstance taskInstance
             setK8sTaskRelation(k8sTaskExecutionContext, taskInstance);
         }
 
+        Map<String, Property> businessParamsMap = curingParamsService.preBuildBusinessParams(processInstance);
+
+        AbstractParameters baseParam = taskPluginManager.getParameters(ParametersNode.builder().taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build());
+        Map<String, Property> propertyMap = curingParamsService.paramParsingPreparation(taskInstance, baseParam, processInstance);
         return TaskExecutionContextBuilder.get()
                 .buildTaskInstanceRelatedInfo(taskInstance)
                 .buildTaskDefinitionRelatedInfo(taskInstance.getTaskDefine())
@@ -309,6 +318,8 @@ protected TaskExecutionContext getTaskExecutionContext(TaskInstance taskInstance
                 .buildResourceParametersInfo(resources)
                 .buildDataQualityTaskExecutionContext(dataQualityTaskExecutionContext)
                 .buildK8sTaskRelatedInfo(k8sTaskExecutionContext)
+                .buildBusinessParamsMap(businessParamsMap)
+                .buildParamInfo(propertyMap)
                 .create();
     }