Merge pull request #1102 from WeBankFinTech/dev-1.1.2

Dev 1.1.2

Author: zqburde

Diff for: .github/workflows/dlc.json

@@ -2,6 +2,12 @@
   "ignorePatterns": [
     {
       "pattern": "https://github.com/WeBankFinTech/DataSphereStudio-Doc"
+    },
+    {
+      "pattern": "http://localhost"
+    },
+    {
+      "pattern": "http://127.0.0.1"
     }
   ],
   "timeout": "10s",
@@ -13,5 +19,4 @@
     401,
     403
   ]
-}
-
+}

Diff for: .gitignore

@@ -1,33 +1,115 @@
+.idea
+.idea/*
+
 .DS_Store
-.cache
 
-# for ide
+assembly/target
+assembly/dss-package/target
+
+
+# dss-commons
+dss-commons/dss-sender-service/target
+
+# dss-appconn
+dss-appconn/appconns/dss-datachecker-appconn/target
+dss-appconn/appconns/dss-eventchecker-appconn/target
+dss-appconn/appconns/dss-orchestrator-framework-appconn/target
+dss-appconn/appconns/dss-schedulis-appconn/target
+dss-appconn/appconns/dss-sendemail-appconn/esb-email-support/target
+dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/target
+dss-appconn/appconns/dss-workflow-appconn/target
+dss-appconn/appconns/dss-sso-appconn/target
+dss-appconn/appconns/dss-scriptis-appconn-appconn/target
+dss-appconn/dss-appconn-core/target
+dss-appconn/dss-appconn-loader/target
+dss-appconn/dss-schedule-appconn-core/target
+dss-appconn/linkis-appconn-engineplugin/target
+dss-appconn/appconns/dss-dolphinscheduler-appconn/target
+dss-appconn/dss-appconn-manager/dss-appconn-manager-client/target
+dss-appconn/dss-appconn-manager/dss-appconn-manager-core/target
+dss-appconn/dss-scheduler-appconn/target
+dss-appconn/appconns/dss-scriptis-appconn/target/
+
+# dss-apps
+dss-apps/dss-apiservice-server/target
+dss-apps/dss-scriptis-server/target
+dss-apps/dss-apps-server/target
+dss-server/target
+
+#dss-guide
+dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/
+dss-apps/dss-user-guide/dss-user-guide-server/target
+
+# dss-commons
+dss-commons/dss-common/target
+dss-commons/dss-contextservice/target
+dss-commons/dss-bmlservice/target
+
+# dss-framework
+dss-framework/dss-appconn-framework/target
+dss-framework/dss-framework-common/target
+dss-framework/dss-framework-orchestrator-server/target
+dss-framework/dss-framework-project-server/target
+dss-framework/dss-framework-workspace-server/target
+dss-framework/framework-plugins/dss-framework-orchestrator-publish/target
+dss-framework/framework-plugins/dss-framework-release-server/target
+dss-framework/framework-plugins/dss-framework-migrate-server/target
+dss-framework/dss-framework-admin/target
+dss-framework/dss-framework-sql-template/target
+dss-framework/dss-framework-dbapi-server/target
+dss-framework/dss-framework-admin-service/target
+dss-framework/dss-framework-proxy-user-service/target
+
+
+# dss-orchestrator
+dss-orchestrator/dss-orchestrator-common/target
+dss-orchestrator/dss-orchestrator-core/target
+dss-orchestrator/dss-orchestrator-db/target
+dss-orchestrator/dss-orchestrator-loader/target
+dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/target
+dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/target
+dss-orchestrator/dss-orchestrator-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/target
+
+# dss-standars
+dss-standard/dss-standard-common/target
+dss-standard/sso-standard/origin-sso-integration-standard/target
+dss-standard/sso-standard/spring-origin-sso-integration-plugin/target
+dss-standard/sso-standard/sso-integration-standard/target
+dss-standard/development-standard/development-process-standard/target
+dss-standard/development-standard/development-process-standard-execution/target
+dss-standard/structure-standard/dss-project-plugin/target
+dss-standard/structure-standard/dss-role-plugin/target
+dss-standard/structure-standard/dss-structure-integration-standard/target
+dss-standard/structure-standard/spring-origin-dss-project-plugin/target
+
+#dss-data-api
+dss-apps/dss-data-api/dss-data-api-server/src/main/resources
+dss-apps/dss-data-api/dss-api-sql-template/target
+dss-apps/dss-data-api/dss-data-api-server/target
+
+#dss-data-governance
+dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources
+dss-apps/dss-data-governance/dss-data-governance-server/target
+dss-apps/dss-data-governance/dss-data-governance-common/target
+dss-apps/dss-data-governance/dss-data-asset-server/target
+dss-apps/dss-data-governance/dss-data-classification-server/target
+dss-apps/dss-data-governance/dss-data-warehouse-dao/target
+dss-apps/dss-data-governance/dss-data-warehouse-service/target
+dss-apps/dss-data-governance/dss-data-warehouse-server/target
+
+
+# plugins
+plugins/azkaban/linkis-jobtype/target
+plugins/linkis/dss-gateway-support/target
+
+logs
+logs/*
+
 *.iml
-*.ipr
-*.iws
-*.pyc
-*.pyo
-*.swp
-.idea/
-.idea_modules/
-.project
-.pydevproject
-.scala_dependencies
-.settings
-.classpath
-
-# For SBT
-.jvmopts
-
-# For Node.js
-node_modules/
-
-# generated file
-.mvn/wrapper/maven-wrapper.jar
-dist/
-out/
-target/
-
-# log folder
-logs/
-*.log
+/plugins/dolphinscheduler/dss-dolphinscheduler-client/target
+/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/target
+/plugins/dolphinscheduler/dss-dolphinscheduler-token/target

Diff for: assembly/bin/appconn-install.sh

@@ -26,7 +26,7 @@ function isSuccess(){
   fi
 }
 
-PROC_NAME=DSSProjectServerApplication
+PROC_NAME=DSSServerApplication
 ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l`
 if [ $ProcNumber -le 0 ];then
    echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started."

Diff for: assembly/bin/appconn-refresh.sh

@@ -11,6 +11,17 @@ if [ -z $SOURCE_ROOT ]; then
   source ${SOURCE_ROOT}/conf/db.sh
 fi
 
+function refresh() {
+  while true; do
+    response=$(curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load)
+    if [[ $response == *"succeed"* ]]; then
+      break
+    else
+      sleep 5
+    fi
+  done
+}
+
 function isSuccess(){
   if [ $? -ne 0 ]; then
       echo "Failed to " + $1
@@ -39,18 +50,18 @@ if [ -z $1 ];then
     sh $SOURCE_ROOT/sbin/dss-start-all.sh
   else
     echo "You chose to wait for 5 minutes."
-    echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
-    curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
-    isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
+    echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
+    refresh
+    isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
     echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
     echo ""
     exit 0
   fi
 else
   APPCONN_NAME=$1
-  echo "Now try to call dss-framework-project to reload the plugin of $APPCONN_NAME AppConn."
-  curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
-  isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-framework-project."
+  echo "Now try to call dss-server to reload the plugin of $APPCONN_NAME AppConn."
+  refresh
+  isSuccess "reload the plugin of $APPCONN_NAME AppConn in dss-server."
   echo "Now please wait for 5 minutes, then all of the DSS micro-services will refresh the ${APPCONN_NAME} AppConn plugin."
   echo ""
   exit 0

Diff for: assembly/bin/appconn-uninstall.sh

@@ -0,0 +1,107 @@
+#!/bin/sh
+#Actively load user env
+source ~/.bashrc
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+SOURCE_ROOT=${workDir}
+
+#load config
+source ${SOURCE_ROOT}/conf/config.sh
+source ${SOURCE_ROOT}/conf/db.sh
+
+APPCONN_NAME=''
+
+LOCAL_IP="`ifconfig | grep 'inet' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $2}'`"
+
+function isSuccess(){
+  if [ $? -ne 0 ]; then
+      echo "Failed to " $1
+      exit 1
+  else
+      echo "Succeed to " $1
+  fi
+}
+
+
+PROC_NAME=DSSProjectServerApplication
+ProcNumber=`ps -ef | grep -w $PROC_NAME | grep -v grep | wc -l`
+if [ $ProcNumber -le 0 ];then
+   echo "${PROC_NAME} is not running, please ensure whether DSS is installed and started."
+   exit 10
+else
+   echo "Begin to uninstall AppConn plugin..."
+fi
+
+
+function getUninstallAppConn() {
+  echo "Please input the name of uninstallation AppConn, e.g: schedulis."
+  read -p "Please input the AppConn name:"  idx
+  if [[ 'exit' = "$idx" ]];then
+    echo "exit!"
+    exit 1
+  else
+    APPCONN_NAME=$idx
+  fi
+  echo "Current uninstallation AppConn is ${APPCONN_NAME}"
+  echo ""
+  echo -e "\e[1;31m Are you sure you want to uninstall AppConn ${APPCONN_NAME}, the workflow associated with the AppConn will not be available after uninstallation.\e[0m"
+  echo -e "\e[1;31mIf you want to uninstall AppConn ${APPCONN_NAME}, please enter 1, otherwise enter 0.\e[0m"
+  echo ""
+  read -p "Please input your choice:" idx
+  
+  if [[ '0' = "$idx" ]]; then
+    echo "exit!"
+    exit 1
+  fi
+}
+
+##choose execute mysql mode
+function executeSQL() {
+  TEMP_DB_DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}/db
+  DB_DML_PATH=$TEMP_DB_DML_PATH/uninstall.sql
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
+  isSuccess "complete the cleanup of the database $DB_DML_PATH"
+  echo "Implement the $TEMP_DB_DML_PATH/uninstall.sql for $APPCONN_NAME succeed."
+}
+
+
+function deleteDML() {
+  echo ""
+  echo -e "\e[1;31m If you want to delete AppConn ${APPCONN_NAME} file, please enter 1, otherwise enter 0.\e[0m"
+  echo ""
+  read -p "Please input your choice:" idx
+  if [[ '1' = "$idx" ]];then
+    DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}
+    suffix=.zip
+    rm -rf $DML_PATH$suffix
+    rm -rf $DML_PATH
+    isSuccess "complete the cleanup of the ${APPCONN_NAME} file."
+  fi
+}
+
+
+echo ""
+echo "Step1: Get the uninstall AppConn name."
+getUninstallAppConn
+echo ""
+
+echo "Step2: Delete AppConn $APPCONN_NAME database info."
+executeSQL
+echo ""
+
+echo "Step3: Clear the plugin of $APPCONN_NAME AppConn in DSS."
+deleteDML
+echo ""
+echo "Now try to delete the plugin of ${APPCONN_NAME} AppConn in all DSS micro-services."
+echo "The following 2 ways can take effect:"
+echo "1. Restart DSS, we will use sh $SOURCE_ROOT/sbin/dss-start-all.sh to restart, it will spend 1 minute."
+echo "2. Do nothing, just wait for 5 minutes. Since the DSS micro-services will refresh all the AppConn plugins every 10 minutes."
+echo ""
+read -p "Please input the choise: "  choise
+if [[ '1' = "$choise" ]]; then
+  echo "You chose to restart dss-framework-project micro-services, now try to restart ..."
+  sh $SOURCE_ROOT/sbin/dss-start-all.sh
+else
+  echo "You chose to wait for 5 minutes."
+fi