Skip to content

Commit c0f70b8

Browse files
SbloodySSbloodyS
authored
[DSIP-38] Support upload/download plugins from maven repo (#16654)
1 parent f56a311 commit c0f70b8

36 files changed

Lines changed: 590 additions & 146 deletions

File tree

‎.github/workflows/api-test.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,15 @@ jobs:
8383
-Dmaven.test.skip=true \
8484
-Dmaven.javadoc.skip=true \
8585
-Dspotless.skip=true \
86-
-Pdocker,release -Ddocker.tag=ci
86+
-Pdocker,staging -Ddocker.tag=ci
8787
- name: Export Docker Images
8888
run: |
8989
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
9090
&& du -sh /tmp/standalone-image.tar
9191
- uses: actions/upload-artifact@v4
9292
name: Upload Docker Images
9393
with:
94-
name: standalone-image
94+
name: standalone-image-api-test
9595
path: /tmp/standalone-image.tar
9696
retention-days: 1
9797
api-test:
@@ -140,7 +140,7 @@ jobs:
140140
- uses: actions/download-artifact@v4
141141
name: Download Docker Images
142142
with:
143-
name: standalone-image
143+
name: standalone-image-api-test
144144
path: /tmp
145145
- name: Load Docker Images
146146
run: |

‎.github/workflows/backend.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ jobs:
8989
- name: Build and Package on ${{ matrix.java }}
9090
run: |
9191
./mvnw -B clean install \
92-
-Prelease \
92+
-Pstaging \
9393
-Dmaven.test.skip=true \
9494
-Dspotless.skip=true
9595
- name: Check dependency license

‎.github/workflows/e2e-k8s.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
-Dmaven.test.skip=true \
7474
-Dmaven.javadoc.skip=true \
7575
-Dspotless.skip=true \
76-
-Pdocker,release -Ddocker.tag=ci
76+
-Pdocker,staging -Ddocker.tag=ci
7777
- name: Create k8s Kind Cluster
7878
run: |
7979
# install kubectl

‎.github/workflows/e2e.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,15 @@ jobs:
8585
-Dmaven.test.skip=true \
8686
-Dmaven.javadoc.skip=true \
8787
-Dspotless.skip=true \
88-
-Pdocker,release -Ddocker.tag=ci
88+
-Pdocker,staging -Ddocker.tag=ci
8989
- name: Export Docker Images
9090
run: |
9191
docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
9292
&& du -sh /tmp/standalone-image.tar
9393
- uses: actions/upload-artifact@v4
9494
name: Upload Docker Images
9595
with:
96-
name: standalone-image
96+
name: standalone-image-e2e
9797
path: /tmp/standalone-image.tar
9898
retention-days: 1
9999
e2e:
@@ -168,7 +168,7 @@ jobs:
168168
- uses: actions/download-artifact@v4
169169
name: Download Docker Images
170170
with:
171-
name: standalone-image
171+
name: standalone-image-e2e
172172
path: /tmp
173173
- name: Load Docker Images
174174
run: |

‎.github/workflows/publish-nexus.yaml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
name: Publish Snapshot
19+
20+
on:
21+
release:
22+
types:
23+
- released
24+
25+
jobs:
26+
publish-snapshot:
27+
if: github.repository == 'apache/dolphinscheduler'
28+
runs-on: ubuntu-latest
29+
steps:
30+
- name: Checkout repository
31+
uses: actions/checkout@v4
32+
with:
33+
submodules: true
34+
- name: Maximize runner space
35+
uses: ./.github/actions/maximize-build-space
36+
with:
37+
root-reserve-mb: 30720
38+
temp-reserve-mb: 10240
39+
remove-dotnet: 'true'
40+
remove-android: 'true'
41+
remove-haskell: 'true'
42+
remove-codeql: 'true'
43+
remove-docker-images: 'true'
44+
- uses: actions/checkout@v4
45+
- name: Cache local Maven repository
46+
uses: actions/cache@v4
47+
with:
48+
path: ~/.m2/repository
49+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
50+
restore-keys: ${{ runner.os }}-maven-
51+
- name: Setup JDK 8
52+
uses: actions/setup-java@v4
53+
with:
54+
distribution: 'adopt'
55+
java-version: 8
56+
- name: Set environment variables
57+
run: |
58+
if [[ ${{ github.event_name }} == "schedule" ]]; then
59+
echo "SETTINGS_PATH=./.github/workflows/publish-nexus/snapshot-asf-settings.xml" >> $SETTINGS_PATH
60+
elif [[ ${{ github.event_name }} == "release" ]]; then
61+
echo "SETTINGS_PATH=./.github/workflows/publish-nexus/release-asf-settings.xml" >> $SETTINGS_PATH
62+
else
63+
echo "unknown event name: ${{ github.event_name }}"
64+
exit 2
65+
fi
66+
- name: Publish snapshot with dev branch
67+
env:
68+
ASF_USERNAME: ${{ secrets.NEXUS_USER }}
69+
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
70+
run: |
71+
./mvnw clean deploy \
72+
-s ${{ env.SETTINGS_PATH }} \
73+
-Dmaven.test.skip=true \
74+
-Dspotless.skip=true \
75+
-Pstaging

‎.github/workflows/publish-nexus/release-asf-settings.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
22+
<servers>
23+
<server>
24+
<id>apache.snapshots.https</id>
25+
<username>${env.ASF_USERNAME}</username>
26+
<password>${env.ASF_PASSWORD}</password>
27+
</server>
28+
<server>
29+
<id>apache.releases.https</id>
30+
<username>${env.ASF_USERNAME}</username>
31+
<password>${env.ASF_PASSWORD}</password>
32+
</server>
33+
</servers>
34+
</settings>

‎.github/workflows/publish-nexus/snapshot-asf-settings.xml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
22+
<servers>
23+
<server>
24+
<id>apache.snapshots.https</id>
25+
<username>${env.ASF_USERNAME}</username>
26+
<password>${env.ASF_PASSWORD}</password>
27+
</server>
28+
</servers>
29+
</settings>

‎.licenserc.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ header:
4848
- 'org.mockito.plugins.MockMaker'
4949
- tools/dependencies/known-dependencies.txt
5050
- '**/banner.txt'
51+
- '**/*-banner.txt'
5152
- '.terraform.lock.hcl'
5253
- deploy/kubernetes/dolphinscheduler/README.md.gotmpl
5354
- .idea/vcs.xml

‎config/plugins_config‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
# This mapping is used to resolve the Jar package name without version (or call artifactId)
18+
#
19+
# corresponding to the module in the user Config, helping Dolphinscheduler to load the correct Jar package.
20+
# Don't modify the delimiter " -- ", just select the plugin you need
21+
22+
--alert-plugins--
23+
dolphinscheduler-alert-aliyunVoice
24+
dolphinscheduler-alert-dingtalk
25+
dolphinscheduler-alert-email
26+
dolphinscheduler-alert-feishu
27+
dolphinscheduler-alert-http
28+
dolphinscheduler-alert-pagerduty
29+
dolphinscheduler-alert-prometheus
30+
dolphinscheduler-alert-script
31+
dolphinscheduler-alert-slack
32+
dolphinscheduler-alert-telegram
33+
dolphinscheduler-alert-webexteams
34+
dolphinscheduler-alert-wechat
35+
--end--
36+
37+
--datasource-plugins--
38+
dolphinscheduler-datasource-aliyunserverlessspark
39+
dolphinscheduler-datasource-athena
40+
dolphinscheduler-datasource-azure-sql
41+
dolphinscheduler-datasource-clickhouse
42+
dolphinscheduler-datasource-dameng
43+
dolphinscheduler-datasource-databend
44+
dolphinscheduler-datasource-db2
45+
dolphinscheduler-datasource-doris
46+
dolphinscheduler-datasource-hana
47+
dolphinscheduler-datasource-hive
48+
dolphinscheduler-datasource-k8s
49+
dolphinscheduler-datasource-kyuubi
50+
dolphinscheduler-datasource-mysql
51+
dolphinscheduler-datasource-oceanbase
52+
dolphinscheduler-datasource-oracle
53+
dolphinscheduler-datasource-postgresql
54+
dolphinscheduler-datasource-presto
55+
dolphinscheduler-datasource-redshift
56+
dolphinscheduler-datasource-sagemaker
57+
dolphinscheduler-datasource-snowflake
58+
dolphinscheduler-datasource-spark
59+
dolphinscheduler-datasource-sqlserver
60+
dolphinscheduler-datasource-ssh
61+
dolphinscheduler-datasource-starrocks
62+
dolphinscheduler-datasource-trino
63+
dolphinscheduler-datasource-vertica
64+
dolphinscheduler-datasource-zeppelin
65+
--end--
66+
67+
--storage-plugins--
68+
dolphinscheduler-storage-abs
69+
dolphinscheduler-storage-gcs
70+
dolphinscheduler-storage-hdfs
71+
dolphinscheduler-storage-obs
72+
dolphinscheduler-storage-oss
73+
dolphinscheduler-storage-s3
74+
--end--
75+
76+
--task-plugins--
77+
dolphinscheduler-task-aliyunserverlessspark
78+
dolphinscheduler-task-chunjun
79+
dolphinscheduler-task-datafactory
80+
dolphinscheduler-task-dataquality
81+
dolphinscheduler-task-datasync
82+
dolphinscheduler-task-datax
83+
dolphinscheduler-task-dinky
84+
dolphinscheduler-task-dms
85+
dolphinscheduler-task-dvc
86+
dolphinscheduler-task-emr
87+
dolphinscheduler-task-flink
88+
dolphinscheduler-task-flink-stream
89+
dolphinscheduler-task-hivecli
90+
dolphinscheduler-task-http
91+
dolphinscheduler-task-java
92+
dolphinscheduler-task-jupyter
93+
dolphinscheduler-task-k8s
94+
dolphinscheduler-task-kubeflow
95+
dolphinscheduler-task-linkis
96+
dolphinscheduler-task-mlflow
97+
dolphinscheduler-task-mr
98+
dolphinscheduler-task-openmldb
99+
dolphinscheduler-task-procedure
100+
dolphinscheduler-task-python
101+
dolphinscheduler-task-pytorch
102+
dolphinscheduler-task-remoteshell
103+
dolphinscheduler-task-sagemaker
104+
dolphinscheduler-task-seatunnel
105+
dolphinscheduler-task-shell
106+
dolphinscheduler-task-spark
107+
dolphinscheduler-task-sql
108+
dolphinscheduler-task-sqoop
109+
dolphinscheduler-task-zeppelin
110+
--end--

‎docs/docs/en/guide/installation/pseudo-cluster.md‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,27 @@ Pseudo-cluster deployment of DolphinScheduler requires external software support
1616
- `pstree` for macOS
1717
- `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
1818

19-
> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
19+
## Download Plugins Dependencies
20+
21+
Starting from version 3.3.0, the binary package no longer provides plugin dependencies, and users need to download them by themselves. The plugin dependency package download address: [Plugin Dependency Package](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
22+
You can also execute the following command to install plugin dependencies:
23+
24+
```shell
25+
bash ./bin/install-plugins.sh 3.3.0
26+
```
27+
28+
Usually, you do not need all connector plugins, you can specify the plugins you need by configuring `conf/plugins_config`. For example, if you only need the `dolphinscheduler-task-shell` plugin, you can modify the configuration file as follows:
29+
30+
```
31+
--task-plugins--
32+
dolphinscheduler-task-shell
33+
--end--
34+
```
2035

2136
## DolphinScheduler Startup Environment
2237

38+
> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
39+
2340
### Configure User Exemption and Permissions
2441

2542
Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:

0 commit comments

Comments
 (0)