Skip to content

Commit 1e3ae7b

Browse files
RadeityRadeity
authored
[improv][Resource Center] Implement getResourceFileName in StorageOperator (#14097)
1 parent 8944fdc commit 1e3ae7b

12 files changed

Lines changed: 66 additions & 69 deletions

File tree

‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1534,7 +1534,7 @@ public DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer
15341534

15351535
String tenantCode = getTenantCode(user);
15361536

1537-
String baseFolder = storageOperate.getResourceFileName(tenantCode, "DATA_TRANSFER");
1537+
String baseFolder = storageOperate.getResourceFullName(tenantCode, "DATA_TRANSFER");
15381538

15391539
LocalDateTime now = LocalDateTime.now();
15401540
now = now.minus(days, ChronoUnit.DAYS);

‎dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExec
178178
public static String getValCheckSum(Property fileProperty, TaskExecutionContext context,
179179
StorageOperate storageOperate) {
180180
String resourceCRCPath = fileProperty.getValue() + CRC_SUFFIX;
181-
String resourceCRCWholePath = storageOperate.getResourceFileName(context.getTenantCode(), resourceCRCPath);
181+
String resourceCRCWholePath = storageOperate.getResourceFullName(context.getTenantCode(), resourceCRCPath);
182182
String targetPath = String.format("%s/%s", context.getExecutePath(), resourceCRCPath);
183183
log.info("{} --- Remote:{} to Local:{}", "CRC file", resourceCRCWholePath, targetPath);
184184
String crcString = "";

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,19 +60,19 @@ public interface StorageOperate {
6060
boolean mkdir(String tenantCode, String path) throws IOException;
6161

6262
/**
63-
* get the path of the resource file
63+
* get the path of the resource file (fullName)
6464
* @param tenantCode
65-
* @param fullName
65+
* @param fileName
6666
* @return
6767
*/
68-
String getResourceFileName(String tenantCode, String fullName);
68+
String getResourceFullName(String tenantCode, String fileName);
6969

7070
/**
71-
* get the path of the resource file excluding the base path.
71+
* get the path of the resource file excluding the base path (fileName)
7272
* @param fullName
7373
* @return
7474
*/
75-
String getResourceFileName(String fullName);
75+
String getResourceFileName(String tenantCode, String fullName);
7676

7777
/**
7878
* get the path of the file

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,16 +122,17 @@ public String getUdfDir(String tenantCode) {
122122
}
123123

124124
@Override
125-
public String getResourceFileName(String tenantCode, String fileName) {
125+
public String getResourceFullName(String tenantCode, String fileName) {
126126
if (fileName.startsWith(FOLDER_SEPARATOR)) {
127127
fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING);
128128
}
129129
return String.format(FORMAT_S_S, getGcsResDir(tenantCode), fileName);
130130
}
131131

132132
@Override
133-
public String getResourceFileName(String fullName) {
134-
return null;
133+
public String getResourceFileName(String tenantCode, String fullName) {
134+
String resDir = getResDir(tenantCode);
135+
return fullName.replaceFirst(resDir, "");
135136
}
136137

137138
@Override

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,11 +146,20 @@ public void mkdirWhenDirNotExists() {
146146
Assertions.assertTrue(isSuccess);
147147
}
148148

149+
@Test
150+
public void getResourceFullName() {
151+
final String expectedResourceFullName =
152+
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
153+
final String resourceFullName = gcsStorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
154+
Assertions.assertEquals(expectedResourceFullName, resourceFullName);
155+
}
156+
149157
@Test
150158
public void getResourceFileName() {
151-
final String expectedResourceFileName =
159+
final String expectedResourceFileName = FILE_NAME_MOCK;
160+
final String resourceFullName =
152161
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
153-
final String resourceFileName = gcsStorageOperator.getResourceFileName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
162+
final String resourceFileName = gcsStorageOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName);
154163
Assertions.assertEquals(expectedResourceFileName, resourceFileName);
155164
}
156165

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java‎

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@
5656
import java.nio.file.Files;
5757
import java.security.PrivilegedExceptionAction;
5858
import java.util.ArrayList;
59-
import java.util.Arrays;
6059
import java.util.Collections;
6160
import java.util.Date;
6261
import java.util.LinkedList;
@@ -69,7 +68,6 @@
6968
import lombok.extern.slf4j.Slf4j;
7069

7170
import com.fasterxml.jackson.databind.node.ObjectNode;
72-
import com.google.common.base.Joiner;
7371
import com.google.common.cache.CacheBuilder;
7472
import com.google.common.cache.CacheLoader;
7573
import com.google.common.cache.LoadingCache;
@@ -298,28 +296,14 @@ public boolean mkdir(String tenantCode, String hdfsPath) throws IOException {
298296
}
299297

300298
@Override
301-
public String getResourceFileName(String tenantCode, String fullName) {
299+
public String getResourceFullName(String tenantCode, String fullName) {
302300
return getHdfsResourceFileName(tenantCode, fullName);
303301
}
304302

305303
@Override
306-
public String getResourceFileName(String fullName) {
307-
// here is a quick fix here to get fileName. We get the resource upload path and
308-
// get the index of the first appearance of resource upload path. The index is put
309-
// in the start index of the substring function and get the result substring containing
310-
// tenantcode and "resource" directory and the fileName.
311-
// Then we split the result substring
312-
// with "/" and join all elements except the first two elements because they are
313-
// tenantCode and "resource" directory.
314-
String resourceUploadPath =
315-
RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
316-
: RESOURCE_UPLOAD_PATH;
317-
// +1 because we want to skip the "/" after resource upload path as well.
318-
String pathContainingTenantNResource = fullName.substring(
319-
fullName.indexOf(resourceUploadPath)
320-
+ resourceUploadPath.length() + 1);
321-
String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
322-
return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
304+
public String getResourceFileName(String tenantCode, String fullName) {
305+
String resDir = getResDir(tenantCode);
306+
return fullName.replaceFirst(resDir, "");
323307
}
324308

325309
@Override

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,16 +175,17 @@ protected void createOssPrefix(final String bucketName, final String key) {
175175
}
176176

177177
@Override
178-
public String getResourceFileName(String tenantCode, String fileName) {
178+
public String getResourceFullName(String tenantCode, String fileName) {
179179
if (fileName.startsWith(FOLDER_SEPARATOR)) {
180180
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, "");
181181
}
182182
return String.format(FORMAT_S_S, getOssResDir(tenantCode), fileName);
183183
}
184184

185185
@Override
186-
public String getResourceFileName(String fullName) {
187-
return null;
186+
public String getResourceFileName(String tenantCode, String fullName) {
187+
String resDir = getResDir(tenantCode);
188+
return fullName.replaceFirst(resDir, "");
188189
}
189190

190191
@Override

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,11 +158,20 @@ public void mkdirWhenDirNotExists() {
158158
Assertions.assertTrue(isSuccess);
159159
}
160160

161+
@Test
162+
public void getResourceFullName() {
163+
final String expectedResourceFullName =
164+
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
165+
final String resourceFullName = ossOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
166+
Assertions.assertEquals(expectedResourceFullName, resourceFullName);
167+
}
168+
161169
@Test
162170
public void getResourceFileName() {
163-
final String expectedResourceFileName =
171+
final String expectedResourceFileName = FILE_NAME_MOCK;
172+
final String resourceFullName =
164173
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
165-
final String resourceFileName = ossOperator.getResourceFileName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
174+
final String resourceFileName = ossOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName);
166175
Assertions.assertEquals(expectedResourceFileName, resourceFileName);
167176
}
168177

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java‎

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
import java.nio.file.Path;
4747
import java.nio.file.Paths;
4848
import java.util.ArrayList;
49-
import java.util.Arrays;
5049
import java.util.Collections;
5150
import java.util.LinkedList;
5251
import java.util.List;
@@ -76,7 +75,6 @@
7675
import com.amazonaws.services.s3.transfer.MultipleFileDownload;
7776
import com.amazonaws.services.s3.transfer.TransferManager;
7877
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
79-
import com.google.common.base.Joiner;
8078

8179
@Slf4j
8280
@Data
@@ -182,31 +180,17 @@ public boolean mkdir(String tenantCode, String path) throws IOException {
182180
}
183181

184182
@Override
185-
public String getResourceFileName(String tenantCode, String fileName) {
183+
public String getResourceFullName(String tenantCode, String fileName) {
186184
if (fileName.startsWith(FOLDER_SEPARATOR)) {
187185
fileName = fileName.replaceFirst(FOLDER_SEPARATOR, "");
188186
}
189187
return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName);
190188
}
191189

192190
@Override
193-
public String getResourceFileName(String fullName) {
194-
// here is a quick fix here to get fileName. We get the resource upload path and
195-
// get the index of the first appearance of resource upload path. The index is put
196-
// in the start index of the substring function and get the result substring containing
197-
// tenantcode and "resource" directory and the fileName.
198-
// Then we split the result substring
199-
// with "/" and join all elements except the first two elements because they are
200-
// tenantCode and "resource" directory.
201-
String resourceUploadPath =
202-
RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
203-
: RESOURCE_UPLOAD_PATH;
204-
// +1 because we want to skip the "/" after resource upload path as well.
205-
String pathContainingTenantNResource = fullName.substring(
206-
fullName.indexOf(resourceUploadPath)
207-
+ resourceUploadPath.length() + 1);
208-
String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
209-
return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
191+
public String getResourceFileName(String tenantCode, String fullName) {
192+
String resDir = getResDir(tenantCode);
193+
return fullName.replaceFirst(resDir, "");
210194
}
211195

212196
@Override

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,20 @@ public void mkdirWhenDirNotExists() {
164164
Assertions.assertTrue(isSuccess);
165165
}
166166

167+
@Test
168+
public void getResourceFullName() {
169+
final String expectedResourceFullName =
170+
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
171+
final String resourceFullName = s3StorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
172+
Assertions.assertEquals(expectedResourceFullName, resourceFullName);
173+
}
174+
167175
@Test
168176
public void getResourceFileName() {
169-
final String expectedResourceFileName =
177+
final String expectedResourceFileName = FILE_NAME_MOCK;
178+
final String resourceFullName =
170179
String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK);
171-
final String resourceFileName = s3StorageOperator.getResourceFileName(TENANT_CODE_MOCK, FILE_NAME_MOCK);
180+
final String resourceFileName = s3StorageOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName);
172181
Assertions.assertEquals(expectedResourceFileName, resourceFileName);
173182
}
174183

0 commit comments

Comments
 (0)