优化微软 OneDrive、SharePoint 相关存储的代码, 更好的输出日志和重构代码。

This commit is contained in:
zhaojun
2022-11-26 18:00:16 +08:00
parent 33c751ab33
commit 38161f96e1
2 changed files with 149 additions and 107 deletions

View File

@@ -2,22 +2,23 @@ package im.zhaojun.zfile.module.storage.service.base;
import cn.hutool.core.util.StrUtil;
import cn.hutool.core.util.URLUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.ContentType;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import im.zhaojun.zfile.module.storage.constant.StorageConfigConstant;
import im.zhaojun.zfile.module.storage.model.dto.OAuth2TokenDTO;
import im.zhaojun.zfile.module.storage.model.entity.StorageSourceConfig;
import im.zhaojun.zfile.module.storage.model.param.MicrosoftDriveParam;
import im.zhaojun.zfile.module.storage.service.StorageSourceConfigService;
import im.zhaojun.zfile.module.storage.model.bo.RefreshTokenCacheBO;
import im.zhaojun.zfile.core.constant.ZFileConstant;
import im.zhaojun.zfile.core.exception.StorageSourceRefreshTokenException;
import im.zhaojun.zfile.core.exception.ZFileRetryException;
import im.zhaojun.zfile.core.util.StringUtils;
import im.zhaojun.zfile.module.storage.constant.StorageConfigConstant;
import im.zhaojun.zfile.module.storage.model.bo.RefreshTokenCacheBO;
import im.zhaojun.zfile.module.storage.model.dto.OAuth2TokenDTO;
import im.zhaojun.zfile.module.storage.model.entity.StorageSourceConfig;
import im.zhaojun.zfile.module.storage.model.enums.FileTypeEnum;
import im.zhaojun.zfile.module.storage.model.param.MicrosoftDriveParam;
import im.zhaojun.zfile.module.storage.model.result.FileItemResult;
import im.zhaojun.zfile.module.storage.service.StorageSourceConfigService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.http.HttpEntity;
@@ -25,6 +26,8 @@ import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
@@ -80,7 +83,18 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
* OneDrive 文件类型
*/
private static final String ONE_DRIVE_FILE_FLAG = "file";
/**
* 访问令牌字段名称
*/
public static final String ACCESS_TOKEN_FIELD_NAME = "access_token";
/**
* 刷新令牌字段名称
*/
public static final String REFRESH_TOKEN_FIELD_NAME = "refresh_token";
@Resource
@Lazy
private RestTemplate oneDriveRestTemplate;
@@ -93,7 +107,7 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
*
* @return 刷新后的 Token
*/
public OAuth2TokenDTO getRefreshToken() {
public OAuth2TokenDTO getAndRefreshToken() {
StorageSourceConfig refreshStorageSourceConfig =
storageSourceConfigService.findByStorageIdAndName(storageId, StorageConfigConstant.REFRESH_TOKEN_KEY);
@@ -103,33 +117,27 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
"&refresh_token=" + refreshStorageSourceConfig.getValue() +
"&grant_type=refresh_token";
log.info("存储源 {}({}) 尝试刷新令牌", storageId, this.getStorageTypeEnum().getDescription());
if (log.isDebugEnabled()) {
log.debug("存储源 {}({}) 尝试刷新令牌, 参数信息为: {}", storageId, this.getStorageTypeEnum().getDescription(), param);
log.debug("{} 尝试刷新令牌, 请求参数: [{}]", getStorageSimpleInfo(), param);
}
String fullAuthenticateUrl = AUTHENTICATE_URL.replace("{authenticateEndPoint}", getAuthenticateEndPoint());
HttpRequest post = HttpUtil.createPost(fullAuthenticateUrl);
post.body(param, "application/x-www-form-urlencoded");
HttpResponse response = post.execute();
String authenticateUrl = AUTHENTICATE_URL.replace("{authenticateEndPoint}", getAuthenticateEndPoint());
HttpResponse response = HttpUtil.createPost(authenticateUrl)
.body(param, ContentType.FORM_URLENCODED.getValue())
.execute();
String responseBody = response.body();
log.info("存储源 {}({}) 刷新令牌完成, 响应信息为: httpStatus: {}", storageId, this.getStorageTypeEnum().getDescription(), response.getStatus());
int responseStatus = response.getStatus();
if (log.isDebugEnabled()) {
log.debug("存储源 {}({}) 刷新令牌完成, 响应信息为: {}", storageId, this.getStorageTypeEnum().getDescription(), responseBody);
}
log.info("{} 刷新令牌完成. [httpStatus: {}]", getStorageSimpleInfo(), responseStatus);
JSONObject jsonBody = JSONObject.parseObject(responseBody);
if (response.getStatus() != HttpStatus.OK.value()) {
if (responseStatus != HttpStatus.OK.value()) {
return OAuth2TokenDTO.fail(getClientId(), getClientSecret(), getRedirectUri(), responseBody);
}
String accessToken = jsonBody.getString("access_token");
String refreshToken = jsonBody.getString("refresh_token");
JSONObject jsonBody = JSONObject.parseObject(responseBody);
String accessToken = jsonBody.getString(ACCESS_TOKEN_FIELD_NAME);
String refreshToken = jsonBody.getString(REFRESH_TOKEN_FIELD_NAME);
return OAuth2TokenDTO.success(getClientId(), getClientSecret(), getRedirectUri(), accessToken, refreshToken, responseBody);
}
@@ -149,86 +157,79 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
"&scope=" + getScope() +
"&grant_type=authorization_code";
log.info("根据授权回调 code 获取存储源类型为 {} 的令牌, code: {}", this.getStorageTypeEnum().getDescription(), code);
if (log.isDebugEnabled()) {
log.debug("根据授权回调 code 获取存储源类型为 {} 的令牌, 参数信息为: {}", this.getStorageTypeEnum().getDescription(), param);
log.debug("根据授权回调 code 获取存储源类型为 [{}] 的令牌, 请求参数: [{}]", this.getStorageTypeEnum().getDescription(), param);
}
String fullAuthenticateUrl = AUTHENTICATE_URL.replace("{authenticateEndPoint}", getAuthenticateEndPoint());
HttpRequest post = HttpUtil.createPost(fullAuthenticateUrl);
HttpResponse response = HttpUtil.createPost(fullAuthenticateUrl)
.body(param, "application/x-www-form-urlencoded")
.execute();
post.body(param, "application/x-www-form-urlencoded");
HttpResponse response = post.execute();
String responseBody = response.body();
int responseStatus = response.getStatus();
log.info("根据授权回调 code 获取存储源类型为 [{}] 的令牌完成. [httpStatus: {}]", this.getStorageTypeEnum().getDescription(), responseStatus);
log.info("根据授权回调 code 获取存储源类型为 {} 的令牌完成, 响应信息为: httpStatus: {}", this.getStorageTypeEnum().getDescription(), response.getStatus());
if (log.isDebugEnabled()) {
log.debug("根据授权回调 code 获取存储源类型为 {} 的令牌完成, 响应信息为: {}", this.getStorageTypeEnum().getDescription(), responseBody);
}
JSONObject jsonBody = JSONObject.parseObject(responseBody);
if (response.getStatus() != HttpStatus.OK.value()) {
if (responseStatus != HttpStatus.OK.value()) {
return OAuth2TokenDTO.fail(clientId, clientSecret, redirectUri, responseBody);
}
String accessToken = jsonBody.getString("access_token");
String refreshToken = jsonBody.getString("refresh_token");
JSONObject jsonBody = JSONObject.parseObject(responseBody);
String accessToken = jsonBody.getString(ACCESS_TOKEN_FIELD_NAME);
String refreshToken = jsonBody.getString(REFRESH_TOKEN_FIELD_NAME);
return OAuth2TokenDTO.success(clientId, clientSecret, redirectUri, accessToken, refreshToken, responseBody);
}
@Override
public List<FileItemResult> fileList(String folderPath) {
folderPath = StringUtils.trimStartSlashes(folderPath);
String fullPath = StringUtils.concat(param.getBasePath(), folderPath);
String fullPath = StringUtils.concatTrimEndSlashes(param.getBasePath(), folderPath);
List<FileItemResult> result = new ArrayList<>();
String nextLink = null;
String nextPageLink = null;
do {
String requestUrl;
if (nextLink != null) {
nextLink = nextLink.replace("+", "%2B");
requestUrl = URLUtil.decode(nextLink);
}else if (ZFileConstant.PATH_SEPARATOR.equalsIgnoreCase(fullPath) || "".equalsIgnoreCase(fullPath)) {
// 如果有下一页链接,则优先取下一页
// 如果没有则判断是根目录还是子目录
if (nextPageLink != null) {
nextPageLink = nextPageLink.replace("+", "%2B");
requestUrl = URLUtil.decode(nextPageLink);
} else if (ZFileConstant.PATH_SEPARATOR.equalsIgnoreCase(fullPath) || "".equalsIgnoreCase(fullPath)) {
requestUrl = DRIVER_ROOT_URL;
} else {
requestUrl = DRIVER_ITEMS_URL;
}
fullPath = StringUtils.trimEndSlashes(fullPath);
JSONObject root;
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
HttpEntity<Object> entity = new HttpEntity<>(headers);
try {
root = oneDriveRestTemplate.exchange(requestUrl, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
} catch (HttpClientErrorException e) {
log.error("调用 OneDrive 时出现了网络异常, 响应信息: {}, 已尝试重新刷新 token 后再试.", e.getResponseBodyAsString(), e);
refreshAccessToken();
root = oneDriveRestTemplate.exchange(requestUrl, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
}
JSONObject root = executeRetryableRequest(context -> {
int retryCount = context.getRetryCount();
if (retryCount > 0) {
HttpClientErrorException ex = (HttpClientErrorException) context.getLastThrowable();
log.warn("{} 调用 OneDrive 列表时出现了网络异常, 响应信息: [{}], 将尝试重新刷新 token 后再试. 文件路径为: [{}]",
getStorageSimpleInfo(), ex.getResponseBodyAsString(), fullPath, ex);
refreshAccessToken();
}
HttpEntity<Object> entity = getAuthorizationHttpEntity();
return oneDriveRestTemplate.exchange(requestUrl, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
});
if (root == null) {
return Collections.emptyList();
}
nextLink = root.getString("@odata.nextLink");
JSONArray fileList = root.getJSONArray("value");
for (int i = 0; i < fileList.size(); i++) {
JSONObject fileItem = fileList.getJSONObject(i);
FileItemResult fileItemResult =jsonToFileItem(fileItem, folderPath);
FileItemResult fileItemResult = jsonToFileItem(fileItem, folderPath);
result.add(fileItemResult);
}
} while (nextLink != null);
nextPageLink = root.getString("@odata.nextLink");
} while (nextPageLink != null);
return result;
}
@@ -236,19 +237,19 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
public FileItemResult getFileItem(String pathAndName) {
String fullPath = StringUtils.concat(param.getBasePath(), pathAndName);
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
HttpEntity<Object> entity = new HttpEntity<>(headers);
JSONObject fileItem;
try {
fileItem = oneDriveRestTemplate.exchange(DRIVER_ITEM_URL, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
} catch (HttpClientErrorException e) {
log.error("调用 OneDrive 时出现了网络异常, 响应信息: {}, 已尝试重新刷新 token 后再试.", e.getResponseBodyAsString(), e);
refreshAccessToken();
fileItem = oneDriveRestTemplate.exchange(DRIVER_ITEM_URL, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
}
JSONObject fileItem = executeRetryableRequest(context -> {
int retryCount = context.getRetryCount();
if (retryCount > 0) {
refreshAccessToken();
HttpClientErrorException ex = (HttpClientErrorException) context.getLastThrowable();
log.warn("{} 调用 OneDrive 获取文件信息时出现了网络异常, 响应信息: [{}], 将尝试重新刷新 token 后再试. 获取文件路径为: {}",
getStorageSimpleInfo(), ex.getResponseBodyAsString(), fullPath, ex);
}
HttpEntity<Object> entity = getAuthorizationHttpEntity();
return oneDriveRestTemplate.exchange(DRIVER_ITEM_URL, HttpMethod.GET, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath).getBody();
});
if (fileItem == null) {
return null;
@@ -262,7 +263,7 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
@Override
public boolean newFolder(String path, String name) {
path = StringUtils.trimStartSlashes(path);
String fullPath = StringUtils.concat(param.getBasePath(), path);
String fullPath = StringUtils.concatTrimEndSlashes(param.getBasePath(), path);
String requestUrl;
@@ -272,16 +273,12 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
requestUrl = DRIVER_ITEMS_URL;
}
fullPath = StringUtils.trimEndSlashes(fullPath);
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
HashMap<Object, Object> data = new HashMap<>();
data.put("name", name);
data.put("folder", new HashMap<>());
data.put("@microsoft.graph.conflictBehavior", "replace");
HttpEntity<Object> entity = new HttpEntity<>(data, headers);
HttpEntity<HashMap<Object, Object>> entity = getAuthorizationHttpEntity(data);
ResponseEntity<JSONObject> responseEntity = oneDriveRestTemplate.exchange(requestUrl, HttpMethod.POST, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath);
return responseEntity.getStatusCode().is2xxSuccessful();
}
@@ -295,10 +292,7 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
public boolean deleteFile(String path, String name) {
String fullPath = StringUtils.concat(param.getBasePath(), path, name);
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
HttpEntity<Object> entity = new HttpEntity<>(headers);
HttpEntity<Object> entity = getAuthorizationHttpEntity();
ResponseEntity<JSONObject> responseEntity = oneDriveRestTemplate.exchange(DRIVER_ITEM_OPERATOR_URL, HttpMethod.DELETE, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath);
return responseEntity.getStatusCode().is2xxSuccessful();
}
@@ -307,12 +301,9 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
public boolean renameFile(String path, String name, String newName) {
String fullPath = StringUtils.concat(param.getBasePath(), path, name);
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
JSONObject jsonObject = new JSONObject();
jsonObject.put("name", newName);
HttpEntity<Object> entity = new HttpEntity<>(jsonObject, headers);
JSONObject jsonObject = new JSONObject().fluentPut("name", newName);
HttpEntity<Object> entity = getAuthorizationHttpEntity(jsonObject);
ResponseEntity<JSONObject> responseEntity = oneDriveRestTemplate.exchange(DRIVER_ITEM_OPERATOR_URL, HttpMethod.PATCH, entity, JSONObject.class, getGraphEndPoint(), getType(), fullPath);
return responseEntity.getStatusCode().is2xxSuccessful();
}
@@ -326,10 +317,7 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
public String getUploadUrl(String path, String name, Long size) {
String fullPath = StringUtils.concat(param.getBasePath(), path, name);
HttpHeaders headers = new HttpHeaders();
headers.set("storageId", storageId.toString());
HttpEntity<Object> entity = new HttpEntity<>(headers);
HttpEntity<Object> entity = getAuthorizationHttpEntity();
ResponseEntity<JSONObject> responseEntity = oneDriveRestTemplate.exchange(CREATE_UPLOAD_SESSION_URL,
HttpMethod.POST, entity, JSONObject.class,
getGraphEndPoint(), getType(), fullPath);
@@ -388,7 +376,7 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
@Override
public void refreshAccessToken() {
try {
OAuth2TokenDTO refreshToken = getRefreshToken();
OAuth2TokenDTO refreshToken = getAndRefreshToken();
if (refreshToken.getAccessToken() == null || refreshToken.getRefreshToken() == null) {
throw new StorageSourceRefreshTokenException("存储源刷新令牌失败, 获取到令牌为空.", storageId);
@@ -403,7 +391,6 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
storageSourceConfigService.updateBatch(storageId, Arrays.asList(accessTokenConfig, refreshTokenConfig));
RefreshTokenCacheBO.putRefreshTokenInfo(storageId, RefreshTokenCacheBO.RefreshTokenInfo.success());
log.info("存储源 {} 刷新 AccessToken 成功", storageId);
} catch (Exception e) {
RefreshTokenCacheBO.putRefreshTokenInfo(storageId, RefreshTokenCacheBO.RefreshTokenInfo.fail(getStorageTypeEnum().getDescription() + " AccessToken 刷新失败: " + e.getMessage()));
throw new StorageSourceRefreshTokenException("存储源刷新令牌失败,获取时发生异常", e, storageId);
@@ -430,5 +417,57 @@ public abstract class AbstractMicrosoftDriveService<P extends MicrosoftDrivePara
fileItemResult.setPath(folderPath);
return fileItemResult;
}
/**
* 获取存储源默认的 HttpEntity 对象.
* <br>
* 该对象默认包含了当前存储源的 AccessToken.
*
* @return HttpEntity 对象
*/
private HttpEntity<Object> getAuthorizationHttpEntity() {
return getAuthorizationHttpEntity(null);
}
/**
* 获取存储源默认的 HttpEntity 对象.
* <br>
* 该对象默认包含了当前存储源的 AccessToken.
*
* @param body
* 请求体
*
* @return HttpEntity 对象
*/
private <T> HttpEntity<T> getAuthorizationHttpEntity(T body) {
HttpHeaders headers = new HttpHeaders();
StorageSourceConfig accessTokenConfig =
storageSourceConfigService.findByStorageIdAndName(storageId, StorageConfigConstant.ACCESS_TOKEN_KEY);
headers.setBearerAuth(accessTokenConfig.getValue());
return new HttpEntity<>(body, headers);
}
/**
* 执行可重试 1 次的任务, 对抛出的异常转为 ZFileRetryException(Unchecked Exception)
* @param retryCallback 可重试的任务
* @return 任务执行结果
*
* @param <T> 任务执行结果类型
*/
private <T> T executeRetryableRequest(RetryCallback<T, Throwable> retryCallback) {
RetryTemplate retryTemplate = RetryTemplate.builder().maxAttempts(2).retryOn(HttpClientErrorException.class).build();
T result;
try {
result = retryTemplate.execute(retryCallback);
} catch (Throwable e) {
throw new ZFileRetryException("请求失败", e);
}
return result;
}
}