renun/zfile
1
0
Fork 0
mirror of https://github.com/zfile-dev/zfile.git synced 2025-04-19 05:34:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

✨ 缓存功能优化, 更高效的管理缓存.

Browse Source
This commit is contained in:
zhaojun1998
2020-01-03 15:27:45 +08:00
parent 6997b15dd0
commit 5eeea23703
9 changed files with 164 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/im/zhaojun/common/aspect/StorageStrategyInitCheckAspect.java
View File

@@ -22,7 +22,7 @@ public class StorageStrategyInitCheckAspect {
if (currentFileService == null) {
throw new StorageStrategyUninitializedException("存储策略尚未初始化, 请联系管理员!");
}
if (!currentFileService.getIsInitialized()) {
if (currentFileService.getIsUnInitialized()) {
throw new StorageStrategyUninitializedException("存储策略异常, 请联系管理员!");
}

60
src/main/java/im/zhaojun/common/controller/AdminController.java
View File

@@ -1,6 +1,7 @@
package im.zhaojun.common.controller;
import im.zhaojun.common.model.StorageConfig;
import im.zhaojun.common.model.dto.CacheConfigDTO;
import im.zhaojun.common.model.dto.ResultBean;
import im.zhaojun.common.model.dto.SystemConfigDTO;
import im.zhaojun.common.model.enums.StorageTypeEnum;
@@ -17,6 +18,7 @@ import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.util.List;
import java.util.Set;
/**
* 后台管理
@@ -46,15 +48,6 @@ public class AdminController {
return ResultBean.success(systemConfigDTO);
}
/**
* 更新系统配置
*/
@PostMapping("/update-pwd")
public ResultBean updatePwd(String username, String password) {
systemConfigService.updateUsernameAndPwd(username, password);
return ResultBean.success();
}
/**
* 更新系统配置
*/
@@ -72,23 +65,61 @@ public class AdminController {
return ResultBean.success();
}
/**
* 修改管理员登陆密码
*/
@PostMapping("/update-pwd")
public ResultBean updatePwd(String username, String password) {
systemConfigService.updateUsernameAndPwd(username, password);
return ResultBean.success();
}
/**
* 获取指定存储引擎的设置
* @param storageType 存储引擎
* @return 所有设置
*/
@GetMapping("/strategy-form")
public ResultBean getFormByStorageType(StorageTypeEnum storageType) {
List<StorageConfig> storageConfigList = storageConfigService.selectStorageConfigByType(storageType);
return ResultBean.success(storageConfigList);
}
/**
* 清理当前启用的存储引擎的缓存
*/
@PostMapping("/clear-cache")
public ResultBean clearCache() {
@GetMapping("/cache/config")
public ResultBean cacheConfig() throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
Set<String> cacheKeys = fileService.getCacheKeys();
CacheConfigDTO cacheConfigDTO = new CacheConfigDTO();
cacheConfigDTO.setEnableCache(systemConfigService.getEnableCache());
cacheConfigDTO.setCacheFinish(fileAsyncCacheService.isCacheFinish());
cacheConfigDTO.setCacheKeys(cacheKeys);
return ResultBean.success(cacheConfigDTO);
}
@PostMapping("/cache/refresh")
public ResultBean refreshCache(String key) throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
fileService.refreshCache(key);
return ResultBean.success();
}
@PostMapping("/cache/clear")
public ResultBean clearCache(String key) throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
fileService.clearCache();
return ResultBean.success();
}
@PostMapping("/cache/all")
public ResultBean cacheAll() throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
fileService.clearCache();
fileAsyncCacheService.cacheGlobalFile();
return ResultBean.success();
}
/**
* 更新存储策略
*/
@@ -112,4 +143,5 @@ public class AdminController {
fileAsyncCacheService.cacheGlobalFile();
}
}
}

4
src/main/java/im/zhaojun/common/controller/FileController.java
View File

@@ -86,7 +86,7 @@ public class FileController {
int start = (page - 1) * PAGE_SIZE;
int end = page * PAGE_SIZE;
end = Math.min(end, total);
List<FileItemDTO> fileSubItem = fileItemList.subList(start, end);
List<FileItemDTO> fileSubItem = new ArrayList<>(fileItemList.subList(start, end));
return ResultBean.successData(fileSubItem);
}
@@ -115,7 +115,7 @@ public class FileController {
@CheckStorageStrategyInit
@GetMapping("/clearCache")
public ResultBean clearCache() {
public ResultBean clearCache() throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
if (fileService != null) {
fileService.clearCache();

2
src/main/java/im/zhaojun/common/controller/InstallController.java
View File

@@ -79,7 +79,7 @@ public class InstallController {
@PostMapping("/storage-strategy")
@ResponseBody
public ResultBean save(@RequestParam Map<String, String> storageStrategyConfig, StorageTypeEnum storageStrategy) {
public ResultBean save(@RequestParam Map<String, String> storageStrategyConfig, StorageTypeEnum storageStrategy) throws Exception {
List<StorageConfig> storageConfigList = storageConfigService.selectStorageConfigByType(storageStrategy);
for (StorageConfig storageConfig : storageConfigList) {
String key = storageConfig.getKey();

16
src/main/java/im/zhaojun/common/model/dto/CacheConfigDTO.java Normal file
View File

@@ -0,0 +1,16 @@
package im.zhaojun.common.model.dto;
import lombok.Data;
import java.util.Set;
/**
* @author zhaojun
* @date 2020/1/3 12:39
*/
@Data
public class CacheConfigDTO {
private boolean enableCache;
private boolean cacheFinish;
private Set<String> cacheKeys;
}

83
src/main/java/im/zhaojun/common/service/AbstractFileService.java
View File

@@ -1,8 +1,10 @@
package im.zhaojun.common.service;
import com.alicp.jetcache.Cache;
import com.alicp.jetcache.anno.CacheRefresh;
import com.alicp.jetcache.anno.CacheType;
import com.alicp.jetcache.anno.Cached;
import com.alicp.jetcache.anno.CreateCache;
import im.zhaojun.common.model.dto.FileItemDTO;
import im.zhaojun.common.model.enums.FileTypeEnum;
import im.zhaojun.common.model.enums.StorageTypeEnum;
@@ -15,8 +17,11 @@ import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @author zhaojun
@@ -25,6 +30,8 @@ import java.util.concurrent.TimeUnit;
@Slf4j
public abstract class AbstractFileService implements FileService {
public static final String SYSTEM_CONFIG_CACHE_PREFIX = "zfile-cache:";
@Value("${zfile.cache.timeout}")
protected Long timeout;
@@ -33,6 +40,12 @@ public abstract class AbstractFileService implements FileService {
@Resource
private SystemConfigService systemConfigService;
@Resource
private FileAsyncCacheService fileAsyncCacheService;
@CreateCache(name = SYSTEM_CONFIG_CACHE_PREFIX, cacheType = CacheType.LOCAL)
private Cache<String, List<FileItemDTO>> cache;
/***
* 获取指定路径下的文件及文件夹, 默认缓存 60 分钟，每隔 30 分钟刷新一次.
* @param path 文件路径
@@ -40,17 +53,19 @@ public abstract class AbstractFileService implements FileService {
* @throws Exception 获取文件列表中出现的异常
*/
@Override
@Cached(name = "zfile-cache:",
@Cached(name = SYSTEM_CONFIG_CACHE_PREFIX,
key = "args[0]",
cacheType = CacheType.LOCAL, condition = "mvel{bean('systemConfigService').enableCache}")
cacheType = CacheType.LOCAL, localLimit = 100000, condition = "mvel{bean('systemConfigService').enableCache}")
@CacheRefresh(refresh = 30, timeUnit = TimeUnit.MINUTES)
public abstract List<FileItemDTO> fileList(String path) throws Exception;
/**
* 清理当前存储引擎的缓存
*/
public void clearCache() {
public void clearCache() throws Exception {
Set<String> cacheKeys = getCacheKeys();
cache.removeAll(cacheKeys);
fileAsyncCacheService.setCacheFinish(false);
}
/**
@@ -72,22 +87,22 @@ public abstract class AbstractFileService implements FileService {
/**
* 获取是否初始化成功
* @return 初始化成功与否
* @return 初始化成功与否
*/
public boolean getIsInitialized() {
return isInitialized;
public boolean getIsUnInitialized() {
return !isInitialized;
}
/**
* 获取存储引擎类型
* @return 存储引擎类型枚举
* @return 存储引擎类型枚举
*/
public abstract StorageTypeEnum getStorageTypeEnum();
/**
* 搜索文件
* @param name 文件名
* @return 包含该文件名的所有文件或文件夹
* @param name 文件名
* @return 包含该文件名的所有文件或文件夹
* @throws Exception 搜索过程出现的异常
*/
public List<FileItemDTO> search(String name) throws Exception {
@@ -104,23 +119,20 @@ public abstract class AbstractFileService implements FileService {
}
/**
* 查询所有文件
* @return 所有文件
* @throws Exception 异常现象
* 查询所有文件, 仅去缓存中查询.
* @return 所有文件
*/
public List<FileItemDTO> selectAllFileList() throws Exception {
List<FileItemDTO> result = new ArrayList<>();
boolean enableCache = systemConfigService.getEnableCache();
if (!enableCache) {
log.debug("未开启缓存, 不支持查询所有文件.");
return null;
}
String path = "/";
FileService currentFileService = (FileService) AopContext.currentProxy();
List<FileItemDTO> fileItemList = currentFileService.fileList(path);
List<FileItemDTO> fileItemList = cache.get(path);
fileItemList = fileItemList == null ? new ArrayList<>() : fileItemList;
ArrayDeque<FileItemDTO> queue = new ArrayDeque<>(fileItemList);
while (!queue.isEmpty()) {
@@ -128,10 +140,43 @@ public abstract class AbstractFileService implements FileService {
result.add(fileItemDTO);
if (fileItemDTO.getType() == FileTypeEnum.FOLDER) {
String filePath = StringUtils.removeDuplicateSeparator("/" + fileItemDTO.getPath() + "/" + fileItemDTO.getName() + "/");
queue.addAll(currentFileService.fileList(filePath));
List<FileItemDTO> cacheList = cache.get(filePath);
if (cacheList != null) {
queue.addAll(cacheList);
}
}
}
}
return result;
}
/**
* 获取所有缓存的 Key, 仅当开启缓存, 且缓存完成时, 可获取.
* @return 所有缓存的 Key
* @throws Exception 可能出现的异常
*/
public Set<String> getCacheKeys() throws Exception {
if (systemConfigService.getEnableCache() && fileAsyncCacheService.isCacheFinish()) {
Set<String> collect = selectAllFileList().stream().map(fileItemDTO -> {
if (fileItemDTO.getType() == FileTypeEnum.FOLDER) {
return StringUtils.removeDuplicateSeparator("/" + fileItemDTO.getPath() + "/" + fileItemDTO.getName() + "/");
}
return null;
}).collect(Collectors.toSet());
collect.remove(null);
return collect;
} else {
return Collections.emptySet();
}
}
/**
* 刷新缓存
*/
public void refreshCache(String key) throws Exception {
cache.remove(key);
FileService currentFileService = (FileService) AopContext.currentProxy();
currentFileService.fileList(key);
}
}

33
src/main/java/im/zhaojun/common/service/FileAsyncCacheService.java
View File

@@ -1,12 +1,17 @@
package im.zhaojun.common.service;
import im.zhaojun.common.config.StorageTypeFactory;
import im.zhaojun.common.model.dto.FileItemDTO;
import im.zhaojun.common.model.enums.FileTypeEnum;
import im.zhaojun.common.model.enums.StorageTypeEnum;
import im.zhaojun.common.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayDeque;
import java.util.List;
/**
* @author zhaojun
@@ -29,12 +34,35 @@ public class FileAsyncCacheService {
return;
}
boolean enableCache = systemConfigService.getEnableCache();
if (!enableCache) {
log.info("未开启缓存, 跳过启动缓存");
return;
}
AbstractFileService fileService = StorageTypeFactory.getStorageTypeService(storageStrategy);
if (fileService.getIsUnInitialized()) {
log.info("存储引擎 {} 未初始化成功, 跳过启动缓存.", storageStrategy.getDescription());
return;
}
log.info("缓存 {} 所有文件开始", storageStrategy.getDescription());
long startTime = System.currentTimeMillis();
try {
if (fileService.getIsInitialized()) {
fileService.selectAllFileList();
String path = "/";
FileService currentFileService = systemConfigService.getCurrentFileService();
List<FileItemDTO> fileItemList = currentFileService.fileList(path);
ArrayDeque<FileItemDTO> queue = new ArrayDeque<>(fileItemList);
while (!queue.isEmpty()) {
FileItemDTO fileItemDTO = queue.pop();
if (fileItemDTO.getType() == FileTypeEnum.FOLDER) {
String filePath = StringUtils.removeDuplicateSeparator("/" + fileItemDTO.getPath() + "/" + fileItemDTO.getName() + "/");
queue.addAll(currentFileService.fileList(filePath));
}
}
} catch (Exception e) {
log.error("缓存所有文件失败", e);
@@ -45,6 +73,7 @@ public class FileAsyncCacheService {
cacheFinish = true;
}
public boolean isCacheFinish() {
return cacheFinish;
}

2
src/main/java/im/zhaojun/common/service/FileService.java
View File

@@ -10,7 +10,7 @@ import java.util.List;
public interface FileService {
/***
* 获取指定路径下的文件及文件夹, 默认缓存 60 分钟，每隔 30 分钟刷新一次.
* 获取指定路径下的文件及文件夹
* @param path 文件路径
* @return 文件及文件夹列表
* @throws Exception 获取文件列表中出现的异常

3
src/main/java/im/zhaojun/common/service/SystemService.java
View File

@@ -7,6 +7,7 @@ import im.zhaojun.common.util.HttpUtil;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
/**
@@ -27,7 +28,7 @@ public class SystemService {
SiteConfigDTO siteConfigDTO = new SiteConfigDTO();
AbstractFileService fileService = systemConfigService.getCurrentFileService();
List<FileItemDTO> fileItemList = fileService.fileList(path);
List<FileItemDTO> fileItemList = new ArrayList<>(fileService.fileList(path));
for (FileItemDTO fileItemDTO : fileItemList) {
if (ZFileConstant.FOOTER_FILE_NAME.equalsIgnoreCase(fileItemDTO.getName())) {
siteConfigDTO.setFooter(HttpUtil.getTextContent(fileItemDTO.getUrl()));