renun/zfile
1
0
Fork 0
mirror of https://github.com/zfile-dev/zfile.git synced 2025-04-19 05:34:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

🏗️ 缓存架构调整, 增强稳定性

Browse Source
This commit is contained in:
zhaojun1998
2020-02-29 15:43:56 +08:00
parent d29c498457
commit 04f94b4bf5
9 changed files with 149 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/main/java/im/zhaojun/common/cache/ZFileCache.java vendored
View File

@@ -3,13 +3,11 @@ package im.zhaojun.common.cache;
import cn.hutool.core.util.StrUtil;
import im.zhaojun.common.model.dto.FileItemDTO;
import im.zhaojun.common.model.dto.SystemConfigDTO;
import im.zhaojun.common.model.enums.FileTypeEnum;
import im.zhaojun.common.service.SystemConfigService;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -21,22 +19,14 @@ public class ZFileCache {
private ConcurrentMap<String, List<FileItemDTO>> fileCache = new ConcurrentHashMap<>();
private ConcurrentMap<String, Integer> fileCountCache = new ConcurrentHashMap<>();
private SystemConfigDTO systemConfigCache;
public static final String CACHE_FILE_COUNT_KEY = "file-count";
public Date lastCacheAutoRefreshDate;
public static final String CACHE_DIRECTORY_COUNT_KEY = "directory-count";
@Resource
private SystemConfigService systemConfigService;
public synchronized void put(String key, List<FileItemDTO> value) {
for (FileItemDTO fileItemDTO : value) {
if (FileTypeEnum.FILE.equals(fileItemDTO.getType())) {
incrCacheFileCount();
} else {
incrCacheDirectoryCount();
}
}
fileCache.put(key, value);
}
@@ -46,10 +36,9 @@ public class ZFileCache {
public void clear() {
fileCache.clear();
fileCountCache.clear();
}
public long cacheCount() {
public int cacheCount() {
return fileCache.size();
}
@@ -83,24 +72,6 @@ public class ZFileCache {
fileCache.remove(key);
}
private void incrCacheFileCount() {
Integer originValue = fileCountCache.getOrDefault(CACHE_FILE_COUNT_KEY, 0);
fileCountCache.put(CACHE_FILE_COUNT_KEY, originValue + 1);
}
private void incrCacheDirectoryCount() {
Integer originValue = fileCountCache.getOrDefault(CACHE_DIRECTORY_COUNT_KEY, 0);
fileCountCache.put(CACHE_DIRECTORY_COUNT_KEY, originValue + 1);
}
public int getCacheFileCount() {
return fileCountCache.getOrDefault(CACHE_FILE_COUNT_KEY, 0);
}
public int getCacheDirectorCount() {
return fileCountCache.getOrDefault(CACHE_DIRECTORY_COUNT_KEY, 0);
}
public void updateConfig(SystemConfigDTO systemConfigCache) {
this.systemConfigCache = systemConfigCache;
}
@@ -112,4 +83,12 @@ public class ZFileCache {
public void removeConfig() {
this.systemConfigCache = null;
}
public Date getLastCacheAutoRefreshDate() {
return lastCacheAutoRefreshDate;
}
public void setLastCacheAutoRefreshDate(Date lastCacheAutoRefreshDate) {
this.lastCacheAutoRefreshDate = lastCacheAutoRefreshDate;
}
}

31
src/main/java/im/zhaojun/common/controller/AdminController.java
View File

@@ -17,20 +17,13 @@ import im.zhaojun.common.util.FileUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.ScheduledExecutorService;
/**
* 后台管理
@@ -51,6 +44,8 @@ public class AdminController {
@Resource
private FileAsyncCacheService fileAsyncCacheService;
private ScheduledExecutorService scheduledExecutorService;
/**
* 获取系统配置
*/
@@ -65,18 +60,20 @@ public class AdminController {
*/
@PostMapping("/config")
public ResultBean updateConfig(SystemConfigDTO systemConfigDTO) throws Exception {
AbstractFileService currentFileService = systemConfigService.getCurrentFileService();
systemConfigDTO.setId(1);
systemConfigService.updateSystemConfig(systemConfigDTO);
StorageTypeEnum currentStorageStrategy = currentFileService.getStorageTypeEnum();
StorageTypeEnum currentStorageStrategy = systemConfigService.getCurrentStorageStrategy();
if (!Objects.equals(currentStorageStrategy, systemConfigDTO.getStorageStrategy())) {
if (systemConfigService.getEnableCache()) {
return ResultBean.error("不支持缓存开启状态下, 切换存储策略, 请先手动关闭缓存");
}
log.info("已将存储策略由 {} 切换为 {}",
currentStorageStrategy.getDescription(),
systemConfigDTO.getStorageStrategy().getDescription());
refreshStorageStrategy();
}
systemConfigDTO.setId(1);
systemConfigService.updateSystemConfig(systemConfigDTO);
return ResultBean.success();
}
@@ -161,7 +158,7 @@ public class AdminController {
/**
* 更新存储策略
*/
public void refreshStorageStrategy() throws Exception {
public void refreshStorageStrategy() {
StorageTypeEnum storageStrategy = systemConfigService.getCurrentStorageStrategy();
refreshStorageStrategy(storageStrategy);
}
@@ -169,7 +166,7 @@ public class AdminController {
/**
* 更新存储策略
*/
public void refreshStorageStrategy(StorageTypeEnum storageStrategy) throws Exception {
private void refreshStorageStrategy(StorageTypeEnum storageStrategy) {
if (storageStrategy == null) {
log.info("尚未配置存储策略.");
} else {

14
src/main/java/im/zhaojun/common/controller/CacheController.java
View File

@@ -35,13 +35,13 @@ public class CacheController {
private ZFileCache zFileCache;
@PostMapping("/enable")
public ResultBean enableCache() throws Exception {
public ResultBean enableCache() {
fileCacheService.enableCache();
return ResultBean.success();
}
@PostMapping("/disable")
public ResultBean disableCache() throws Exception {
public ResultBean disableCache() {
fileCacheService.disableCache();
return ResultBean.success();
}
@@ -53,11 +53,12 @@ public class CacheController {
cacheConfigDTO.setEnableCache(systemConfigService.getEnableCache());
cacheConfigDTO.setCacheFinish(fileAsyncCacheService.isCacheFinish());
cacheConfigDTO.setCacheKeys(zFileCache.keySet());
cacheConfigDTO.setCacheDirectoryCount(zFileCache.getCacheDirectorCount());
cacheConfigDTO.setCacheFileCount(zFileCache.getCacheFileCount());
cacheConfigDTO.setCacheDirectoryCount(zFileCache.cacheCount());
cacheConfigDTO.setLastCacheAutoRefreshDate(zFileCache.getLastCacheAutoRefreshDate());
return ResultBean.success(cacheConfigDTO);
}
/*
@PostMapping("/refresh")
public ResultBean refreshCache(String key) throws Exception {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
@@ -66,14 +67,15 @@ public class CacheController {
}
@PostMapping("/clear")
public ResultBean clearCache(String key) throws Exception {
public ResultBean clearCache(String key) {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
fileService.clearFileCache();
return ResultBean.success();
}
*/
@PostMapping("/all")
public ResultBean cacheAll() throws Exception {
public ResultBean cacheAll() {
AbstractFileService fileService = systemConfigService.getCurrentFileService();
fileService.clearFileCache();
fileAsyncCacheService.cacheGlobalFile();

2
src/main/java/im/zhaojun/common/model/dto/CacheConfigDTO.java
View File

@@ -2,6 +2,7 @@ package im.zhaojun.common.model.dto;
import lombok.Data;
import java.util.Date;
import java.util.Set;
/**
@@ -14,4 +15,5 @@ public class CacheConfigDTO {
private Set<String> cacheKeys;
private Integer cacheDirectoryCount;
private Integer cacheFileCount;
private Date lastCacheAutoRefreshDate;
}

14
src/main/java/im/zhaojun/common/service/AbstractFileService.java
View File

@@ -52,13 +52,10 @@ public abstract class AbstractFileService extends FileCacheService implements Fi
/**
* 清理当前存储策略的缓存
* 1. 删除全部缓存
* 2. 关闭自动刷新
* 3. 重置缓存个数
* 4. 标记为当前处于未完成缓存状态
* 2. 标记为当前处于未完成缓存状态
*/
public void clearFileCache() {
zFileCache.clear();
closeCacheAutoRefresh();
fileAsyncCacheService.setCacheFinish(false);
}
@@ -141,15 +138,6 @@ public abstract class AbstractFileService extends FileCacheService implements Fi
currentFileService.fileList(key);
}
public void closeCacheAutoRefresh() {
// cache.config().setRefreshPolicy(null);
}
public void openCacheAutoRefresh() {
// RefreshPolicy refreshPolicy = RefreshPolicy.newPolicy(30, TimeUnit.MINUTES);
// cache.config().setRefreshPolicy(refreshPolicy);
}
public abstract FileItemDTO getFileItem(String path);
}

104
src/main/java/im/zhaojun/common/service/FileAsyncCacheService.java
View File

@@ -1,17 +1,21 @@
package im.zhaojun.common.service;
import im.zhaojun.common.cache.ZFileCache;
import im.zhaojun.common.config.StorageTypeFactory;
import im.zhaojun.common.model.dto.FileItemDTO;
import im.zhaojun.common.model.enums.FileTypeEnum;
import im.zhaojun.common.model.enums.StorageTypeEnum;
import im.zhaojun.common.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayDeque;
import java.util.List;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* @author zhaojun
@@ -27,8 +31,23 @@ public class FileAsyncCacheService {
@Resource
private SystemConfigService systemConfigService;
private volatile boolean stopFlag = false;
@Resource
private ZFileCache zFileCache;
@Value("${zfile.cache.auto-refresh.enable}")
protected boolean enableAutoRefreshCache;
@Value("${zfile.cache.auto-refresh.delay}")
protected Long delay;
@Value("${zfile.cache.auto-refresh.interval}")
protected Long interval;
@Async
public void cacheGlobalFile() {
stopFlag = false;
StorageTypeEnum storageStrategy = systemConfigService.getCurrentStorageStrategy();
if (storageStrategy == null) {
@@ -61,6 +80,11 @@ public class FileAsyncCacheService {
while (!queue.isEmpty()) {
FileItemDTO fileItemDTO = queue.pop();
if (stopFlag) {
zFileCache.clear();
break;
}
if (fileItemDTO.getType() == FileTypeEnum.FOLDER) {
String filePath = StringUtils.removeDuplicateSeparator("/" + fileItemDTO.getPath() + "/" + fileItemDTO.getName() + "/");
@@ -73,10 +97,82 @@ public class FileAsyncCacheService {
e.printStackTrace();
}
long endTime = System.currentTimeMillis();
log.info("缓存 {} 所有文件结束, 用时: {} 秒", storageStrategy.getDescription(), ((endTime - startTime) / 1000));
cacheFinish = true;
if (stopFlag) {
log.info("缓存 {} 所有文件被强制结束, 用时: {} 秒", storageStrategy.getDescription(), ((endTime - startTime) / 1000));
cacheFinish = false;
stopFlag = false;
} else {
log.info("缓存 {} 所有文件结束, 用时: {} 秒", storageStrategy.getDescription(), ((endTime - startTime) / 1000));
enableCacheAutoRefreshTask();
cacheFinish = true;
stopFlag = false;
}
}
private void enableCacheAutoRefreshTask() {
StorageTypeEnum currentStorageStrategy = systemConfigService.getCurrentStorageStrategy();
if (enableAutoRefreshCache) {
ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
scheduledExecutorService.scheduleWithFixedDelay(() -> {
zFileCache.setLastCacheAutoRefreshDate(new Date());
boolean enableCache = systemConfigService.getEnableCache();
if (!enableCache) {
log.debug("当前存储引擎未开启缓存, 跳过自动刷新缓存");
zFileCache.clear();
return;
}
log.debug("开始调用自动刷新缓存");
Set<String> keySet = zFileCache.keySet();
ArrayList<String> keys = new ArrayList<>(keySet);
for (String key : keys) {
try {
Thread.sleep(300);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (stopFlag) {
break;
}
zFileCache.remove(key);
AbstractFileService currentFileService = systemConfigService.getCurrentFileService();
try {
if (Objects.equals(currentStorageStrategy, systemConfigService.getCurrentStorageStrategy())) {
currentFileService.fileList(key);
}
} catch (Exception e) {
log.error("刷新过程中出错 : [" + key + "]", e);
}
}
if (stopFlag) {
log.debug("检测到停止 [{}] 缓存指令, 已停止自动刷新任务", currentStorageStrategy);
scheduledExecutorService.shutdownNow();
stopFlag = false;
} else {
log.debug("自动刷新缓存完成");
}
}, delay, interval, TimeUnit.SECONDS);
}
}
public void stopScheduled() {
this.stopFlag = true;
}
public void enableScheduled() {
this.stopFlag = false;
}
public boolean isCacheFinish() {
return cacheFinish;

15
src/main/java/im/zhaojun/common/service/FileCacheService.java
View File

@@ -1,5 +1,6 @@
package im.zhaojun.common.service;
import im.zhaojun.common.cache.ZFileCache;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@@ -18,19 +19,19 @@ public class FileCacheService {
@Lazy
private FileAsyncCacheService fileAsyncCacheService;
@Resource
private ZFileCache zFileCache;
public void enableCache() {
systemConfigService.updateCacheEnableConfig(true);
AbstractFileService currentFileService = systemConfigService.getCurrentFileService();
currentFileService.openCacheAutoRefresh();
fileAsyncCacheService.cacheGlobalFile();
}
public void disableCache() throws Exception {
public void disableCache() {
systemConfigService.updateCacheEnableConfig(false);
AbstractFileService currentFileService = systemConfigService.getCurrentFileService();
currentFileService.clearFileCache();
zFileCache.clear();
fileAsyncCacheService.setCacheFinish(false);
fileAsyncCacheService.stopScheduled();
}
}

14
src/main/java/im/zhaojun/common/service/SystemConfigService.java
View File

@@ -89,22 +89,8 @@ public class SystemConfigService {
}
}
boolean oldEnableCache = getEnableCache();
boolean curEnableCache = BooleanUtil.isTrue(systemConfigDTO.getEnableCache());
zFileCache.removeConfig();
systemConfigRepository.saveAll(systemConfigList);
if (!oldEnableCache && curEnableCache) {
log.debug("检测到开启了缓存, 开启预热缓存");
fileCacheService.enableCache();
}
if (oldEnableCache && !curEnableCache) {
log.debug("检测到关闭了缓存, 正在清理缓存数据及关闭自动刷新");
fileCacheService.disableCache();
}
}

42
src/main/java/im/zhaojun/common/util/StartupListener.java
View File

@@ -16,10 +16,7 @@ import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.*;
/**
* 项目启动监听器, 当项目启动时, 遍历当前对象存储的所有内容, 添加到缓存中.
@@ -35,18 +32,6 @@ public class StartupListener implements ApplicationListener<ApplicationStartedEv
@Resource
private Environment environment;
@Resource
private ZFileCache zFileCache;
@Value("${zfile.cache.auto-refresh.enable}")
protected boolean enableAutoRefreshCache;
@Value("${zfile.cache.auto-refresh.delay}")
protected Long delay;
@Value("${zfile.cache.auto-refresh.interval}")
protected Long interval;
@Resource
private SystemConfigService systemConfigService;
@@ -54,33 +39,8 @@ public class StartupListener implements ApplicationListener<ApplicationStartedEv
public void onApplicationEvent(@NonNull ApplicationStartedEvent event) {
printStartInfo();
cacheAllFile();
enableCacheAutoRefreshTask();
}
private void enableCacheAutoRefreshTask() {
if (enableAutoRefreshCache) {
new Timer("testTimer").schedule(new TimerTask() {
@SneakyThrows
@Override
public void run() {
boolean enableCache = systemConfigService.getEnableCache();
if (!enableCache) {
return;
}
log.debug("开始调用自动刷新缓存");
Set<String> keySet = zFileCache.keySet();
for (String key : keySet) {
zFileCache.remove(key);
AbstractFileService currentFileService = systemConfigService.getCurrentFileService();
currentFileService.fileList(key);
}
}
}, delay * 1000,interval * 1000);
}
}
private void printStartInfo() {
String serverPort = environment.getProperty("server.port", "8080");