Merge pull request #220 from xream/fix/FullConfig

fix: Full Config Preprocessor

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: "14"
+          node-version: "16"
       - name: Install dependencies
         run: |
           npm install -g pnpm
@@ -51,4 +51,4 @@ jobs:
             ./backend/dist/sub-store-0.min.js
             ./backend/dist/sub-store-1.min.js
             ./backend/dist/sub-store-parser.loon.min.js
-            ./backend/dist/cron-sync-artifacts.js
+            ./backend/dist/cron-sync-artifacts.min.js

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.13.2",
+  "version": "2.14.1",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {
@@ -30,7 +30,7 @@
     "@babel/preset-env": "^7.18.0",
     "@babel/register": "^7.17.7",
     "@types/gulp": "^4.0.9",
-    "axios": "^0.20.0",
+    "axios": "^0.21.2",
     "babel-plugin-relative-path-import": "^2.0.1",
     "babelify": "^10.0.0",
     "browser-pack-flat": "^3.4.2",
@@ -51,4 +51,4 @@
     "prettier-plugin-sort-imports": "^1.6.1",
     "tinyify": "^3.0.0"
   }
-}
+}

backend/src/constants.js

@@ -9,3 +9,5 @@ export const GIST_BACKUP_FILE_NAME = 'Sub-Store';
 export const ARTIFACT_REPOSITORY_KEY = 'Sub-Store Artifacts Repository';
 export const RESOURCE_CACHE_KEY = '#sub-store-cached-resource';
 export const CACHE_EXPIRATION_TIME_MS = 60 * 60 * 1000; // 1 hour
+export const SCRIPT_RESOURCE_CACHE_KEY = '#sub-store-cached-script-resource'; // cached-script-resource CSR
+export const CSR_EXPIRATION_TIME_KEY = '#sub-store-csr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 48 hour

backend/src/core/proxy-utils/preprocessors/index.js

@@ -95,22 +95,10 @@ function FullConfig() {
         return /^(\[server_local\]|\[Proxy\])/gm.test(raw);
     };
     const parse = function (raw) {
-        const regex = /^\[server_local]|\[Proxy]/gm;
-        const match = regex.exec(raw);
-        const results = [];
-
-        let first = true;
-        if (match) {
-            raw = raw.substring(match.index);
-            for (const line of raw.split('\n')) {
-                if (!first && !line.test(/^\s*\[/)) {
-                    results.push(line);
-                }
-                // skip the first line
-                first = false;
-            }
-            return results.join('\n');
-        }
+        const match = raw.match(
+            /^\[server_local|Proxy\]([\s\S]+?)^\[.+?\](\r?\n|$)/im,
+        )?.[1];
+        return match || raw;
     };
     return { name, test, parse };
 }

backend/src/core/proxy-utils/processors/index.js

@@ -1,4 +1,5 @@
 import resourceCache from '@/utils/resource-cache';
+import scriptResourceCache from '@/utils/script-resource-cache';
 import { isIPv4, isIPv6 } from '@/utils';
 import { FULL } from '@/utils/logical';
 import { getFlag } from '@/utils/geo';
@@ -633,6 +634,7 @@ function createDynamicFunction(name, script, $arguments) {
             '$httpClient',
             '$notification',
             'ProxyUtils',
+            'scriptResourceCache',
             `${script}\n return ${name}`,
         )(
             $arguments,
@@ -645,6 +647,7 @@ function createDynamicFunction(name, script, $arguments) {
             // eslint-disable-next-line no-undef
             $notification,
             ProxyUtils,
+            scriptResourceCache,
         );
     } else {
         return new Function(
@@ -652,7 +655,8 @@ function createDynamicFunction(name, script, $arguments) {
             '$substore',
             'lodash',
             'ProxyUtils',
+            'scriptResourceCache',
             `${script}\n return ${name}`,
-        )($arguments, $, lodash, ProxyUtils);
+        )($arguments, $, lodash, ProxyUtils, scriptResourceCache);
     }
 }

backend/src/core/proxy-utils/producers/surge.js

@@ -55,6 +55,10 @@ function shadowsocks(proxy) {
 
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
+
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 
@@ -87,6 +91,10 @@ function trojan(proxy) {
 
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
+
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 
@@ -127,6 +135,9 @@ function vmess(proxy) {
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
 
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 
@@ -155,6 +166,10 @@ function http(proxy) {
 
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
+
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 
@@ -185,6 +200,10 @@ function socks5(proxy) {
 
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
+
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 
@@ -210,6 +229,10 @@ function snell(proxy) {
 
     // udp
     result.appendIfPresent(`,udp-relay=${proxy.udp}`, 'udp');
+
+    // test-url
+    result.appendIfPresent(`,test-url=${proxy['test-url']}`, 'test-url');
+
     return result.toString();
 }
 

backend/src/products/cron-sync-artifacts.js

@@ -51,12 +51,14 @@ async function doSync() {
         const body = JSON.parse(resp.body);
 
         for (const artifact of allArtifacts) {
-            artifact.updated = new Date().getTime();
-            // extract real url from gist
-            artifact.url = body.files[artifact.name].raw_url.replace(
-                /\/raw\/[^/]*\/(.*)/,
-                '/raw/$1',
-            );
+            if (artifact.sync) {
+                artifact.updated = new Date().getTime();
+                // extract real url from gist
+                artifact.url = body.files[artifact.name].raw_url.replace(
+                    /\/raw\/[^/]*\/(.*)/,
+                    '/raw/$1',
+                );
+            }
         }
 
         $.write(allArtifacts, ARTIFACTS_KEY);

backend/src/restful/subscriptions.js

@@ -66,8 +66,12 @@ async function getFlowInfo(req, res) {
         }
 
         // unit is KB
-        const upload = Number(flowHeaders.match(/upload=(\d+)/)[1]);
-        const download = Number(flowHeaders.match(/download=(\d+)/)[1]);
+        const uploadMatch = flowHeaders.match(/upload=(-?)(\d+)/)
+        const upload = Number(uploadMatch[1] + uploadMatch[2]);
+
+        const downloadMatch = flowHeaders.match(/download=(-?)(\d+)/)
+        const download = Number(downloadMatch[1] + downloadMatch[2]);
+
         const total = Number(flowHeaders.match(/total=(\d+)/)[1]);
 
         // optional expire timestamp

backend/src/restful/sync.js

@@ -201,12 +201,14 @@ async function syncAllArtifacts(_, res) {
         const body = JSON.parse(resp.body);
 
         for (const artifact of allArtifacts) {
-            artifact.updated = new Date().getTime();
-            // extract real url from gist
-            artifact.url = body.files[artifact.name].raw_url.replace(
-                /\/raw\/[^/]*\/(.*)/,
-                '/raw/$1',
-            );
+            if (artifact.sync) {
+                artifact.updated = new Date().getTime();
+                // extract real url from gist
+                artifact.url = body.files[artifact.name].raw_url.replace(
+                    /\/raw\/[^/]*\/(.*)/,
+                    '/raw/$1',
+                );
+            }
         }
 
         $.write(allArtifacts, ARTIFACTS_KEY);

backend/src/utils/download.js

@@ -1,13 +1,14 @@
-import { HTTP } from '@/vendor/open-api';
+import { HTTP, ENV } from '@/vendor/open-api';
 import { hex_md5 } from '@/vendor/md5';
 import resourceCache from '@/utils/resource-cache';
 
 const tasks = new Map();
 
 export default async function download(url, ua) {
+    const { isNode } = ENV();
     ua = ua || 'Quantumult%20X/1.0.29 (iPhone14,5; iOS 15.4.1)';
     const id = hex_md5(ua + url);
-    if (tasks.has(id)) {
+    if (!isNode && tasks.has(id)) {
         return tasks.get(id);
     }
 
@@ -39,6 +40,8 @@ export default async function download(url, ua) {
         }
     });
 
-    tasks.set(id, result);
+    if (!isNode) {
+        tasks.set(id, result);
+    }
     return result;
 }

backend/src/utils/resource-cache.js

@@ -16,8 +16,13 @@ class ResourceCache {
         let clear = false;
         Object.entries(this.resourceCache).forEach((entry) => {
             const [id, updated] = entry;
-            if (new Date().getTime() - updated > this.expires) {
+            if (!updated.time) {
+                // clear old version cache
+                delete this.resourceCache[id];
                 $.delete(`#${id}`);
+                clear = true;
+            }
+            if (new Date().getTime() - updated.time > this.expires) {
                 delete this.resourceCache[id];
                 clear = true;
             }
@@ -26,9 +31,6 @@ class ResourceCache {
     }
 
     revokeAll() {
-        Object.keys(this.resourceCache).forEach((id) => {
-            $.delete(`#${id}`);
-        });
         this.resourceCache = {};
         this._persist();
     }
@@ -38,17 +40,16 @@ class ResourceCache {
     }
 
     get(id) {
-        const updated = this.resourceCache[id];
+        const updated = this.resourceCache[id] && this.resourceCache[id].time;
         if (updated && new Date().getTime() - updated <= this.expires) {
-            return $.read(`#${id}`);
+            return this.resourceCache[id].data;
         }
         return null;
     }
 
     set(id, value) {
-        this.resourceCache[id] = new Date().getTime();
+        this.resourceCache[id] = { time: new Date().getTime(), data: value }
         this._persist();
-        $.write(value, `#${id}`);
     }
 }
 

backend/src/utils/script-resource-cache.js

@@ -0,0 +1,105 @@
+import $ from '@/core/app';
+import {
+    SCRIPT_RESOURCE_CACHE_KEY,
+    CSR_EXPIRATION_TIME_KEY,
+} from '@/constants';
+
+class ResourceCache {
+    constructor() {
+        this.expires = getExpiredTime();
+        if (!$.read(SCRIPT_RESOURCE_CACHE_KEY)) {
+            $.write('{}', SCRIPT_RESOURCE_CACHE_KEY);
+        }
+        this.resourceCache = JSON.parse($.read(SCRIPT_RESOURCE_CACHE_KEY));
+        this._cleanup();
+    }
+
+    _cleanup() {
+        // clear obsolete cached resource
+        let clear = false;
+        Object.entries(this.resourceCache).forEach((entry) => {
+            const [id, updated] = entry;
+            if (!updated.time) {
+                // clear old version cache
+                delete this.resourceCache[id];
+                $.delete(`#${id}`);
+                clear = true;
+            }
+            if (new Date().getTime() - updated.time > this.expires) {
+                delete this.resourceCache[id];
+                clear = true;
+            }
+        });
+        if (clear) this._persist();
+    }
+
+    revokeAll() {
+        this.resourceCache = {};
+        this._persist();
+    }
+
+    _persist() {
+        $.write(JSON.stringify(this.resourceCache), SCRIPT_RESOURCE_CACHE_KEY);
+    }
+
+    get(id) {
+        const updated = this.resourceCache[id] && this.resourceCache[id].time;
+        if (updated && new Date().getTime() - updated <= this.expires) {
+            return this.resourceCache[id].data;
+        }
+        return null;
+    }
+
+    gettime(id) {
+        const updated = this.resourceCache[id] && this.resourceCache[id].time;
+        if (updated && new Date().getTime() - updated <= this.expires) {
+            return this.resourceCache[id].time;
+        }
+        return null;
+    }
+
+    set(id, value) {
+        this.resourceCache[id] = { time: new Date().getTime(), data: value };
+        this._persist();
+    }
+}
+
+function getExpiredTime() {
+    console.log($.read(CSR_EXPIRATION_TIME_KEY));
+    if (!$.read(CSR_EXPIRATION_TIME_KEY)) {
+        $.write('1728e5', CSR_EXPIRATION_TIME_KEY); // 48 * 3600 * 1000
+    }
+    let expiration = 1728e5;
+    if ($.env.isLoon) {
+        const loont = {
+            // Loon 插件自义定
+            '1\u5206\u949f': 6e4,
+            '5\u5206\u949f': 3e5,
+            '10\u5206\u949f': 6e5,
+            '30\u5206\u949f': 18e5, // "30分钟"
+            '1\u5c0f\u65f6': 36e5,
+            '2\u5c0f\u65f6': 72e5,
+            '3\u5c0f\u65f6': 108e5,
+            '6\u5c0f\u65f6': 216e5,
+            '12\u5c0f\u65f6': 432e5,
+            '24\u5c0f\u65f6': 864e5,
+            '48\u5c0f\u65f6': 1728e5,
+            '72\u5c0f\u65f6': 2592e5, // "72小时"
+            '\u53c2\u6570\u4f20\u5165': 'readcachets', // "参数输入"
+        };
+        let intimed = $.read('#\u8282\u70b9\u7f13\u5b58\u6709\u6548\u671f'); // Loon #节点缓存有效期
+        console.log(intimed);
+        if (intimed in loont) {
+            expiration = loont[intimed];
+            if (expiration === 'readcachets') {
+                expiration = intimed;
+            }
+        }
+        return expiration;
+    } else {
+        expiration = $.read(CSR_EXPIRATION_TIME_KEY);
+        return expiration;
+    }
+}
+
+export default new ResourceCache();

config/Loon.plugin

@@ -4,12 +4,16 @@
 #!author=Peng-YM
 #!homepage=https://github.com/Peng-YM/Sub-Store
 #!icon=https://raw.githubusercontent.com/58xinian/icon/master/Sub-Store1.png
+#!select = 节点缓存有效期,1分钟,5分钟,10分钟,30分钟,1小时,2小时,3小时,6小时,12小时,24小时,48小时,72小时,参数传入
+
+[Rule]
+DOMAIN,sub-store.vercel.app,PROXY
 
 [MITM]
 hostname=sub.store
 
 [Script]
 http-request ^https?:\/\/sub\.store\/((download)|api\/(preview|sync|(utils\/node-info))) script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/sub-store-1.min.js, requires-body=true, timeout=120, tag=Sub-Store Core
-http-request https?:\/\/sub\.store script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/sub-store-0.min.js, requires-body=true, timeout=120, tag=Sub-Store Simple
+http-request ^https?:\/\/sub\.store script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/sub-store-0.min.js, requires-body=true, timeout=120, tag=Sub-Store Simple
 
 cron "0 0 * * *" script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js, tag=Sub-Store Sync

scripts/ip-flag.js

@@ -17,8 +17,13 @@ class ResourceCache {
         let clear = false;
         Object.entries(this.resourceCache).forEach((entry) => {
             const [id, updated] = entry;
-            if (new Date().getTime() - updated > this.expires) {
+            if (!updated.time) {
+                // clear old version cache
+                delete this.resourceCache[id];
                 $.delete(`#${id}`);
+                clear = true;
+            }
+            if (new Date().getTime() - updated.time > this.expires) {
                 delete this.resourceCache[id];
                 clear = true;
             }
@@ -27,9 +32,6 @@ class ResourceCache {
     }
 
     revokeAll() {
-        Object.keys(this.resourceCache).forEach((id) => {
-            $.delete(`#${id}`);
-        });
         this.resourceCache = {};
         this._persist();
     }
@@ -39,17 +41,16 @@ class ResourceCache {
     }
 
     get(id) {
-        const updated = this.resourceCache[id];
+        const updated = this.resourceCache[id] && this.resourceCache[id].time;
         if (updated && new Date().getTime() - updated <= this.expires) {
-            return $.read(`#${id}`);
+            return this.resourceCache[id].data;
         }
         return null;
     }
 
     set(id, value) {
-        this.resourceCache[id] = new Date().getTime();
+        this.resourceCache[id] = { time: new Date().getTime(), data: value }
         this._persist();
-        $.write(value, `#${id}`);
     }
 }