feat: 支持按顺序合并本地和远程订阅(前端版本 > 2.14.14 可输入)

feat: add sync task for qx

.github/workflows/main.yml

@@ -44,7 +44,7 @@ jobs:
         run: |
           cd backend
           SUBSTORE_RELEASE=`node --eval="process.stdout.write(require('./package.json').version)"`
-          echo "::set-output name=release_tag::$SUBSTORE_RELEASE"
+          echo "release_tag=$SUBSTORE_RELEASE" >> $GITHUB_OUTPUT
       - name: Prepare release
         run: |
           cd backend

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.14.93",
+  "version": "2.14.106",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {

backend/src/core/proxy-utils/parsers/index.js

@@ -33,7 +33,9 @@ function URI_SS() {
         const serverAndPort = serverAndPortArray[1];
         const portIdx = serverAndPort.lastIndexOf(':');
         proxy.server = serverAndPort.substring(0, portIdx);
-        proxy.port = serverAndPort.substring(portIdx + 1);
+        proxy.port = `${serverAndPort.substring(portIdx + 1)}`.match(
+            /\d+/,
+        )?.[0];
 
         const userInfo = userInfoStr.split(':');
         proxy.cipher = userInfo[0];

backend/src/core/proxy-utils/processors/index.js

@@ -451,7 +451,9 @@ function ResolveDomainOperator({ provider }) {
             const limit = 15; // more than 20 concurrency may result in surge TCP connection shortage.
             const totalDomain = [
                 ...new Set(
-                    proxies.filter((p) => !isIP(p.server)).map((c) => c.server),
+                    proxies
+                        .filter((p) => !isIP(p.server) && !p['no-resolve'])
+                        .map((c) => c.server),
                 ),
             ];
             const totalBatch = Math.ceil(totalDomain.length / limit);
@@ -475,8 +477,15 @@ function ResolveDomainOperator({ provider }) {
                 }
                 await Promise.all(currentBatch);
             }
-            proxies.forEach((proxy) => {
-                proxy.server = results[proxy.server] || proxy.server;
+            proxies.forEach((p) => {
+                if (!p['no-resolve']) {
+                    if (results[p.server]) {
+                        p.server = results[p.server];
+                        p.resolved = true;
+                    } else {
+                        p.resolved = false;
+                    }
+                }
             });
 
             return proxies;

backend/src/core/proxy-utils/producers/stash.js

@@ -112,6 +112,14 @@ export default function Stash_Producer() {
                             proxy['up-speed'] = proxy.up;
                             delete proxy.up;
                         }
+                        if (isPresent(proxy, 'down-speed')) {
+                            proxy['down-speed'] =
+                                `${proxy['down-speed']}`.match(/\d+/)?.[0] || 0;
+                        }
+                        if (isPresent(proxy, 'up-speed')) {
+                            proxy['up-speed'] =
+                                `${proxy['up-speed']}`.match(/\d+/)?.[0] || 0;
+                        }
                     } else if (proxy.type === 'hysteria2') {
                         if (
                             isPresent(proxy, 'password') &&
@@ -141,6 +149,14 @@ export default function Stash_Producer() {
                             proxy['up-speed'] = proxy.up;
                             delete proxy.up;
                         }
+                        if (isPresent(proxy, 'down-speed')) {
+                            proxy['down-speed'] =
+                                `${proxy['down-speed']}`.match(/\d+/)?.[0] || 0;
+                        }
+                        if (isPresent(proxy, 'up-speed')) {
+                            proxy['up-speed'] =
+                                `${proxy['up-speed']}`.match(/\d+/)?.[0] || 0;
+                        }
                     } else if (proxy.type === 'wireguard') {
                         proxy.keepalive =
                             proxy.keepalive ?? proxy['persistent-keepalive'];

backend/src/restful/download.js

@@ -20,18 +20,22 @@ async function downloadSubscription(req, res) {
         req.query.target || getPlatformFromHeaders(req.headers) || 'JSON';
 
     $.info(`正在下载订阅：${name}`);
-    let { url, ua, content } = req.query;
+    let { url, ua, content, mergeSources } = req.query;
     if (url) {
         url = decodeURIComponent(url);
-        $.info(`指定 url: ${url}`);
+        $.info(`指定远程订阅 URL: ${url}`);
     }
     if (ua) {
         ua = decodeURIComponent(ua);
-        $.info(`指定 ua: ${ua}`);
+        $.info(`指定远程订阅 User-Agent: ${ua}`);
     }
     if (content) {
         content = decodeURIComponent(content);
-        $.info(`指定 content: ${content}`);
+        $.info(`指定本地订阅: ${content}`);
+    }
+    if (mergeSources) {
+        mergeSources = decodeURIComponent(mergeSources);
+        $.info(`指定合并来源: ${mergeSources}`);
     }
 
     const allSubs = $.read(SUBS_KEY);
@@ -45,6 +49,7 @@ async function downloadSubscription(req, res) {
                 url,
                 ua,
                 content,
+                mergeSources,
             });
 
             if (sub.source !== 'local' || url) {

backend/src/restful/index.js

@@ -15,8 +15,13 @@ import registerMiscRoutes from './miscs';
 import registerNodeInfoRoutes from './node-info';
 
 export default function serve() {
-    const $app = express({ substore: $ });
-
+    let port;
+    let host;
+    if ($.env.isNode) {
+        port = eval('process.env.SUB_STORE_BACKEND_API_PORT');
+        host = eval('process.env.SUB_STORE_BACKEND_API_HOST');
+    }
+    const $app = express({ substore: $, port, host });
     // register routes
     registerCollectionRoutes($app);
     registerSubscriptionRoutes($app);

backend/src/restful/preview.js

@@ -16,11 +16,20 @@ async function compareSub(req, res) {
         const sub = req.body;
         const target = req.query.target || 'JSON';
         let content;
-        if (sub.source === 'local') {
+        if (
+            sub.source === 'local' &&
+            !['localFirst', 'remoteFirst'].includes(sub.mergeSources)
+        ) {
             content = sub.content;
         } else {
             try {
-                content = await download(sub.url, sub.ua);
+                content = await Promise.all(
+                    sub.url
+                        .split(/[\r\n]+/)
+                        .map((i) => i.trim())
+                        .filter((i) => i.length)
+                        .map((url) => download(url, sub.ua)),
+                );
             } catch (err) {
                 failed(
                     res,
@@ -32,9 +41,16 @@ async function compareSub(req, res) {
                 );
                 return;
             }
+            if (sub.mergeSources === 'localFirst') {
+                content.unshift(sub.content);
+            } else if (sub.mergeSources === 'remoteFirst') {
+                content.push(sub.content);
+            }
         }
         // parse proxies
-        const original = ProxyUtils.parse(content);
+        const original = (Array.isArray(content) ? content : [content])
+            .map((i) => ProxyUtils.parse(i))
+            .flat();
 
         // add id
         original.forEach((proxy, i) => {
@@ -77,13 +93,31 @@ async function compareCollection(req, res) {
                 const sub = findByName(allSubs, name);
                 try {
                     let raw;
-                    if (sub.source === 'local') {
+                    if (
+                        sub.source === 'local' &&
+                        !['localFirst', 'remoteFirst'].includes(
+                            sub.mergeSources,
+                        )
+                    ) {
                         raw = sub.content;
                     } else {
-                        raw = await download(sub.url, sub.ua);
+                        raw = await Promise.all(
+                            sub.url
+                                .split(/[\r\n]+/)
+                                .map((i) => i.trim())
+                                .filter((i) => i.length)
+                                .map((url) => download(url, sub.ua)),
+                        );
+                        if (sub.mergeSources === 'localFirst') {
+                            raw.unshift(sub.content);
+                        } else if (sub.mergeSources === 'remoteFirst') {
+                            raw.push(sub.content);
+                        }
                     }
                     // parse proxies
-                    let currentProxies = ProxyUtils.parse(raw);
+                    let currentProxies = (Array.isArray(raw) ? raw : [raw])
+                        .map((i) => ProxyUtils.parse(i))
+                        .flat();
 
                     currentProxies.forEach((proxy) => {
                         proxy.subName = sub.name;

backend/src/restful/sync.js

@@ -22,24 +22,60 @@ export default function register($app) {
     $app.get('/api/sync/artifact/:name', syncArtifact);
 }
 
-async function produceArtifact({ type, name, platform, url, ua, content }) {
+async function produceArtifact({
+    type,
+    name,
+    platform,
+    url,
+    ua,
+    content,
+    mergeSources,
+}) {
     platform = platform || 'JSON';
 
     if (type === 'subscription') {
         const allSubs = $.read(SUBS_KEY);
         const sub = findByName(allSubs, name);
         let raw;
-        if (url) {
-            raw = await download(url, ua);
-        } else if (content) {
+        if (content && !['localFirst', 'remoteFirst'].includes(mergeSources)) {
             raw = content;
-        } else if (sub.source === 'local') {
+        } else if (url) {
+            raw = await Promise.all(
+                url
+                    .split(/[\r\n]+/)
+                    .map((i) => i.trim())
+                    .filter((i) => i.length)
+                    .map((url) => download(url, ua)),
+            );
+            if (mergeSources === 'localFirst') {
+                raw.unshift(content);
+            } else if (mergeSources === 'remoteFirst') {
+                raw.push(content);
+            }
+        } else if (
+            sub.source === 'local' &&
+            !['localFirst', 'remoteFirst'].includes(sub.mergeSources)
+        ) {
             raw = sub.content;
         } else {
-            raw = await download(sub.url, sub.ua);
+            raw = await Promise.all(
+                sub.url
+                    .split(/[\r\n]+/)
+                    .map((i) => i.trim())
+                    .filter((i) => i.length)
+                    .map((url) => download(url, sub.ua)),
+            );
+            if (sub.mergeSources === 'localFirst') {
+                raw.unshift(sub.content);
+            } else if (sub.mergeSources === 'remoteFirst') {
+                raw.push(sub.content);
+            }
         }
         // parse proxies
-        let proxies = ProxyUtils.parse(raw);
+        let proxies = (Array.isArray(raw) ? raw : [raw])
+            .map((i) => ProxyUtils.parse(i))
+            .flat();
+
         proxies.forEach((proxy) => {
             proxy.subName = sub.name;
         });
@@ -87,13 +123,31 @@ async function produceArtifact({ type, name, platform, url, ua, content }) {
                 try {
                     $.info(`正在处理子订阅：${sub.name}...`);
                     let raw;
-                    if (sub.source === 'local') {
+                    if (
+                        sub.source === 'local' &&
+                        !['localFirst', 'remoteFirst'].includes(
+                            sub.mergeSources,
+                        )
+                    ) {
                         raw = sub.content;
                     } else {
-                        raw = await download(sub.url, sub.ua);
+                        raw = await await Promise.all(
+                            sub.url
+                                .split(/[\r\n]+/)
+                                .map((i) => i.trim())
+                                .filter((i) => i.length)
+                                .map((url) => download(url, sub.ua)),
+                        );
+                        if (sub.mergeSources === 'localFirst') {
+                            raw.unshift(sub.content);
+                        } else if (sub.mergeSources === 'remoteFirst') {
+                            raw.push(sub.content);
+                        }
                     }
                     // parse proxies
-                    let currentProxies = ProxyUtils.parse(raw);
+                    let currentProxies = (Array.isArray(raw) ? raw : [raw])
+                        .map((i) => ProxyUtils.parse(i))
+                        .flat();
 
                     currentProxies.forEach((proxy) => {
                         proxy.subName = sub.name;

backend/src/utils/download.js

@@ -1,4 +1,4 @@
-import { FILES_KEY, MODULES_KEY } from '@/constants';
+import { FILES_KEY, MODULES_KEY, SETTINGS_KEY } from '@/constants';
 import { findByName } from '@/utils/database';
 import { HTTP, ENV } from '@/vendor/open-api';
 import { hex_md5 } from '@/vendor/md5';
@@ -45,7 +45,8 @@ export default async function download(url, ua) {
     }
 
     const { isNode } = ENV();
-    ua = ua || 'Quantumult%20X/1.0.29 (iPhone14,5; iOS 15.4.1)';
+    const { defaultUserAgent } = $.read(SETTINGS_KEY);
+    ua = ua || defaultUserAgent || 'clash.meta';
     const id = hex_md5(ua + url);
     if (!isNode && tasks.has(id)) {
         return tasks.get(id);
@@ -63,6 +64,7 @@ export default async function download(url, ua) {
         if (!$arguments?.noCache && cached) {
             resolve(cached);
         } else {
+            $.info(`Downloading...\nUser-Agent: ${ua}\nURL: ${url}`);
             http.get(url)
                 .then((resp) => {
                     const body = resp.body;

backend/src/vendor/express.js

@@ -1,8 +1,9 @@
 /* eslint-disable no-undef */
 import { ENV } from './open-api';
 
-export default function express({ substore: $, port }) {
+export default function express({ substore: $, port, host }) {
     port = port || 3000;
+    host = host || '::';
     const { isNode } = ENV();
     const DEFAULT_HEADERS = {
         'Content-Type': 'text/plain;charset=UTF-8',
@@ -29,8 +30,9 @@ export default function express({ substore: $, port }) {
 
         // adapter
         app.start = () => {
-            app.listen(port, () => {
-                $.info(`Express started on port: ${port}`);
+            const listener = app.listen(port, host, () => {
+                const { address, port } = listener.address();
+                $.info(`Express started on ${address}:${port}`);
             });
         };
         return app;

config/QX-Task.json

@@ -0,0 +1,7 @@
+{
+    "name":"Sub-Store",
+    "description":"",
+    "task":[
+      "0 0 * * * https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js, tag=Sub-Store Sync, img-url=https://raw.githubusercontent.com/58xinian/icon/master/Sub-Store1.png"
+    ]
+}