feat: Surge includeUnsupportedProxy 去除 HTTP 传输层(不一定能通, 由服务端配置确定)

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.14.317",
+  "version": "2.14.326",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {

backend/src/core/proxy-utils/index.js

@@ -199,6 +199,10 @@ function produce(proxies, targetPlatform, type, opts = {}) {
     );
 
     proxies = proxies.map((proxy) => {
+        proxy._subName = proxy.subName;
+        proxy._collectionName = proxy.collectionName;
+        proxy._resolved = proxy.resolved;
+
         if (!isNotBlank(proxy.name)) {
             proxy.name = `${proxy.type} ${proxy.server}:${proxy.port}`;
         }

backend/src/core/proxy-utils/parsers/index.js

@@ -340,6 +340,8 @@ function URI_VMess() {
                         transportPath = Array.isArray(transportPath)
                             ? transportPath[0]
                             : transportPath;
+                    } else {
+                        transportPath = '/';
                     }
                 }
                 if (transportPath || transportHost) {

backend/src/core/proxy-utils/processors/index.js

@@ -512,12 +512,17 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
     return {
         name: 'Resolve Domain Operator',
         func: async (proxies) => {
+            proxies.forEach((p, i) => {
+                if (!p['_no-resolve'] && p['no-resolve']) {
+                    proxies[i]['_no-resolve'] = p['no-resolve'];
+                }
+            });
             const results = {};
             const limit = 15; // more than 20 concurrency may result in surge TCP connection shortage.
             const totalDomain = [
                 ...new Set(
                     proxies
-                        .filter((p) => !isIP(p.server) && !p['no-resolve'])
+                        .filter((p) => !isIP(p.server) && !p['_no-resolve'])
                         .map((c) => c.server),
                 ),
             ];
@@ -543,7 +548,7 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
                 await Promise.all(currentBatch);
             }
             proxies.forEach((p) => {
-                if (!p['no-resolve']) {
+                if (!p['_no-resolve']) {
                     if (results[p.server]) {
                         if (_type === 'IP4P') {
                             const { server, port } = parseIP4P(
@@ -578,7 +583,7 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
 
             return proxies.filter((p) => {
                 if (filter === 'removeFailed') {
-                    return isIP(p.server) || p['no-resolve'] || p.resolved;
+                    return isIP(p.server) || p['_no-resolve'] || p.resolved;
                 } else if (filter === 'IPOnly') {
                     return isIP(p.server);
                 } else if (filter === 'IPv4Only') {

backend/src/core/proxy-utils/producers/clash.js

@@ -158,6 +158,7 @@ export default function Clash_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/clashmeta.js

@@ -173,6 +173,7 @@ export default function ClashMeta_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/shadowrocket.js

@@ -176,6 +176,7 @@ export default function ShadowRocket_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/stash.js

@@ -265,6 +265,7 @@ export default function Stash_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/surfboard.js

@@ -6,7 +6,7 @@ const targetPlatform = 'Surfboard';
 
 export default function Surfboard_Producer() {
     const produce = (proxy) => {
-        proxy.name = proxy.name.replace(/=/g, '');
+        proxy.name = proxy.name.replace(/=|,/g, '');
         switch (proxy.type) {
             case 'ss':
                 return shadowsocks(proxy);

backend/src/core/proxy-utils/producers/surge.js

@@ -14,13 +14,14 @@ const ipVersions = {
 
 export default function Surge_Producer() {
     const produce = (proxy, type, opts = {}) => {
+        proxy.name = proxy.name.replace(/=|,/g, '');
         switch (proxy.type) {
             case 'ss':
                 return shadowsocks(proxy);
             case 'trojan':
                 return trojan(proxy);
             case 'vmess':
-                return vmess(proxy);
+                return vmess(proxy, opts['include-unsupported-proxy']);
             case 'http':
                 return http(proxy);
             case 'socks5':
@@ -263,7 +264,7 @@ function trojan(proxy) {
     return result.toString();
 }
 
-function vmess(proxy) {
+function vmess(proxy, includeUnsupportedProxy) {
     const result = new Result(proxy);
     result.append(`${proxy.name}=${proxy.type},${proxy.server},${proxy.port}`);
     result.appendIfPresent(`,username=${proxy.uuid}`, 'uuid');
@@ -277,7 +278,7 @@ function vmess(proxy) {
     );
 
     // transport
-    handleTransport(result, proxy);
+    handleTransport(result, proxy, includeUnsupportedProxy);
 
     // AEAD
     if (isPresent(proxy, 'aead')) {
@@ -1012,7 +1013,7 @@ function hysteria2(proxy) {
     return result.toString();
 }
 
-function handleTransport(result, proxy) {
+function handleTransport(result, proxy, includeUnsupportedProxy) {
     if (isPresent(proxy, 'network')) {
         if (proxy.network === 'ws') {
             result.append(`,ws=true`);
@@ -1038,7 +1039,13 @@ function handleTransport(result, proxy) {
                 }
             }
         } else {
-            throw new Error(`network ${proxy.network} is unsupported`);
+            if (includeUnsupportedProxy && ['http'].includes(proxy.network)) {
+                $.info(
+                    `Include Unsupported Proxy: nework ${proxy.network} -> tcp`,
+                );
+            } else {
+                throw new Error(`network ${proxy.network} is unsupported`);
+            }
         }
     }
 }

backend/src/core/proxy-utils/producers/uri.js

@@ -10,6 +10,7 @@ export default function URI_Producer() {
         delete proxy.collectionName;
         delete proxy.id;
         delete proxy.resolved;
+        delete proxy['no-resolve'];
         for (const key in proxy) {
             if (proxy[key] == null || /^_/i.test(key)) {
                 delete proxy[key];

backend/src/products/cron-sync-artifacts.js

@@ -11,17 +11,65 @@ import { syncToGist } from '@/restful/artifacts';
 import { findByName } from '@/utils/database';
 
 !(async function () {
-    const settings = $.read(SETTINGS_KEY);
-    // if GitHub token is not configured
-    if (!settings.githubUser || !settings.gistToken) return;
+    let arg;
+    if (typeof $argument != 'undefined') {
+        arg = Object.fromEntries(
+            // eslint-disable-next-line no-undef
+            $argument.split('&').map((item) => item.split('=')),
+        );
+    } else {
+        arg = {};
+    }
+    let sub_names = (arg?.subscription ?? arg?.sub ?? '')
+        .split(/,|，/g)
+        .map((i) => i.trim())
+        .filter((i) => i.length > 0)
+        .map((i) => decodeURIComponent(i));
+    let col_names = (arg?.collection ?? arg?.col ?? '')
+        .split(/,|，/g)
+        .map((i) => i.trim())
+        .filter((i) => i.length > 0)
+        .map((i) => decodeURIComponent(i));
+    if (sub_names.length > 0 || col_names.length > 0) {
+        if (sub_names.length > 0)
+            await produceArtifacts(sub_names, 'subscription');
+        if (col_names.length > 0)
+            await produceArtifacts(col_names, 'collection');
+    } else {
+        const settings = $.read(SETTINGS_KEY);
+        // if GitHub token is not configured
+        if (!settings.githubUser || !settings.gistToken) return;
 
-    const artifacts = $.read(ARTIFACTS_KEY);
-    if (!artifacts || artifacts.length === 0) return;
+        const artifacts = $.read(ARTIFACTS_KEY);
+        if (!artifacts || artifacts.length === 0) return;
 
-    const shouldSync = artifacts.some((artifact) => artifact.sync);
-    if (shouldSync) await doSync();
+        const shouldSync = artifacts.some((artifact) => artifact.sync);
+        if (shouldSync) await doSync();
+    }
 })().finally(() => $.done());
 
+async function produceArtifacts(names, type) {
+    try {
+        if (names.length > 0) {
+            $.info(`produceArtifacts ${type} 开始: ${names.join(', ')}`);
+            await Promise.all(
+                names.map(async (name) => {
+                    try {
+                        await produceArtifact({
+                            type,
+                            name,
+                        });
+                    } catch (e) {
+                        $.error(`${type} ${name} error: ${e.message ?? e}`);
+                    }
+                }),
+            );
+            $.info(`produceArtifacts ${type} 完成: ${names.join(', ')}`);
+        }
+    } catch (e) {
+        $.error(`produceArtifacts error: ${e.message ?? e}`);
+    }
+}
 async function doSync() {
     console.log(
         `

backend/src/utils/download.js

@@ -57,7 +57,11 @@ export default async function download(
                 timeout,
                 proxy,
                 true,
-            );
+            ).catch((e) => {
+                $.error(
+                    `乐观缓存: URL ${url} 更新缓存发生错误 ${e.message ?? e}`,
+                );
+            });
             return cached;
         }
     }

backend/src/utils/geo.js

@@ -438,11 +438,21 @@ export class MMDB {
             const countryFile =
                 country || eval('process.env.SUB_STORE_MMDB_COUNTRY_PATH');
             const asnFile = asn || eval('process.env.SUB_STORE_MMDB_ASN_PATH');
+            // $.info(
+            //     `GeoLite2 Country MMDB: ${countryFile}, exists: ${fs.existsSync(
+            //         countryFile,
+            //     )}`,
+            // );
             if (countryFile) {
                 this.countryReader = Reader.openBuffer(
                     fs.readFileSync(countryFile),
                 );
             }
+            // $.info(
+            //     `GeoLite2 ASN MMDB: ${asnFile}, exists: ${fs.existsSync(
+            //         asnFile,
+            //     )}`,
+            // );
             if (asnFile) {
                 if (!fs.existsSync(asnFile))
                     throw new Error('GeoLite2 ASN MMDB does not exist');

backend/src/vendor/open-api.js

@@ -382,6 +382,12 @@ export function HTTP(defaultOptions = { baseURL: '' }) {
                         url: options.url,
                         headers: options.headers,
                         body: options.body,
+                        options: {
+                            Proxy: options.proxy,
+                            Timeout: options.timeout
+                                ? options.timeout / 1000
+                                : 15,
+                        },
                     });
                     resolve({
                         statusCode: response.status,

config/Surge-Beta.sgmodule

@@ -1,8 +1,8 @@
 #!name=Sub-Store(β)
 #!desc=支持 Surge 正式版的参数设置功能. 测落地功能 ability: http-client-policy, 同步配置的定时 cronexp: 55 23 * * *
 #!category=订阅管理
-#!arguments=ability:http-client-policy,cronexp:55 23 * * *,sync:"Sub-Store Sync",timeout:120,engine:auto
-#!arguments-desc=\n1️⃣ ability\n\n默认已开启测落地能力\n需要配合脚本操作\n如 https://raw.githubusercontent.com/Keywos/rule/main/cname.js\n填写任意其他值关闭\n\n2️⃣ cronexp\n\n同步配置定时任务\n默认为每天 23 点 55 分\n\n定时任务指定时将订阅/文件上传到私有 Gist. 在前端, 叫做 '同步' 或 '同步配置'\n\n3️⃣ sync\n\n自定义定时任务名\n便于在脚本编辑器中选择\n若设为 # 可取消定时任务\n\n4️⃣ timeout\n\n超时, 单位为秒\n\n5️⃣ engine\n\n默认为自动使用 webview 引擎, 可设为指定 jsc, 但 jsc 容易爆内存
+#!arguments=ability:http-client-policy,cronexp:55 23 * * *,sync:"Sub-Store Sync",timeout:120,engine:auto,produce:"# Sub-Store Produce",produce_cronexp:50 */6 * * *,produce_sub:"sub1,sub2",produce_col:"col1,col2"
+#!arguments-desc=\n1️⃣ ability\n\n默认已开启测落地能力\n需要配合脚本操作\n如 https://raw.githubusercontent.com/Keywos/rule/main/cname.js\n填写任意其他值关闭\n\n2️⃣ cronexp\n\n同步配置定时任务\n默认为每天 23 点 55 分\n\n定时任务指定时将订阅/文件上传到私有 Gist. 在前端, 叫做 '同步' 或 '同步配置'\n\n3️⃣ sync\n\n自定义定时任务名\n便于在脚本编辑器中选择\n若设为 # 可取消定时任务\n\n4️⃣ timeout\n\n脚本超时, 单位为秒\n\n5️⃣ engine\n\n默认为自动使用 webview 引擎, 可设为指定 jsc, 但 jsc 容易爆内存\n\n6️⃣ produce\n\n自定义处理订阅的定时任务名\n一般用于定时处理耗时较长的订阅, 以更新缓存\n这样 Surge 中拉取的时候就能用到缓存, 不至于总是超时\n若设为 # 可取消此定时任务\n默认不开启\n\n7️⃣ produce_cronexp\n\n配置处理订阅的定时任务\n\n默认为每 6 小时\n\n9️⃣ produce_sub\n\n自定义需定时处理的单条订阅名\n多个用 , 连接\n\n🔟 produce_col\n\n自定义需定时处理的组合订阅名\n多个用 , 连接\n\n⚠️ 注意: 是 名称(name) 不是 显示名称(displayName)\n如果名称需要编码, 请编码后再用 , 连接\n顺序: 并发执行单条订阅, 然后并发执行组合订阅
 
 [MITM]
 hostname = %APPEND% sub.store
@@ -12,4 +12,6 @@ Sub-Store Core=type=http-request,pattern=^https?:\/\/sub\.store\/((download)|api
 
 Sub-Store Simple=type=http-request,pattern=^https?:\/\/sub\.store,script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/sub-store-0.min.js,requires-body=true,timeout={{{timeout}}},engine={{{engine}}}
 
-{{{sync}}}=type=cron,cronexp="{{{cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}}
+{{{sync}}}=type=cron,cronexp="{{{cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}}
+
+{{{produce}}}=type=cron,cronexp="{{{produce_cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}},argument="sub={{{produce_sub}}}&col={{{produce_col}}}"

config/Surge.sgmodule

@@ -1,8 +1,8 @@
 #!name=Sub-Store
 #!desc=支持 Surge 正式版的参数设置功能. 测落地功能 ability: http-client-policy, 同步配置的定时 cronexp: 55 23 * * *
 #!category=订阅管理
-#!arguments=ability:http-client-policy,cronexp:55 23 * * *,sync:"Sub-Store Sync",timeout:120,engine:auto
-#!arguments-desc=\n1️⃣ ability\n\n默认已开启测落地能力\n需要配合脚本操作\n如 https://raw.githubusercontent.com/Keywos/rule/main/cname.js\n填写任意其他值关闭\n\n2️⃣ cronexp\n\n同步配置定时任务\n默认为每天 23 点 55 分\n\n定时任务指定时将订阅/文件上传到私有 Gist. 在前端, 叫做 '同步' 或 '同步配置'\n\n3️⃣ sync\n\n自定义定时任务名\n便于在脚本编辑器中选择\n若设为 # 可取消定时任务\n\n4️⃣ timeout\n\n超时, 单位为秒\n\n5️⃣ engine\n\n默认为自动使用 webview 引擎, 可设为指定 jsc, 但 jsc 容易爆内存
+#!arguments=ability:http-client-policy,cronexp:55 23 * * *,sync:"Sub-Store Sync",timeout:120,engine:auto,produce:"# Sub-Store Produce",produce_cronexp:50 */6 * * *,produce_sub:"sub1,sub2",produce_col:"col1,col2"
+#!arguments-desc=\n1️⃣ ability\n\n默认已开启测落地能力\n需要配合脚本操作\n如 https://raw.githubusercontent.com/Keywos/rule/main/cname.js\n填写任意其他值关闭\n\n2️⃣ cronexp\n\n同步配置定时任务\n默认为每天 23 点 55 分\n\n定时任务指定时将订阅/文件上传到私有 Gist. 在前端, 叫做 '同步' 或 '同步配置'\n\n3️⃣ sync\n\n自定义定时任务名\n便于在脚本编辑器中选择\n若设为 # 可取消定时任务\n\n4️⃣ timeout\n\n脚本超时, 单位为秒\n\n5️⃣ engine\n\n默认为自动使用 webview 引擎, 可设为指定 jsc, 但 jsc 容易爆内存\n\n6️⃣ produce\n\n自定义处理订阅的定时任务名\n一般用于定时处理耗时较长的订阅, 以更新缓存\n这样 Surge 中拉取的时候就能用到缓存, 不至于总是超时\n若设为 # 可取消此定时任务\n默认不开启\n\n7️⃣ produce_cronexp\n\n配置处理订阅的定时任务\n\n默认为每 6 小时\n\n9️⃣ produce_sub\n\n自定义需定时处理的单条订阅名\n多个用 , 连接\n\n🔟 produce_col\n\n自定义需定时处理的组合订阅名\n多个用 , 连接\n\n⚠️ 注意: 是 名称(name) 不是 显示名称(displayName)\n如果名称需要编码, 请编码后再用 , 连接\n顺序: 并发执行单条订阅, 然后并发执行组合订阅
 
 [MITM]
 hostname = %APPEND% sub.store
@@ -12,4 +12,6 @@ Sub-Store Core=type=http-request,pattern=^https?:\/\/sub\.store\/((download)|api
 
 Sub-Store Simple=type=http-request,pattern=^https?:\/\/sub\.store,script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/sub-store-0.min.js,requires-body=true,timeout={{{timeout}}},engine={{{engine}}}
 
-{{{sync}}}=type=cron,cronexp="{{{cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}}
+{{{sync}}}=type=cron,cronexp="{{{cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}}
+
+{{{produce}}}=type=cron,cronexp="{{{produce_cronexp}}}",wake-system=1,timeout={{{timeout}}},script-path=https://github.com/sub-store-org/Sub-Store/releases/latest/download/cron-sync-artifacts.min.js,engine={{{engine}}},argument="sub={{{produce_sub}}}&col={{{produce_col}}}"

scripts/demo.js

@@ -7,10 +7,12 @@ function operator(proxies = [], targetPlatform, context) {
   // proxies 为传入的内部节点数组
   // 结构大致参考了 Clash.Meta(mihomo) 有私货
   // 可在预览界面点击节点查看 JSON 结构 或查看 `target=JSON` 的通用订阅
-  // 1. `no-resolve` 为不解析域名
-  // 2. 域名解析后 会多一个 `resolved` 字段
+  // 1. `_no-resolve` 为不解析域名
+  // 2. 域名解析后 会多一个 `_resolved` 字段
   // 3. 域名解析后会有`_IPv4`, `_IPv6`, `_IP`(若有多个步骤, 只取第一次成功的 v4 或 v6 数据), `_domain` 字段
   // 4. 节点字段 `exec` 为 `ssr-local` 路径, 默认 `/usr/local/bin/ssr-local`; 端口从 10000 开始递增(暂不支持配置)
+  // 5. `_subName` 为单条订阅名
+  // 6. `_collectionName` 为组合订阅名
 
   // $arguments 为传入的脚本参数