feat: ProxyUtils.removeFlag

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sub-store",
-  "version": "2.14.321",
+  "version": "2.14.331",
   "description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
   "main": "src/main.js",
   "scripts": {

backend/src/core/proxy-utils/index.js

@@ -15,7 +15,7 @@ import $ from '@/core/app';
 import { FILES_KEY, MODULES_KEY } from '@/constants';
 import { findByName } from '@/utils/database';
 import { produceArtifact } from '@/restful/sync';
-import { getFlag, getISO, MMDB } from '@/utils/geo';
+import { getFlag, removeFlag, getISO, MMDB } from '@/utils/geo';
 import Gist from '@/utils/gist';
 
 function preprocess(raw) {
@@ -276,6 +276,7 @@ export const ProxyUtils = {
     isIP,
     yaml: YAML,
     getFlag,
+    removeFlag,
     getISO,
     MMDB,
     Gist,

backend/src/core/proxy-utils/parsers/index.js

@@ -340,6 +340,8 @@ function URI_VMess() {
                         transportPath = Array.isArray(transportPath)
                             ? transportPath[0]
                             : transportPath;
+                    } else {
+                        transportPath = '/';
                     }
                 }
                 if (transportPath || transportHost) {

backend/src/core/proxy-utils/processors/index.js

@@ -2,7 +2,7 @@ import resourceCache from '@/utils/resource-cache';
 import scriptResourceCache from '@/utils/script-resource-cache';
 import { isIPv4, isIPv6 } from '@/utils';
 import { FULL } from '@/utils/logical';
-import { getFlag } from '@/utils/geo';
+import { getFlag, removeFlag } from '@/utils/geo';
 import lodash from 'lodash';
 import $ from '@/core/app';
 import { hex_md5 } from '@/vendor/md5';
@@ -512,12 +512,17 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
     return {
         name: 'Resolve Domain Operator',
         func: async (proxies) => {
+            proxies.forEach((p, i) => {
+                if (!p['_no-resolve'] && p['no-resolve']) {
+                    proxies[i]['_no-resolve'] = p['no-resolve'];
+                }
+            });
             const results = {};
             const limit = 15; // more than 20 concurrency may result in surge TCP connection shortage.
             const totalDomain = [
                 ...new Set(
                     proxies
-                        .filter((p) => !isIP(p.server) && !p['no-resolve'])
+                        .filter((p) => !isIP(p.server) && !p['_no-resolve'])
                         .map((c) => c.server),
                 ),
             ];
@@ -543,7 +548,7 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
                 await Promise.all(currentBatch);
             }
             proxies.forEach((p) => {
-                if (!p['no-resolve']) {
+                if (!p['_no-resolve']) {
                     if (results[p.server]) {
                         if (_type === 'IP4P') {
                             const { server, port } = parseIP4P(
@@ -578,7 +583,7 @@ function ResolveDomainOperator({ provider, type: _type, filter, cache }) {
 
             return proxies.filter((p) => {
                 if (filter === 'removeFailed') {
-                    return isIP(p.server) || p['no-resolve'] || p.resolved;
+                    return isIP(p.server) || p['_no-resolve'] || p.resolved;
                 } else if (filter === 'IPOnly') {
                     return isIP(p.server);
                 } else if (filter === 'IPv4Only') {
@@ -864,13 +869,6 @@ function clone(object) {
     return JSON.parse(JSON.stringify(object));
 }
 
-// remove flag
-function removeFlag(str) {
-    return str
-        .replace(/[\uD83C][\uDDE6-\uDDFF][\uD83C][\uDDE6-\uDDFF]|🏴‍☠️|🏳️‍🌈/g, '')
-        .trim();
-}
-
 function createDynamicFunction(name, script, $arguments) {
     const flowUtils = {
         getFlowField,

backend/src/core/proxy-utils/producers/clash.js

@@ -158,6 +158,7 @@ export default function Clash_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/clashmeta.js

@@ -173,6 +173,7 @@ export default function ClashMeta_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/shadowrocket.js

@@ -176,6 +176,7 @@ export default function ShadowRocket_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/stash.js

@@ -265,6 +265,7 @@ export default function Stash_Producer() {
                 delete proxy.collectionName;
                 delete proxy.id;
                 delete proxy.resolved;
+                delete proxy['no-resolve'];
                 for (const key in proxy) {
                     if (proxy[key] == null || /^_/i.test(key)) {
                         delete proxy[key];

backend/src/core/proxy-utils/producers/surge.js

@@ -21,7 +21,7 @@ export default function Surge_Producer() {
             case 'trojan':
                 return trojan(proxy);
             case 'vmess':
-                return vmess(proxy);
+                return vmess(proxy, opts['include-unsupported-proxy']);
             case 'http':
                 return http(proxy);
             case 'socks5':
@@ -264,7 +264,7 @@ function trojan(proxy) {
     return result.toString();
 }
 
-function vmess(proxy) {
+function vmess(proxy, includeUnsupportedProxy) {
     const result = new Result(proxy);
     result.append(`${proxy.name}=${proxy.type},${proxy.server},${proxy.port}`);
     result.appendIfPresent(`,username=${proxy.uuid}`, 'uuid');
@@ -278,7 +278,7 @@ function vmess(proxy) {
     );
 
     // transport
-    handleTransport(result, proxy);
+    handleTransport(result, proxy, includeUnsupportedProxy);
 
     // AEAD
     if (isPresent(proxy, 'aead')) {
@@ -1013,7 +1013,7 @@ function hysteria2(proxy) {
     return result.toString();
 }
 
-function handleTransport(result, proxy) {
+function handleTransport(result, proxy, includeUnsupportedProxy) {
     if (isPresent(proxy, 'network')) {
         if (proxy.network === 'ws') {
             result.append(`,ws=true`);
@@ -1039,7 +1039,13 @@ function handleTransport(result, proxy) {
                 }
             }
         } else {
-            throw new Error(`network ${proxy.network} is unsupported`);
+            if (includeUnsupportedProxy && ['http'].includes(proxy.network)) {
+                $.info(
+                    `Include Unsupported Proxy: nework ${proxy.network} -> tcp`,
+                );
+            } else {
+                throw new Error(`network ${proxy.network} is unsupported`);
+            }
         }
     }
 }

backend/src/core/proxy-utils/producers/uri.js

@@ -10,6 +10,7 @@ export default function URI_Producer() {
         delete proxy.collectionName;
         delete proxy.id;
         delete proxy.resolved;
+        delete proxy['no-resolve'];
         for (const key in proxy) {
             if (proxy[key] == null || /^_/i.test(key)) {
                 delete proxy[key];

backend/src/products/cron-sync-artifacts.js

@@ -117,6 +117,7 @@ async function doSync() {
                         await produceArtifact({
                             type: 'subscription',
                             name: subName,
+                            awaitCustomCache: true,
                         });
                     } catch (e) {
                         // $.error(`${e.message ?? e}`);

backend/src/restful/download.js

@@ -156,6 +156,7 @@ async function downloadSubscription(req, res) {
                             $arguments.flowUserAgent,
                             undefined,
                             sub.proxy,
+                            $arguments.flowUrl,
                         );
                         if (flowInfo) {
                             res.set('subscription-userinfo', flowInfo);
@@ -314,6 +315,7 @@ async function downloadCollection(req, res) {
                                 $arguments.flowUserAgent,
                                 undefined,
                                 sub.proxy,
+                                $arguments.flowUrl,
                             );
                             if (flowInfo) {
                                 res.set('subscription-userinfo', flowInfo);

backend/src/restful/subscriptions.js

@@ -118,6 +118,7 @@ async function getFlowInfo(req, res) {
                 $arguments.flowUserAgent,
                 undefined,
                 sub.proxy,
+                $arguments.flowUrl,
             );
             if (!flowHeaders) {
                 failed(

backend/src/restful/sync.js

@@ -36,6 +36,7 @@ async function produceArtifact({
     produceType,
     produceOpts = {},
     subscription,
+    awaitCustomCache,
 }) {
     platform = platform || 'JSON';
 
@@ -67,6 +68,8 @@ async function produceArtifact({
                                 ua || sub.ua,
                                 undefined,
                                 sub.proxy,
+                                undefined,
+                                awaitCustomCache,
                             );
                         } catch (err) {
                             errors[url] = err;
@@ -112,6 +115,8 @@ async function produceArtifact({
                                 ua || sub.ua,
                                 undefined,
                                 sub.proxy,
+                                undefined,
+                                awaitCustomCache,
                             );
                         } catch (err) {
                             errors[url] = err;
@@ -503,6 +508,7 @@ async function syncArtifacts() {
                         await produceArtifact({
                             type: 'subscription',
                             name: subName,
+                            awaitCustomCache: true,
                         });
                     } catch (e) {
                         // $.error(`${e.message ?? e}`);

backend/src/utils/download.js

@@ -20,6 +20,7 @@ export default async function download(
     timeout,
     proxy,
     skipCustomCache,
+    awaitCustomCache,
 ) {
     let $arguments = {};
     let url = rawUrl.replace(/#noFlow$/, '');
@@ -41,25 +42,66 @@ export default async function download(
             }
         }
     }
+    const { isNode, isStash, isLoon, isShadowRocket, isQX } = ENV();
+    const { defaultUserAgent, defaultTimeout, cacheThreshold } =
+        $.read(SETTINGS_KEY);
+    const userAgent = ua || defaultUserAgent || 'clash.meta';
+    const requestTimeout = timeout || defaultTimeout;
+    const id = hex_md5(userAgent + url);
+
     const customCacheKey = $arguments?.cacheKey
         ? `#sub-store-cached-custom-${$arguments?.cacheKey}`
         : undefined;
 
     if (customCacheKey && !skipCustomCache) {
-        const cached = $.read(customCacheKey);
-        if (cached) {
+        const customCached = $.read(customCacheKey);
+        const cached = resourceCache.get(id);
+        if (!$arguments?.noCache && cached) {
             $.info(
-                `乐观缓存: URL ${url}\n本次返回自定义缓存 ${$arguments?.cacheKey}\n并进行请求 尝试更新缓存`,
-            );
-            download(
-                rawUrl.replace(/(\?|&)cacheKey=.*?(&|$)/, ''),
-                ua,
-                timeout,
-                proxy,
-                true,
+                `乐观缓存: URL ${url}\n存在有效的常规缓存\n使用常规缓存以避免重复请求`,
             );
             return cached;
         }
+        if (customCached) {
+            if (awaitCustomCache) {
+                $.info(`乐观缓存: URL ${url}\n本次进行请求 尝试更新缓存`);
+                try {
+                    await download(
+                        rawUrl.replace(/(\?|&)cacheKey=.*?(&|$)/, ''),
+                        ua,
+                        timeout,
+                        proxy,
+                        true,
+                    );
+                } catch (e) {
+                    $.error(
+                        `乐观缓存: URL ${url} 更新缓存发生错误 ${
+                            e.message ?? e
+                        }`,
+                    );
+                    $.info('使用乐观缓存的数据刷新缓存, 防止后续请求');
+                    resourceCache.set(id, customCached);
+                }
+            } else {
+                $.info(
+                    `乐观缓存: URL ${url}\n本次返回自定义缓存 ${$arguments?.cacheKey}\n并进行请求 尝试异步更新缓存`,
+                );
+                download(
+                    rawUrl.replace(/(\?|&)cacheKey=.*?(&|$)/, ''),
+                    ua,
+                    timeout,
+                    proxy,
+                    true,
+                ).catch((e) => {
+                    $.error(
+                        `乐观缓存: URL ${url} 异步更新缓存发生错误 ${
+                            e.message ?? e
+                        }`,
+                    );
+                });
+            }
+            return customCached;
+        }
     }
 
     // const downloadUrlMatch = url.match(/^\/api\/(file|module)\/(.+)/);
@@ -79,12 +121,6 @@ export default async function download(
     //     return item.content;
     // }
 
-    const { isNode, isStash, isLoon, isShadowRocket, isQX } = ENV();
-    const { defaultUserAgent, defaultTimeout, cacheThreshold } =
-        $.read(SETTINGS_KEY);
-    const userAgent = ua || defaultUserAgent || 'clash.meta';
-    const requestTimeout = timeout || defaultTimeout;
-    const id = hex_md5(userAgent + url);
     if (!isNode && tasks.has(id)) {
         return tasks.get(id);
     }
@@ -104,9 +140,13 @@ export default async function download(
 
     // try to find in app cache
     const cached = resourceCache.get(id);
-    if (!$arguments?.noCache && cached && !skipCustomCache) {
+    if (!$arguments?.noCache && cached) {
         $.info(`使用缓存: ${url}`);
         result = cached;
+        if (customCacheKey) {
+            $.info(`URL ${url}\n写入自定义缓存 ${$arguments?.cacheKey}`);
+            $.write(cached, customCacheKey);
+        }
     } else {
         $.info(
             `Downloading...\nUser-Agent: ${userAgent}\nTimeout: ${requestTimeout}\nProxy: ${proxy}\nURL: ${url}`,
@@ -177,6 +217,7 @@ export default async function download(
                     $arguments.flowUserAgent,
                     undefined,
                     proxy,
+                    $arguments.flowUrl,
                 ),
             ),
         );

backend/src/utils/flow.js

@@ -10,8 +10,8 @@ export function getFlowField(headers) {
     )[0];
     return headers[subkey];
 }
-export async function getFlowHeaders(rawUrl, ua, timeout, proxy) {
-    let url = rawUrl;
+export async function getFlowHeaders(rawUrl, ua, timeout, proxy, flowUrl) {
+    let url = flowUrl || rawUrl;
     let $arguments = {};
     const rawArgs = url.split('#');
     url = url.split('#')[0];
@@ -48,60 +48,76 @@ export async function getFlowHeaders(rawUrl, ua, timeout, proxy) {
             'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)';
         const requestTimeout = timeout || defaultTimeout;
         const http = HTTP();
-        try {
+        if (flowUrl) {
             $.info(
-                `使用 HEAD 方法获取流量信息: ${url}, User-Agent: ${
+                `使用 GET 方法从响应体获取流量信息: ${flowUrl}, User-Agent: ${
                     userAgent || ''
                 }`,
             );
-            const { headers } = await http.head({
-                url: url
-                    .split(/[\r\n]+/)
-                    .map((i) => i.trim())
-                    .filter((i) => i.length)[0],
-                headers: {
-                    'User-Agent': userAgent,
-                    ...(isStash && proxy
-                        ? {
-                              'X-Stash-Selected-Proxy':
-                                  encodeURIComponent(proxy),
-                          }
-                        : {}),
-                    ...(isShadowRocket && proxy
-                        ? { 'X-Surge-Policy': proxy }
-                        : {}),
-                },
-                timeout: requestTimeout,
-                ...(proxy ? { proxy } : {}),
-                ...(isLoon && proxy ? { node: proxy } : {}),
-                ...(isQX && proxy ? { opts: { policy: proxy } } : {}),
-                ...(proxy ? getPolicyDescriptor(proxy) : {}),
-            });
-            flowInfo = getFlowField(headers);
-        } catch (e) {
-            $.error(
-                `使用 HEAD 方法获取流量信息失败: ${url}, User-Agent: ${
-                    userAgent || ''
-                }: ${e.message ?? e}`,
-            );
-        }
-        if (!flowInfo) {
-            $.info(
-                `使用 GET 方法获取流量信息: ${url}, User-Agent: ${
-                    userAgent || ''
-                }`,
-            );
-            const { headers } = await http.get({
-                url: url
-                    .split(/[\r\n]+/)
-                    .map((i) => i.trim())
-                    .filter((i) => i.length)[0],
+            const { body } = await http.get({
+                url: flowUrl,
                 headers: {
                     'User-Agent': userAgent,
                 },
                 timeout: requestTimeout,
             });
-            flowInfo = getFlowField(headers);
+            flowInfo = body;
+        } else {
+            try {
+                $.info(
+                    `使用 HEAD 方法从响应头获取流量信息: ${url}, User-Agent: ${
+                        userAgent || ''
+                    }`,
+                );
+                const { headers } = await http.head({
+                    url: url
+                        .split(/[\r\n]+/)
+                        .map((i) => i.trim())
+                        .filter((i) => i.length)[0],
+                    headers: {
+                        'User-Agent': userAgent,
+                        ...(isStash && proxy
+                            ? {
+                                  'X-Stash-Selected-Proxy':
+                                      encodeURIComponent(proxy),
+                              }
+                            : {}),
+                        ...(isShadowRocket && proxy
+                            ? { 'X-Surge-Policy': proxy }
+                            : {}),
+                    },
+                    timeout: requestTimeout,
+                    ...(proxy ? { proxy } : {}),
+                    ...(isLoon && proxy ? { node: proxy } : {}),
+                    ...(isQX && proxy ? { opts: { policy: proxy } } : {}),
+                    ...(proxy ? getPolicyDescriptor(proxy) : {}),
+                });
+                flowInfo = getFlowField(headers);
+            } catch (e) {
+                $.error(
+                    `使用 HEAD 方法从响应头获取流量信息失败: ${url}, User-Agent: ${
+                        userAgent || ''
+                    }: ${e.message ?? e}`,
+                );
+            }
+            if (!flowInfo) {
+                $.info(
+                    `使用 GET 方法获取流量信息: ${url}, User-Agent: ${
+                        userAgent || ''
+                    }`,
+                );
+                const { headers } = await http.get({
+                    url: url
+                        .split(/[\r\n]+/)
+                        .map((i) => i.trim())
+                        .filter((i) => i.length)[0],
+                    headers: {
+                        'User-Agent': userAgent,
+                    },
+                    timeout: requestTimeout,
+                });
+                flowInfo = getFlowField(headers);
+            }
         }
         if (flowInfo) {
             headersResourceCache.set(url, flowInfo);

backend/src/utils/geo.js

@@ -430,6 +430,13 @@ export function getISO(name) {
     return ISOFlags[getFlag(name)]?.[0];
 }
 
+// remove flag
+export function removeFlag(str) {
+    return str
+        .replace(/[\uD83C][\uDDE6-\uDDFF][\uD83C][\uDDE6-\uDDFF]|🏴‍☠️|🏳️‍🌈/g, '')
+        .trim();
+}
+
 export class MMDB {
     constructor({ country, asn } = {}) {
         if ($.env.isNode) {

backend/src/vendor/open-api.js

@@ -382,6 +382,12 @@ export function HTTP(defaultOptions = { baseURL: '' }) {
                         url: options.url,
                         headers: options.headers,
                         body: options.body,
+                        options: {
+                            Proxy: options.proxy,
+                            Timeout: options.timeout
+                                ? options.timeout / 1000
+                                : 15,
+                        },
                     });
                     resolve({
                         statusCode: response.status,

config/README.md

@@ -6,13 +6,23 @@ Sub-Store Releases: [`https://github.com/sub-store-org/Sub-Store/releases`](http
 
 Telegram 频道: [`https://t.me/cool_scripts` ](https://t.me/cool_scripts)
 
-## 脚本配置：
+## 服务器/云平台/Docker/Android 版
+
+https://xream.notion.site/Sub-Store-abe6a96944724dc6a36833d5c9ab7c87
+
+## App 版
 
 ### 1. Loon
 安装使用 插件 [`https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Loon.plugin`](https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Loon.plugin) 即可。
 
 ### 2. Surge
 
+#### 关于 Surge 的格外说明
+
+Surge Mac 版如何支持 SSR, 如何去除 HTTP 传输层以支持 类似 VMess HTTP 节点等 请查看 [链接参数说明](https://github.com/sub-store-org/Sub-Store/wiki/%E9%93%BE%E6%8E%A5%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E)
+
+定时处理订阅 功能, 避免 App 内拉取超时, 请查看 [定时处理订阅](https://t.me/zhetengsha/1449)
+
 0. 最新 Surge iOS TestFlight 版本 可使用 Beta 版(支持最新 Surge iOS TestFlight 版本的特性): [`https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Surge-Beta.sgmodule`](https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Surge-Beta.sgmodule)
 
 1. 官方默认版模块(支持 App 内使用编辑参数): [`https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Surge.sgmodule`](https://raw.githubusercontent.com/sub-store-org/Sub-Store/master/config/Surge.sgmodule)
@@ -39,3 +49,11 @@ Telegram 频道: [`https://t.me/cool_scripts` ](https://t.me/cool_scripts)
 1. 使用 Safari 打开这个 https://sub.store 如网页正常打开并且未弹出任何错误提示，说明 Sub-Store 已经配置成功。
 2. 可以把 Sub-Store 添加到主屏幕，即可获得类似于 APP 的使用体验。
 3. 更详细的使用指南请参考[文档](https://www.notion.so/Sub-Store-6259586994d34c11a4ced5c406264b46)。
+
+## 链接参数说明
+
+https://github.com/sub-store-org/Sub-Store/wiki/%E9%93%BE%E6%8E%A5%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E
+
+## 脚本使用说明
+
+https://github.com/sub-store-org/Sub-Store/wiki/%E8%84%9A%E6%9C%AC%E4%BD%BF%E7%94%A8%E8%AF%B4%E6%98%8E

scripts/demo.js

@@ -7,7 +7,7 @@ function operator(proxies = [], targetPlatform, context) {
   // proxies 为传入的内部节点数组
   // 结构大致参考了 Clash.Meta(mihomo) 有私货
   // 可在预览界面点击节点查看 JSON 结构 或查看 `target=JSON` 的通用订阅
-  // 1. `no-resolve` 为不解析域名
+  // 1. `_no-resolve` 为不解析域名
   // 2. 域名解析后 会多一个 `_resolved` 字段
   // 3. 域名解析后会有`_IPv4`, `_IPv6`, `_IP`(若有多个步骤, 只取第一次成功的 v4 或 v6 数据), `_domain` 字段
   // 4. 节点字段 `exec` 为 `ssr-local` 路径, 默认 `/usr/local/bin/ssr-local`; 端口从 10000 开始递增(暂不支持配置)
@@ -40,12 +40,15 @@ function operator(proxies = [], targetPlatform, context) {
   //     isIP,
   //     yaml, // yaml 解析和生成
   //     getFlag, // 获取 emoji 旗帜
+  //     removeFlag, // 移除 emoji 旗帜
   //     getISO, // 获取 ISO 3166-1 alpha-2 代码
   //     Gist, // Gist 类
   // }
 
   // 示例: 给节点名添加前缀
   // $server.name = `[${ProxyUtils.getISO($server.name)}] ${$server.name}`
+  // 示例: 给节点名添加旗帜
+  // $server.name = `[${ProxyUtils.getFlag($server.name).replace(/🇹🇼/g, '🇼🇸')}] ${ProxyUtils.removeFlag($server.name)}`
 
   // 示例: 从 sni 文件中读取内容并进行节点操作
   // const sni = await produceArtifact({