update: cobalt 11.7

.github/workflows/test-services.yml

@@ -15,6 +15,9 @@ jobs:
       services: ${{ steps.checkServices.outputs.service_list }}
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
       - uses: pnpm/action-setup@v4
       - id: checkServices
         run: pnpm i --frozen-lockfile && echo "service_list=$(node api/src/util/test get-services)" >> "$GITHUB_OUTPUT"
@@ -29,6 +32,9 @@ jobs:
     name: "test service: ${{ matrix.service }}"
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
       - uses: pnpm/action-setup@v4
       - run: pnpm i --frozen-lockfile
       - run: node api/src/util/test run-tests-for ${{ matrix.service }}

.github/workflows/test.yml

@@ -10,6 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
       - uses: pnpm/action-setup@v4
       - name: Check that lockfile does not need an update
         run: pnpm install --frozen-lockfile
@@ -32,5 +35,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
       - uses: pnpm/action-setup@v4
       - run: .github/test.sh api

api/README.md

@@ -37,7 +37,6 @@ if the desired service isn't supported yet, feel free to create an appropriate i
 | twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
 | vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
 | vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
-| xiaohongshu       | ✅            | ✅         | ✅         | ➖         | ➖              |
 | youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
 
 | emoji   | meaning                 |

api/package.json

@@ -1,12 +1,12 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "11.5",
+    "version": "11.7",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
     "engines": {
-        "node": ">=18"
+        "node": ">=18.17"
     },
     "scripts": {
         "start": "node src/cobalt",
@@ -23,28 +23,29 @@
     },
     "homepage": "https://github.com/imputnet/cobalt#readme",
     "dependencies": {
-        "@datastructures-js/priority-queue": "^6.3.1",
+        "@datastructures-js/priority-queue": "^6.3.5",
         "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
-        "dotenv": "^16.0.1",
-        "express": "^4.21.2",
-        "express-rate-limit": "^7.4.1",
-        "ffmpeg-static": "^5.1.0",
-        "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.2.0",
-        "mime": "^4.0.4",
-        "nanoid": "^5.0.9",
-        "set-cookie-parser": "2.6.0",
-        "undici": "^6.21.3",
+        "dotenv": "^17.2.3",
+        "express": "^4.22.1",
+        "express-rate-limit": "^8.3.2",
+        "ffmpeg-static": "^5.3.0",
+        "hls-parser": "^0.16.0",
+        "ipaddr.js": "2.3.0",
+        "isolated-vm": "^6.0.2",
+        "mime": "^4.1.0",
+        "nanoid": "^5.1.6",
+        "set-cookie-parser": "2.7.2",
+        "undici": "^6.24.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "15.1.1",
-        "zod": "^3.23.8"
+        "youtubei.js": "17.0.1",
+        "zod": "^4.2.1"
     },
     "optionalDependencies": {
-        "freebind": "^0.2.2",
-        "rate-limit-redis": "^4.2.0",
-        "redis": "^4.7.0"
+        "freebind": "^0.2.3",
+        "rate-limit-redis": "^4.3.1",
+        "redis": "^5.11.0"
     }
 }

api/src/core/env.js

@@ -121,6 +121,7 @@ export const loadEnvs = (env = process.env) => {
         ytSessionReloadInterval: 300,
         ytSessionInnertubeClient: env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
         ytAllowBetterAudio: env.YOUTUBE_ALLOW_BETTER_AUDIO !== "0",
+        ytPlayerIds: env.YOUTUBE_PLAYER_ID?.split(',')?.map(p => p.trim()),
 
         // "never" | "session" | "always"
         forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",

api/src/processing/helpers/youtube-session.js

@@ -9,6 +9,10 @@ const defaultAgent = new Agent();
 let session;
 
 const validateSession = (sessionResponse) => {
+    sessionResponse.visitor_data ??= sessionResponse.contentBinding;
+    sessionResponse.potoken ??= sessionResponse.poToken;
+    sessionResponse.updated ??= new Date().getTime();
+
     if (!sessionResponse.potoken) {
         throw "no poToken in session response";
     }
@@ -33,11 +37,11 @@ const updateSession = (newSession) => {
 
 const loadSession = async () => {
     const sessionServerUrl = new URL(env.ytSessionServer);
-    sessionServerUrl.pathname = "/token";
+    sessionServerUrl.pathname = "/get_pot";
 
     const newSession = await fetch(
         sessionServerUrl,
-        { dispatcher: defaultAgent }
+        { method: 'POST', dispatcher: defaultAgent }
     ).then(a => a.json());
 
     validateSession(newSession);

api/src/processing/match-action.js

@@ -103,7 +103,6 @@ export default function({
                 case "twitter":
                 case "snapchat":
                 case "bsky":
-                case "xiaohongshu":
                     params = { picker: r.picker };
                     break;
 
@@ -145,6 +144,7 @@ export default function({
                     params = { type: r.type };
                     break;
 
+                case "rutube":
                 case "vimeo":
                     if (Array.isArray(r.urls)) {
                         params = { type: "merge" };
@@ -179,7 +179,6 @@ export default function({
                     break;
 
                 case "ok":
-                case "xiaohongshu":
                 case "newgrounds":
                     params = { type: "proxy" };
                     break;

api/src/processing/match.js

@@ -28,7 +28,6 @@ import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
-import xiaohongshu from "./services/xiaohongshu.js";
 import newgrounds from "./services/newgrounds.js";
 
 let freebind;
@@ -260,15 +259,6 @@ export default async function({ host, patternMatch, params, authType }) {
                 });
                 break;
 
-            case "xiaohongshu":
-                r = await xiaohongshu({
-                    ...patternMatch,
-                    h265: params.allowH265,
-                    isAudioOnly,
-                    dispatcher,
-                });
-                break;
-
             case "newgrounds":
                 r = await newgrounds({
                     ...patternMatch,

api/src/processing/service-config.js

@@ -152,7 +152,7 @@ export const services = {
             ":user/photo/:postId",
             "v/:postId.html"
         ],
-        subdomains: ["vt", "vm", "m", "t"],
+        subdomains: ["vt", "vm", "m", "t", "pro"],
     },
     tumblr: {
         patterns: [
@@ -205,14 +205,6 @@ export const services = {
         subdomains: ["m"],
         altDomains: ["vkvideo.ru", "vk.ru"],
     },
-    xiaohongshu: {
-        patterns: [
-            "explore/:id?xsec_token=:token",
-            "discovery/item/:id?xsec_token=:token",
-            ":shareType/:shareId",
-        ],
-        altDomains: ["xhslink.com"],
-    },
     youtube: {
         patterns: [
             "watch?v=:id",

api/src/processing/service-patterns.js

@@ -81,10 +81,6 @@ export const testers = {
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
         (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
 
-    "xiaohongshu": pattern =>
-        pattern.id?.length <= 24 && pattern.token?.length <= 64 ||
-        pattern.shareId?.length <= 24 && pattern.shareType?.length === 1,
-
     "youtube": pattern =>
         pattern.id?.length <= 11,
 }

api/src/processing/services/bilibili.js

@@ -1,4 +1,4 @@
-import { genericUserAgent, env } from "../../config.js";
+import { env } from "../../config.js";
 import { resolveRedirectingURL } from "../url.js";
 
 // TO-DO: higher quality downloads (currently requires an account)
@@ -26,7 +26,7 @@ async function com_download(id, partId) {
 
     const html = await fetch(url, {
         headers: {
-            "user-agent": genericUserAgent
+            "user-agent": "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
         }
     })
     .then(r => r.text())

api/src/processing/services/facebook.js

@@ -24,9 +24,9 @@ const resolveUrl = (url, dispatcher) => {
 }
 
 export default async function({ id, shareType, shortLink, dispatcher }) {
-    let url = `https://web.facebook.com/i/videos/${id}`;
+    let url = `https://www.facebook.com/reel/${id}`;
 
-    if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
+    if (shareType) url = `https://www.facebook.com/share/${shareType}/${id}`;
     if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
 
     const html = await fetch(url, { headers, dispatcher })

api/src/processing/services/rutube.js

@@ -13,7 +13,7 @@ const delta = (a, b) => Math.abs(a - b);
 export default async function(obj) {
     if (obj.yappyId) {
         const yappy = await requestJSON(
-            `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15`
+            `https://rutube.ru/pangolin/api/web/yappy/v4/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=1`
         )
         const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
         if (!yappyURL) return { error: "fetch.empty" };

api/src/processing/services/soundcloud.js

@@ -15,24 +15,27 @@ async function findClientID() {
             return cachedID.id;
         }
 
-        const scripts = sc.matchAll(/<script.+src="(.+)">/g);
+        let clientid = sc.match(/"hydratable"\s*:\s*"apiClient"\s*,\s*"data"\s*:\s*\{\s*"id"\s*:\s*"([^"]+)"/)?.[1];
+        if (!clientid) {
+            const scripts = sc.matchAll(/<script.+src="(.+)">/g);
 
-        let clientid;
-        for (let script of scripts) {
-            const url = script[1];
+            for (let script of scripts) {
+                const url = script[1];
 
-            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
-                return;
-            }
+                if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
+                    return;
+                }
 
-            const scrf = await fetch(url).then(r => r.text()).catch(() => {});
-            const id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
+                const scrf = await fetch(url).then(r => r.text()).catch(() => {});
+                const id = scrf.match(/,client_id:"([A-Za-z0-9]{32})",/);
 
-            if (id && typeof id[0] === 'string') {
-                clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
-                break;
+                if (id && id.length >= 2) {
+                    clientid = id[1];
+                    break;
+                }
             }
         }
+
         cachedID.version = scVersion;
         cachedID.id = clientid;
 

api/src/processing/services/twitter.js

@@ -37,9 +37,19 @@ function needsFixing(media) {
 }
 
 function bestQuality(arr) {
-    return arr.filter(v => v.content_type === "video/mp4")
+    return stripVideoURL(
+        arr.filter(v => v.content_type === "video/mp4")
         .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b)
         .url
+    );
+}
+
+function stripVideoURL(maybeUrl) {
+    if (maybeUrl) {
+        const url = new URL(maybeUrl);
+        url.searchParams.delete('tag');
+        return url.toString();
+    }
 }
 
 let _cachedToken;

api/src/processing/services/vk.js

@@ -2,14 +2,15 @@ import { env } from "../../config.js";
 
 const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
 
-const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
-const apiUrl = "https://api.vk.com/method";
+const authUrl = "https://api.vk.ru/method/auth.getAnonymToken";
+const videoApiUrl = "https://api.vkvideo.ru/method/video.get";
 
 const clientId = "51552953";
 const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
+const clientVersion = "5.274";
 
 // used in stream/shared.js for accessing media files
-export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
+export const vkClientAgent = "com.vk.vkvideo.prod/1955 (iPhone, iOS 16.7.15, iPhone10,4, Scale/2.0) SAK/1.135";
 
 const cachedToken = {
     token: "",
@@ -24,10 +25,11 @@ const getToken = async () => {
 
     const randomDeviceId = crypto.randomUUID().toUpperCase();
 
-    const anonymOauth = new URL(oauthUrl);
+    const anonymOauth = new URL(authUrl);
     anonymOauth.searchParams.set("client_id", clientId);
     anonymOauth.searchParams.set("client_secret", clientSecret);
     anonymOauth.searchParams.set("device_id", randomDeviceId);
+    anonymOauth.searchParams.set("v", clientVersion);
 
     const oauthResponse = await fetch(anonymOauth.toString(), {
         headers: {
@@ -39,11 +41,13 @@ const getToken = async () => {
         }
     });
 
-    if (!oauthResponse) return;
+    if (!oauthResponse || !oauthResponse.response) return;
 
-    if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
-        cachedToken.token = oauthResponse.token;
-        cachedToken.expiry = oauthResponse.expired_at;
+    const res = oauthResponse.response;
+
+    if (res.token && res.expired_at && typeof res.expired_at === "number") {
+        cachedToken.token = res.token;
+        cachedToken.expiry = res.expired_at;
         cachedToken.device_id = randomDeviceId;
     }
 
@@ -53,7 +57,7 @@ const getToken = async () => {
 }
 
 const getVideo = async (ownerId, videoId, accessKey) => {
-    const video = await fetch(`${apiUrl}/video.get`, {
+    const video = await fetch(videoApiUrl, {
         method: "POST",
         headers: {
             "content-type": "application/x-www-form-urlencoded; charset=utf-8",
@@ -63,7 +67,7 @@ const getVideo = async (ownerId, videoId, accessKey) => {
             anonymous_token: cachedToken.token,
             device_id: cachedToken.device_id,
             lang: "en",
-            v: "5.244",
+            v: clientVersion,
             videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
         }).toString()
     })