Cherry pick upstream hotfix + dev branch changes which add needed functions

MarmadileManteater · MarmadileManteater · commit 16d6ccc93688 · 2024-08-02T11:15:17.000-04:00
- ^Update youtubei.js - Use streams from the iOS client to workaround playback issues (FreeTubeApp#5472) - Allow user agent spoofing for usage of iOS client
diff --git a/android/app/src/main/java/io/freetubeapp/freetube/FreeTubeJavaScriptInterface.kt b/android/app/src/main/java/io/freetubeapp/freetube/FreeTubeJavaScriptInterface.kt
@@ -664,6 +664,13 @@ class FreeTubeJavaScriptInterface {
     context.isInAPrompt = false
   }
 
+  @JavascriptInterface
+  fun queueFetchBody(id: String, body: String) {
+    if (body != "undefined") {
+      context.pendingRequestBodies[id] = body
+    }
+  }
+
   private fun addNamedCallbackToPromise(promise: String, name: String) {
     context.runOnUiThread {
       context.webView.loadUrl("javascript: window['${promise}'].callbacks = window['${promise}'].callbacks || {};  window['${promise}'].callbacks.notify = (key, message) => window['${promise}'].callbacks[key].forEach(callback => callback(message)); window['${promise}'].callbacks['${name}'] = window['${promise}'].callbacks['${name}'] || []")
diff --git a/android/app/src/main/java/io/freetubeapp/freetube/MainActivity.kt b/android/app/src/main/java/io/freetubeapp/freetube/MainActivity.kt
@@ -9,8 +9,10 @@ import android.os.Bundle
 import android.view.View
 import android.view.ViewGroup
 import android.view.ViewTreeObserver
+import android.webkit.ConsoleMessage
 import android.webkit.WebChromeClient
 import android.webkit.WebResourceRequest
+import android.webkit.WebResourceResponse
 import android.webkit.WebView
 import android.webkit.WebViewClient
 import android.widget.FrameLayout
@@ -24,9 +26,15 @@ import androidx.core.view.WindowCompat
 import androidx.core.view.WindowInsetsCompat
 import androidx.core.view.WindowInsetsControllerCompat
 import io.freetubeapp.freetube.databinding.ActivityMainBinding
+import org.json.JSONObject
+import java.io.Serializable
+import java.net.HttpURLConnection
+import java.net.URL
 import java.net.URLEncoder
 import java.util.Base64
+import java.util.UUID
 import java.util.concurrent.BlockingQueue
+import java.util.concurrent.ConcurrentHashMap.KeySetView
 import java.util.concurrent.LinkedBlockingQueue
 import java.util.concurrent.ThreadPoolExecutor
 import java.util.concurrent.TimeUnit
@@ -44,10 +52,12 @@ class MainActivity : AppCompatActivity(), OnRequestPermissionsResultCallback {
   lateinit var jsInterface: FreeTubeJavaScriptInterface
   lateinit var activityResultLauncher: ActivityResultLauncher<Intent>
   lateinit var content: View
+  var consoleMessages: MutableList<JSONObject> = mutableListOf()
   var showSplashScreen: Boolean = true
   var darkMode: Boolean = false
   var paused: Boolean = false
   var isInAPrompt: Boolean = false
+  var pendingRequestBodies: MutableMap<String, String> = mutableMapOf()
   /*
    * Gets the number of available cores
    * (not always the same as the maximum number of cores)
@@ -164,6 +174,20 @@ class MainActivity : AppCompatActivity(), OnRequestPermissionsResultCallback {
     webView.addJavascriptInterface(jsInterface, "Android")
     webView.webChromeClient = object: WebChromeClient() {
 
+      override fun onConsoleMessage(consoleMessage: ConsoleMessage): Boolean {
+        val messageData = JSONObject()
+        messageData.put("content", consoleMessage.message())
+        messageData.put("level", consoleMessage.messageLevel())
+        messageData.put("timestamp", System.currentTimeMillis())
+        messageData.put("id", UUID.randomUUID())
+        messageData.put("key", "${messageData["id"]}-${messageData["timestamp"]}")
+        messageData.put("sourceId", consoleMessage.sourceId())
+        messageData.put("lineNumber", consoleMessage.lineNumber())
+        consoleMessages.add(messageData)
+        webView.loadUrl("javascript: var event = new Event(\"console-message\"); event.data = JSON.parse(${btoa(messageData.toString())}); window.dispatchEvent(event)")
+        return super.onConsoleMessage(consoleMessage);
+      }
+
       override fun onShowCustomView(view: View?, callback: CustomViewCallback?) {
         windowInsetsController.hide(WindowInsetsCompat.Type.systemBars())
         fullscreenView = view!!
@@ -186,6 +210,72 @@ class MainActivity : AppCompatActivity(), OnRequestPermissionsResultCallback {
       }
     }
     webView.webViewClient = object: WebViewClient() {
+      override fun shouldInterceptRequest(
+        view: WebView?,
+        request: WebResourceRequest?
+      ): WebResourceResponse? {
+        if (request!!.requestHeaders.containsKey("x-user-agent")) {
+          with (URL(request!!.url.toString()).openConnection() as HttpURLConnection) {
+            requestMethod = request.method
+            val isClient5 = request.requestHeaders.containsKey("x-youtube-client-name") && request.requestHeaders["x-youtube-client-name"] == "5"
+            // map headers
+            for (header in request!!.requestHeaders) {
+              fun getReal(key: String, value: String): Array<String>? {
+                if (key == "x-user-agent") {
+                  return arrayOf("User-Agent", value)
+                }
+                if (key == "User-Agent") {
+                  return null
+                }
+                if (key == "x-fta-request-id") {
+                  return null
+                }
+                if (isClient5) {
+                  if (key == "referrer") {
+                    return null
+                  }
+                  if (key == "origin") {
+                    return null
+                  }
+                  if (key == "Sec-Fetch-Site") {
+                    return null
+                  }
+                  if (key == "Sec-Fetch-Mode") {
+                    return null
+                  }
+                  if (key == "Sec-Fetch-Dest") {
+                    return null
+                  }
+                  if (key == "sec-ch-ua") {
+                    return null
+                  }
+                  if (key == "sec-ch-ua-mobile") {
+                    return null
+                  }
+                  if (key == "sec-ch-ua-platform") {
+                    return null
+                  }
+                }
+                return arrayOf(key, value)
+              }
+              val real = getReal(header.key, header.value)
+              if (real !== null) {
+                setRequestProperty(real[0], real[1])
+              }
+            }
+            if (request.requestHeaders.containsKey("x-fta-request-id")) {
+              if (pendingRequestBodies.containsKey(request.requestHeaders["x-fta-request-id"])) {
+                val body = pendingRequestBodies[request.requestHeaders["x-fta-request-id"]]
+                pendingRequestBodies.remove(request.requestHeaders["x-fta-request-id"])
+                outputStream.write(body!!.toByteArray())
+              }
+            }
+            // 🧝‍♀️ magic
+            return WebResourceResponse(this.contentType, this.contentEncoding, inputStream!!)
+          }
+        }
+        return super.shouldInterceptRequest(view, request)
+      }
       override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
         if (request!!.url!!.scheme == "file") {
           // don't send file url requests to a web browser (it will crash the app)
diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "name": "freetube",
   "productName": "FreeTube",
   "description": "A private YouTube client",
-  "version": "0.21.2",
+  "version": "0.21.3",
   "license": "AGPL-3.0-or-later",
   "main": "./dist/main.js",
   "private": true,
@@ -81,7 +81,7 @@
     "vue-observe-visibility": "^1.0.0",
     "vue-router": "^3.6.5",
     "vuex": "^3.6.2",
-    "youtubei.js": "^10.2.0"
+    "youtubei.js": "^10.3.0"
   },
   "devDependencies": {
     "@babel/core": "^7.24.7",
diff --git a/src/index.ejs b/src/index.ejs
@@ -7,6 +7,31 @@
       <script>
         window.ofetch = window.fetch
         window.fetch = async (...args) => {
+          const requestId = crypto.randomUUID()
+          let body
+          for (const arg of args) {
+            if (typeof arg === 'object') {
+              if ('body' in arg) {
+                body = arg.body
+              }
+              if ('headers' in arg) {
+                if ('append' in arg.headers) {
+                  if (arg.headers.get('x-youtube-client-name') == 5) {
+                    arg.headers.append('x-fta-request-id', requestId)
+                    Android.queueFetchBody(requestId, body)
+                  }
+                } else {
+                  if ('x-youtube-client-name' in arg.headers) {
+                    if (arg.headers['x-youtube-client-name'] == 5) {
+                      arg.headers['x-fta-request-id'] = requestId
+                      Android.queueFetchBody(requestId, body)
+                    }
+                  }
+                }
+              }
+            }
+          }
+
           if (typeof args[0] === 'string' && args[0].startsWith('file://')) {
             // forward to xml http request
             /** @type {Response} */
diff --git a/src/main/index.js b/src/main/index.js
@@ -408,9 +408,24 @@ function runApp() {
       requestHeaders.Origin = 'https://www.youtube.com'
 
       if (url.startsWith('https://www.youtube.com/youtubei/')) {
-        requestHeaders['Sec-Fetch-Site'] = 'same-origin'
-        requestHeaders['Sec-Fetch-Mode'] = 'same-origin'
-        requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false'
+        // Make iOS requests work and look more realistic
+        if (requestHeaders['x-youtube-client-name'] === '5') {
+          delete requestHeaders.Referer
+          delete requestHeaders.Origin
+          delete requestHeaders['Sec-Fetch-Site']
+          delete requestHeaders['Sec-Fetch-Mode']
+          delete requestHeaders['Sec-Fetch-Dest']
+          delete requestHeaders['sec-ch-ua']
+          delete requestHeaders['sec-ch-ua-mobile']
+          delete requestHeaders['sec-ch-ua-platform']
+
+          requestHeaders['User-Agent'] = requestHeaders['x-user-agent']
+          delete requestHeaders['x-user-agent']
+        } else {
+          requestHeaders['Sec-Fetch-Site'] = 'same-origin'
+          requestHeaders['Sec-Fetch-Mode'] = 'same-origin'
+          requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false'
+        }
       } else {
         // YouTube doesn't send the Content-Type header for the media requests, so we shouldn't either
         delete requestHeaders['Content-Type']
diff --git a/src/renderer/helpers/api/local.js b/src/renderer/helpers/api/local.js
@@ -8,6 +8,7 @@ import {
   calculatePublishedDate,
   escapeHTML,
   extractNumberFromString,
+  randomArrayItem,
   toLocalePublicationString
 } from '../utils'
 
@@ -19,6 +20,25 @@ const TRACKING_PARAM_NAMES = [
   'utm_content',
 ]
 
+const IOS_VERSIONS = [
+  '17.5.1',
+  '17.5',
+  '17.4.1',
+  '17.4',
+  '17.3.1',
+  '17.3',
+]
+
+const YOUTUBE_IOS_CLIENT_VERSIONS = [
+  '19.29.1',
+  '19.28.1',
+  '19.26.5',
+  '19.25.4',
+  '19.25.3',
+  '19.24.3',
+  '19.24.2',
+]
+
 /**
  * Creates a lightweight Innertube instance, which is faster to create or
  * an instance that can decode the streaming URLs, which is slower to create
@@ -56,7 +76,36 @@ async function createInnertube({ withPlayer = false, location = undefined, safet
     client_type: clientType,
 
     // use browser fetch
-    fetch: (input, init) => fetch(input, init),
+    fetch: (input, init) => {
+      // Make iOS requests work and look more realistic
+      if (init?.headers instanceof Headers && init.headers.get('x-youtube-client-name') === '5') {
+        // Use a random iOS version and YouTube iOS client version to make the requests look less suspicious
+        const clientVersion = randomArrayItem(YOUTUBE_IOS_CLIENT_VERSIONS)
+        const iosVersion = randomArrayItem(IOS_VERSIONS)
+
+        init.headers.set('x-youtube-client-version', clientVersion)
+
+        // We can't set the user-agent here, but in the main process we take the x-user-agent and set it as the user-agent
+        init.headers.delete('user-agent')
+        init.headers.set('x-user-agent', `com.google.ios.youtube/${clientVersion} (iPhone16,2; CPU iOS ${iosVersion.replaceAll('.', '_')} like Mac OS X; en_US)`)
+
+        const bodyJson = JSON.parse(init.body)
+
+        const client = bodyJson.context.client
+
+        client.clientVersion = clientVersion
+        client.deviceMake = 'Apple'
+        client.deviceModel = 'iPhone16,2' // iPhone 15 Pro Max
+        client.osName = 'iOS'
+        client.osVersion = iosVersion
+        delete client.browserName
+        delete client.browserVersion
+
+        init.body = JSON.stringify(bodyJson)
+      }
+
+      return fetch(input, init)
+    },
     cache,
     generate_session_locally: !!generateSessionLocally
   })
@@ -190,27 +239,69 @@ export async function getLocalSearchContinuation(continuationData) {
   return handleSearchResponse(response)
 }
 
-export async function getLocalVideoInfo(id, attemptBypass = false) {
-  let info
-  let player
+/**
+ * @param {string} id
+ */
+export async function getLocalVideoInfo(id) {
+  const webInnertube = await createInnertube({ withPlayer: true, generateSessionLocally: false })
 
-  if (attemptBypass) {
-    const innertube = await createInnertube({ withPlayer: true, clientType: ClientType.TV_EMBEDDED, generateSessionLocally: false })
-    player = innertube.actions.session.player
+  const info = await webInnertube.getInfo(id)
 
-    // the second request that getInfo makes 404s with the bypass, so we use getBasicInfo instead
-    // that's fine as we have most of the information from the original getInfo request
-    info = await innertube.getBasicInfo(id, 'TV_EMBEDDED')
-  } else {
-    const innertube = await createInnertube({ withPlayer: true, generateSessionLocally: false })
-    player = innertube.actions.session.player
+  const hasTrailer = info.has_trailer
+  const trailerIsAgeRestricted = info.getTrailerInfo() === null
 
-    info = await innertube.getInfo(id)
+  if (hasTrailer) {
+    /** @type {import('youtubei.js').YTNodes.PlayerLegacyDesktopYpcTrailer} */
+    const trailerScreen = info.playability_status.error_screen
+    id = trailerScreen.video_id
   }
 
-  if (info.streaming_data) {
-    decipherFormats(info.streaming_data.adaptive_formats, player)
-    decipherFormats(info.streaming_data.formats, player)
+  // try to bypass the age restriction
+  if (info.playability_status.status === 'LOGIN_REQUIRED' || (hasTrailer && trailerIsAgeRestricted)) {
+    const tvInnertube = await createInnertube({ withPlayer: true, clientType: ClientType.TV_EMBEDDED, generateSessionLocally: false })
+
+    const tvInfo = await tvInnertube.getBasicInfo(id, 'TV_EMBEDDED')
+
+    if (tvInfo.streaming_data) {
+      decipherFormats(tvInfo.streaming_data.adaptive_formats, tvInnertube.actions.session.player)
+      decipherFormats(tvInfo.streaming_data.formats, tvInnertube.actions.session.player)
+    }
+
+    info.playability_status = tvInfo.playability_status
+    info.streaming_data = tvInfo.streaming_data
+    info.basic_info.start_timestamp = tvInfo.basic_info.start_timestamp
+    info.basic_info.duration = tvInfo.basic_info.duration
+    info.captions = tvInfo.captions
+    info.storyboards = tvInfo.storyboards
+  } else {
+    const iosInnertube = await createInnertube({ clientType: ClientType.IOS })
+
+    const iosInfo = await iosInnertube.getBasicInfo(id, 'iOS')
+
+    if (hasTrailer) {
+      info.playability_status = iosInfo.playability_status
+      info.streaming_data = iosInfo.streaming_data
+      info.basic_info.start_timestamp = iosInfo.basic_info.start_timestamp
+      info.basic_info.duration = iosInfo.basic_info.duration
+      info.captions = iosInfo.captions
+      info.storyboards = iosInfo.storyboards
+    } else if (iosInfo.streaming_data) {
+      info.streaming_data.adaptive_formats = iosInfo.streaming_data.adaptive_formats
+      // Use the legacy formats from the original web response as the iOS client doesn't have any legacy formats
+
+      for (const format of info.streaming_data.adaptive_formats) {
+        format.freeTubeUrl = format.url
+      }
+
+      // don't overwrite for live streams
+      if (!info.streaming_data.hls_manifest_url) {
+        info.streaming_data.hls_manifest_url = iosInfo.streaming_data.hls_manifest_url
+      }
+    }
+
+    if (info.streaming_data) {
+      decipherFormats(info.streaming_data.formats, webInnertube.actions.session.player)
+    }
   }
 
   return info
diff --git a/src/renderer/helpers/colors.js b/src/renderer/helpers/colors.js
@@ -1,4 +1,5 @@
 import i18n from '../i18n/index'
+import { randomArrayItem } from './utils'
 
 export const colors = [
   { name: 'Red', value: '#d50000' },
@@ -103,8 +104,7 @@ export function getRandomColorClass() {
 }
 
 export function getRandomColor() {
-  const randomInt = Math.floor(Math.random() * colors.length)
-  return colors[randomInt]
+  return randomArrayItem(colors)
 }
 
 export function calculateColorLuminance(colorValue) {
diff --git a/src/renderer/helpers/utils.js b/src/renderer/helpers/utils.js
diff --git a/src/renderer/views/Watch/Watch.js b/src/renderer/views/Watch/Watch.js
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`import i18n from '../i18n/index'`
	`2`	`+import { randomArrayItem } from './utils'`
`2`	`3`
`3`	`4`	`export const colors = [`
`4`	`5`	`{ name: 'Red', value: '#d50000' },`
`@@ -103,8 +104,7 @@ export function getRandomColorClass() {`
`103`	`104`	`}`
`104`	`105`
`105`	`106`	`export function getRandomColor() {`
`106`		`- const randomInt = Math.floor(Math.random() * colors.length)`
`107`		`- return colors[randomInt]`
	`107`	`+ return randomArrayItem(colors)`
`108`	`108`	`}`
`109`	`109`
`110`	`110`	`export function calculateColorLuminance(colorValue) {`