Fix all recent bugs. Shift to Invidious API.
authorJoseph Nuthalapati <njoseph@riseup.net>
Fri, 25 Dec 2020 14:51:01 +0000 (20:21 +0530)
committerJoseph Nuthalapati <njoseph@riseup.net>
Fri, 25 Dec 2020 14:51:01 +0000 (20:21 +0530)
Unable to keep up with YouTube's JavaScript changes. Using the API from
an Invidious instance to retrieve search results, but playing directly
from YouTube to reduce load on the Invidious instance.

Configuration options are still in code. WIP on moving them to a
configurable file.

Signed-off-by: Joseph Nuthalapati <njoseph@riseup.net>
nimcoon.el
src/config.nim
src/lib.nim
src/nimcoon.nim
src/youtube.nim
tests/tests.nim

index c238fc144a1345d69c02e87db31796ce7e16d49f..90f15e4b23bdbd138add01c4b9bd569e56f1b27e 100644 (file)
@@ -1,5 +1,6 @@
 ;;; nimcoon.el -*- lexical-binding: t; -*-
 ;;;
+;;; Commentary
 ;;; Usage in Doom Emacs
 ;;; Place or symlink the file into ~/.doom.d/
 ;;; (load! "nimcoon")
   "Search by QUERY and play in NimCoon."
   (call-process "nimcoon" nil 0 nil args query))
 
+(defun nimcoon-search(args query)
+  "Search by QUERY with the given ARGS."
+  (with-output-to-temp-buffer "*NimCoon search results*"
+    (call-process "nimcoon" nil "*NimCoon search results*" t args query)
+    (with-current-buffer "*NimCoon search results*"
+      (org-mode))))
+
 ;;; Interactive functions
 (defun nimcoon-feeling-lucky-music(query)
   (interactive "sSearch query: ")
   (interactive)
   (shell-command "kill `pgrep nimcoon` `pgrep mpv` `pgrep vlc`"))
 
-(defun nimcoon-search(args query)
-  "Search by QUERY with the given ARGS."
-  (with-output-to-temp-buffer "*NimCoon search results*"
-    (call-process "nimcoon" nil "*NimCoon search results*" t args query)
-    (with-current-buffer "*NimCoon search results*"
-      (org-mode))))
-
 (defun nimcoon-search-video(query)
   "Search for a video by QUERY."
   (interactive "sSearch query: ")
@@ -72,3 +73,5 @@
        :desc "Kill"  "k" #'nimcoon-kill-background-processes
        :desc "Video" "v" #'nimcoon-feeling-lucky-video
        :desc "Music" "m" #'nimcoon-feeling-lucky-music))
+
+;;; nimcoon.el ends here
index b231a83b96f41d000991e6d1008ccf54b5b9f75b..6f1bd72b0046e8c89e53696c4493be9718acf72d 100644 (file)
@@ -1,4 +1,12 @@
-# Your configuration here.
+discard """"
+Configuration goes through three levels of overrides:
+
+   /etc/nimcoon/config.json - configuration set by system administrator
+   ~/.config/nimcoon/config.json   - per user configuration
+   default configuration provided in this file
+""""
+
+# Default configuration values
 
 # Supported video players in order of preference.
 # Should be able to play YouTube videos directly.
@@ -15,4 +23,24 @@ let musicDownloadDirectory* = "~/Music"
 
 # Rewrite Invidious URLs to YouTube
 # Using Invidious as a proxy makes loading YouTube videos much slower
-let rewriteInvidiousURLs* = false
+let rewriteInvidiousURLs* = true
+
+# Invidious instance for querying
+# This instance should have a valid public API
+# Check like this: curl https://invidious.xyz/api/v1/search\?q\=cats
+let invidiousInstance* = "https://invidious.xyz"
+
+# import os
+
+# func getConfigFile(dir): string = getConfigDir() / "nimcoon" / "config.json"
+
+# const ADMIN_CONFIGURATION = getConfigFile("etc")
+# const USER_CONFIGURATION = getConfigFile(getConfigDir())
+
+# const DEFAULT_CONFIGURATION = {
+#   "entries_per_page": 10,
+#   "video_download_directory": "~/Videos",
+#   "music_download_directory": "~/Music",
+#   "always_fullscreen": false,  # TODO not implemented yet
+#   "rewrite_invidious_urls": false
+# }.toTable  # toTable creates an immutable Table (newTable doesn't)
index 16082c5a6f8ef0936e3a28e650217932149f7fdb..33b15d933dcde8e8dd2b2675890254b6d5d2a3c1 100644 (file)
@@ -16,7 +16,7 @@ import
 
 
 let
-  processOptions = {poStdErrToStdOut, poUsePath} # poEchoCmd can be added to options for debugging
+  processOptions = {poStdErrToStdOut, poUsePath, poEchoCmd}
   PEERTUBE_REGEX = re"videos\/watch\/[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}"
 
 
@@ -49,6 +49,7 @@ proc presentVideoOptions*(searchResults: SearchResults) =
   for index, (title, url) in searchResults:
     styledEcho $index, ". ", styleBright, fgMagenta, title, "\n", resetStyle, fgCyan, "   ", url, "\n"
 
+
 func buildPlayerArgs(url: string, options: Table[string, bool], player: string): seq[string] =
   let musicOnly = if options["musicOnly"]: "--no-video" else: ""
   let fullScreen = if options["fullScreen"]: "--fullscreen" else: ""
@@ -62,7 +63,7 @@ proc play*(player: string, options: Table[string, bool], url: string, title: str
   if "--no-video" in args:
     discard execShellCmd(&"{player} {args.join(\" \")}")
   else:
-    discard startProcess(player, args=args, options=processOptions)
+    discard execProcess(player, args=args, options=processOptions)
 
 
 func buildMusicDownloadArgs(url: string): seq[string] =
@@ -86,9 +87,11 @@ proc download*(args: openArray[string], title: string) =
 func urlLongen(url: string): string = url.replace("youtu.be/", "www.youtube.com/watch?v=")
 
 
-func rewriteInvidiousToYouTube(url: string): string =
+func rewriteInvidiousToYouTube*(url: string): string =
   {.noSideEffect.}:
-    if rewriteInvidiousURLs: url.replace("invidio.us", "www.youtube.com") else: url
+    if rewriteInvidiousURLs and url.replace(".", "").contains("invidious"):
+       &"https://www.youtube.com/watch?v={url.split(\"=\")[1]}"
+    else: url
 
 
 func stripZshEscaping(url: string): string = url.replace("\\", "")
index 4f75d826f6d640fa091949edbce364370c13addf..45fca1748c1f280e51c19dd5b65f327c4c441035 100644 (file)
@@ -12,9 +12,16 @@ import
 
 
 proc parseArguments(): CommandLineOptions =
+
   var
     searchQuery = ""
-    options = to_table({"musicOnly": false, "feelingLucky": false, "fullScreen": false, "download": false, "non-interactive": false})
+    options = to_table({
+      "musicOnly": false,
+      "feelingLucky": false,
+      "fullScreen": false,
+      "download": false,
+      "non-interactive": false
+    })
 
   for kind, key, value in getopt():
     case kind
@@ -45,6 +52,7 @@ proc isValidOptions*(options: Options): bool =
      stderr.writeLine fmt"Incompatible options provided: {combination[0]} and {combination[1]}"
      result = false
 
+
 proc main() =
   let
     player = selectMediaPlayer()
index fd98d41e4f2f22b257789e34cf2fe437008b0ab3..3457062f8bf7eeddab312e01e8ec182612f3d2fb 100644 (file)
@@ -2,41 +2,30 @@ import
   httpClient,
   json,
   strformat,
-  strutils,
-  sequtils,
   uri
 
-import types
-
-proc getYouTubePage(searchQuery: string): string =
-  let queryParam = encodeUrl(searchQuery)
-  let client = newHttpClient()
-  let response = get(client, &"https://www.youtube.com/results?search_query={queryParam}")
-  $response.body
+import
+  config,
+  types
 
 
-proc getSearchResults*(searchQuery: string): SearchResults =
-  let html = getYouTubePage(searchQuery)
-  let lines = html.split('\n').filterIt(it.contains("ytInitialData"))
-  let line = lines[0]
-  let jsonString = line.split('=', maxsplit=1)[1].strip().strip(chars={';'})
-  let jsonData = parseJson(jsonString)
+discard """
+Using Invidious API to retrieve the search results but playing the results directly from YouTube.
 
-  let videos = jsonData["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"]
+API reference:
+https://github.com/iv-org/documentation/blob/master/API.md#get-apiv1search
+"""
 
+proc getSearchResults*(searchQuery: string): SearchResults =
+  let queryParam = encodeUrl(searchQuery)
+  let client = newHttpClient()
+  let response = get(client, &"{invidiousInstance}/api/v1/search?q={queryParam}")
+  let jsonData = parseJson($response.body)
   var searchResults: SearchResults = @[]
-
-  for video in videos:
-    if video.hasKey("videoRenderer"):
-      let title = ($video["videoRenderer"]["title"]["runs"][0]["text"]).strip(chars={'"'})
-      let videoId = ($video["videoRenderer"]["videoId"]).strip(chars={'"'})
-      let videoUrl = &"https://www.youtube.com/watch?v={videoId}"
-      searchResults.add((title, videoUrl))
-
-    elif video.hasKey("playlistRenderer"):
-      let title = ($video["playlistRenderer"]["title"]["simpleText"]).strip(chars={'"'})
-      let playlistId = ($video["playlistRenderer"]["playlistId"]).strip(chars={'"'})
-      let playlistUrl = &"https://www.youtube.com/playlist?list={playlistId}"
-      searchResults.add((title, playlistUrl))
-
+  for item in jsonData:
+    if item["type"].getStr() == "video":
+      searchResults.add((item["title"].getStr(), "https://www.youtube.com/watch?v=" & item["videoId"].getStr()))
+    elif item["type"].getStr() == "playlist":
+      searchResults.add((item["title"].getStr(), "https://www.youtube.com/watch?v=" & item["playlistId"].getStr()))
+    # Not handling type = channel for now
   searchResults
index 8b9e6d21ac1a95de1221d7f66e4344853df077f8..fd44065992227c86e599c42513a6d2229a56e228 100644 (file)
@@ -13,15 +13,19 @@ suite "Playing direct links":
     check(sanitizeURL("https://www.youtube.com/watch\\?v\\=QOEMv0S8AcA") == expected)
 
   test "validate options":
-      let invalidOptionsList = [
-        to_table({"musicOnly": true, "feelingLucky": false, "fullScreen": true, "download": false}),
-        to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": true})
-      ]
-      for invalidOptions in invalidOptionsList:
-        check(not isValidOptions(invalidOptions))
-      let validOptionsList = [
-        to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": false, "download": true}),
-        to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": false})
-      ]
-      for validOptions in validOptionsList:
-        check(isValidOptions(validOptions))
+    let invalidOptionsList = [
+      to_table({"musicOnly": true, "feelingLucky": false, "fullScreen": true, "download": false}),
+      to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": true})
+    ]
+    for invalidOptions in invalidOptionsList:
+      check(not isValidOptions(invalidOptions))
+    let validOptionsList = [
+      to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": false, "download": true}),
+      to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": false})
+    ]
+    for validOptions in validOptionsList:
+      check(isValidOptions(validOptions))
+
+  test "rewrite invidious urls":
+    let url = "https://invidious.snopyta.org/watch?v=sZhxCUay5ks"
+    check(rewriteInvidiousToYouTube(url) == "https://www.youtube.com/watch?v=sZhxCUay5ks")