From a2d28598f44142969997ae56f6bd684c54d95e0b Mon Sep 17 00:00:00 2001 From: Joseph Nuthalapati Date: Fri, 25 Dec 2020 20:21:01 +0530 Subject: [PATCH] Fix all recent bugs. Shift to Invidious API. Unable to keep up with YouTube's JavaScript changes. Using the API from an Invidious instance to retrieve search results, but playing directly from YouTube to reduce load on the Invidious instance. Configuration options are still in code. WIP on moving them to a configurable file. Signed-off-by: Joseph Nuthalapati --- nimcoon.el | 17 ++++++++++------- src/config.nim | 32 ++++++++++++++++++++++++++++++-- src/lib.nim | 11 +++++++---- src/nimcoon.nim | 10 +++++++++- src/youtube.nim | 49 +++++++++++++++++++------------------------------ tests/tests.nim | 28 ++++++++++++++++------------ 6 files changed, 91 insertions(+), 56 deletions(-) diff --git a/nimcoon.el b/nimcoon.el index c238fc1..90f15e4 100644 --- a/nimcoon.el +++ b/nimcoon.el @@ -1,5 +1,6 @@ ;;; nimcoon.el -*- lexical-binding: t; -*- ;;; +;;; Commentary ;;; Usage in Doom Emacs ;;; Place or symlink the file into ~/.doom.d/ ;;; (load! "nimcoon") @@ -19,6 +20,13 @@ "Search by QUERY and play in NimCoon." (call-process "nimcoon" nil 0 nil args query)) +(defun nimcoon-search(args query) + "Search by QUERY with the given ARGS." + (with-output-to-temp-buffer "*NimCoon search results*" + (call-process "nimcoon" nil "*NimCoon search results*" t args query) + (with-current-buffer "*NimCoon search results*" + (org-mode)))) + ;;; Interactive functions (defun nimcoon-feeling-lucky-music(query) (interactive "sSearch query: ") @@ -42,13 +50,6 @@ (interactive) (shell-command "kill `pgrep nimcoon` `pgrep mpv` `pgrep vlc`")) -(defun nimcoon-search(args query) - "Search by QUERY with the given ARGS." - (with-output-to-temp-buffer "*NimCoon search results*" - (call-process "nimcoon" nil "*NimCoon search results*" t args query) - (with-current-buffer "*NimCoon search results*" - (org-mode)))) - (defun nimcoon-search-video(query) "Search for a video by QUERY." (interactive "sSearch query: ") @@ -72,3 +73,5 @@ :desc "Kill" "k" #'nimcoon-kill-background-processes :desc "Video" "v" #'nimcoon-feeling-lucky-video :desc "Music" "m" #'nimcoon-feeling-lucky-music)) + +;;; nimcoon.el ends here diff --git a/src/config.nim b/src/config.nim index b231a83..6f1bd72 100644 --- a/src/config.nim +++ b/src/config.nim @@ -1,4 +1,12 @@ -# Your configuration here. +discard """" +Configuration goes through three levels of overrides: + + /etc/nimcoon/config.json - configuration set by system administrator + ~/.config/nimcoon/config.json - per user configuration + default configuration provided in this file +"""" + +# Default configuration values # Supported video players in order of preference. # Should be able to play YouTube videos directly. @@ -15,4 +23,24 @@ let musicDownloadDirectory* = "~/Music" # Rewrite Invidious URLs to YouTube # Using Invidious as a proxy makes loading YouTube videos much slower -let rewriteInvidiousURLs* = false +let rewriteInvidiousURLs* = true + +# Invidious instance for querying +# This instance should have a valid public API +# Check like this: curl https://invidious.xyz/api/v1/search\?q\=cats +let invidiousInstance* = "https://invidious.xyz" + +# import os + +# func getConfigFile(dir): string = getConfigDir() / "nimcoon" / "config.json" + +# const ADMIN_CONFIGURATION = getConfigFile("etc") +# const USER_CONFIGURATION = getConfigFile(getConfigDir()) + +# const DEFAULT_CONFIGURATION = { +# "entries_per_page": 10, +# "video_download_directory": "~/Videos", +# "music_download_directory": "~/Music", +# "always_fullscreen": false, # TODO not implemented yet +# "rewrite_invidious_urls": false +# }.toTable # toTable creates an immutable Table (newTable doesn't) diff --git a/src/lib.nim b/src/lib.nim index 16082c5..33b15d9 100644 --- a/src/lib.nim +++ b/src/lib.nim @@ -16,7 +16,7 @@ import let - processOptions = {poStdErrToStdOut, poUsePath} # poEchoCmd can be added to options for debugging + processOptions = {poStdErrToStdOut, poUsePath, poEchoCmd} PEERTUBE_REGEX = re"videos\/watch\/[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}" @@ -49,6 +49,7 @@ proc presentVideoOptions*(searchResults: SearchResults) = for index, (title, url) in searchResults: styledEcho $index, ". ", styleBright, fgMagenta, title, "\n", resetStyle, fgCyan, " ", url, "\n" + func buildPlayerArgs(url: string, options: Table[string, bool], player: string): seq[string] = let musicOnly = if options["musicOnly"]: "--no-video" else: "" let fullScreen = if options["fullScreen"]: "--fullscreen" else: "" @@ -62,7 +63,7 @@ proc play*(player: string, options: Table[string, bool], url: string, title: str if "--no-video" in args: discard execShellCmd(&"{player} {args.join(\" \")}") else: - discard startProcess(player, args=args, options=processOptions) + discard execProcess(player, args=args, options=processOptions) func buildMusicDownloadArgs(url: string): seq[string] = @@ -86,9 +87,11 @@ proc download*(args: openArray[string], title: string) = func urlLongen(url: string): string = url.replace("youtu.be/", "www.youtube.com/watch?v=") -func rewriteInvidiousToYouTube(url: string): string = +func rewriteInvidiousToYouTube*(url: string): string = {.noSideEffect.}: - if rewriteInvidiousURLs: url.replace("invidio.us", "www.youtube.com") else: url + if rewriteInvidiousURLs and url.replace(".", "").contains("invidious"): + &"https://www.youtube.com/watch?v={url.split(\"=\")[1]}" + else: url func stripZshEscaping(url: string): string = url.replace("\\", "") diff --git a/src/nimcoon.nim b/src/nimcoon.nim index 4f75d82..45fca17 100644 --- a/src/nimcoon.nim +++ b/src/nimcoon.nim @@ -12,9 +12,16 @@ import proc parseArguments(): CommandLineOptions = + var searchQuery = "" - options = to_table({"musicOnly": false, "feelingLucky": false, "fullScreen": false, "download": false, "non-interactive": false}) + options = to_table({ + "musicOnly": false, + "feelingLucky": false, + "fullScreen": false, + "download": false, + "non-interactive": false + }) for kind, key, value in getopt(): case kind @@ -45,6 +52,7 @@ proc isValidOptions*(options: Options): bool = stderr.writeLine fmt"Incompatible options provided: {combination[0]} and {combination[1]}" result = false + proc main() = let player = selectMediaPlayer() diff --git a/src/youtube.nim b/src/youtube.nim index fd98d41..3457062 100644 --- a/src/youtube.nim +++ b/src/youtube.nim @@ -2,41 +2,30 @@ import httpClient, json, strformat, - strutils, - sequtils, uri -import types - -proc getYouTubePage(searchQuery: string): string = - let queryParam = encodeUrl(searchQuery) - let client = newHttpClient() - let response = get(client, &"https://www.youtube.com/results?search_query={queryParam}") - $response.body +import + config, + types -proc getSearchResults*(searchQuery: string): SearchResults = - let html = getYouTubePage(searchQuery) - let lines = html.split('\n').filterIt(it.contains("ytInitialData")) - let line = lines[0] - let jsonString = line.split('=', maxsplit=1)[1].strip().strip(chars={';'}) - let jsonData = parseJson(jsonString) +discard """ +Using Invidious API to retrieve the search results but playing the results directly from YouTube. - let videos = jsonData["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"] +API reference: +https://github.com/iv-org/documentation/blob/master/API.md#get-apiv1search +""" +proc getSearchResults*(searchQuery: string): SearchResults = + let queryParam = encodeUrl(searchQuery) + let client = newHttpClient() + let response = get(client, &"{invidiousInstance}/api/v1/search?q={queryParam}") + let jsonData = parseJson($response.body) var searchResults: SearchResults = @[] - - for video in videos: - if video.hasKey("videoRenderer"): - let title = ($video["videoRenderer"]["title"]["runs"][0]["text"]).strip(chars={'"'}) - let videoId = ($video["videoRenderer"]["videoId"]).strip(chars={'"'}) - let videoUrl = &"https://www.youtube.com/watch?v={videoId}" - searchResults.add((title, videoUrl)) - - elif video.hasKey("playlistRenderer"): - let title = ($video["playlistRenderer"]["title"]["simpleText"]).strip(chars={'"'}) - let playlistId = ($video["playlistRenderer"]["playlistId"]).strip(chars={'"'}) - let playlistUrl = &"https://www.youtube.com/playlist?list={playlistId}" - searchResults.add((title, playlistUrl)) - + for item in jsonData: + if item["type"].getStr() == "video": + searchResults.add((item["title"].getStr(), "https://www.youtube.com/watch?v=" & item["videoId"].getStr())) + elif item["type"].getStr() == "playlist": + searchResults.add((item["title"].getStr(), "https://www.youtube.com/watch?v=" & item["playlistId"].getStr())) + # Not handling type = channel for now searchResults diff --git a/tests/tests.nim b/tests/tests.nim index 8b9e6d2..fd44065 100644 --- a/tests/tests.nim +++ b/tests/tests.nim @@ -13,15 +13,19 @@ suite "Playing direct links": check(sanitizeURL("https://www.youtube.com/watch\\?v\\=QOEMv0S8AcA") == expected) test "validate options": - let invalidOptionsList = [ - to_table({"musicOnly": true, "feelingLucky": false, "fullScreen": true, "download": false}), - to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": true}) - ] - for invalidOptions in invalidOptionsList: - check(not isValidOptions(invalidOptions)) - let validOptionsList = [ - to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": false, "download": true}), - to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": false}) - ] - for validOptions in validOptionsList: - check(isValidOptions(validOptions)) + let invalidOptionsList = [ + to_table({"musicOnly": true, "feelingLucky": false, "fullScreen": true, "download": false}), + to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": true}) + ] + for invalidOptions in invalidOptionsList: + check(not isValidOptions(invalidOptions)) + let validOptionsList = [ + to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": false, "download": true}), + to_table({"musicOnly": false, "feelingLucky": true, "fullScreen": true, "download": false}) + ] + for validOptions in validOptionsList: + check(isValidOptions(validOptions)) + + test "rewrite invidious urls": + let url = "https://invidious.snopyta.org/watch?v=sZhxCUay5ks" + check(rewriteInvidiousToYouTube(url) == "https://www.youtube.com/watch?v=sZhxCUay5ks") -- 2.43.0