fix: nonsense to install yt dlp and parse youtube in peace

Signed-off-by: thiswillbeyourgithub <[email protected]>

Author: thiswillbeyourgithub

setup.py

@@ -119,7 +119,8 @@ def run(self):
         'goose3 >= 3.1.19',  # url
         "youtube_dl",  # youtube_dl, we try to install yt_dl_patched using PostInstallCommand as it's not in pypi but we install yt_dl anyway just in case. Also the latest version will try to be installed from the git repo directly using the PostInstallCommand function above.
         "youtube-transcript-api >= 0.6.2",  # youtube
-        "pytube >= 15.0.0",  # youtube
+        # "pytube >= 15.0.0",  # youtube
+        "yt-dlp >= 2024.10.23.232902.dev0",  # youtube
         'LogseqMarkdownParser >= 3.3',  # logseq files (I'm the dev behind it)
         'deepgram-sdk >= 3.2.7',  # audio transcription
         'httpx >= 0.27.0',  # to increase deepgram timeout

wdoc/utils/import_tricks.py

@@ -81,7 +81,6 @@ def custom_loading(module: str, strict: bool = True) -> None:
     custom_loading('sqlite3')
     custom_loading('tldextract')
     custom_loading('pyfiglet')
-    custom_loading('youtube_dl')
     custom_loading('yt_dlp', strict=False)
     custom_loading('pandas')
     custom_loading('ankipandas')

wdoc/utils/loaders.py

@@ -59,6 +59,7 @@
 import playwright.sync_api
 import openparse
 from langchain.text_splitter import TextSplitter, RecursiveCharacterTextSplitter
+import yt_dlp as youtube_dl
 
 from .misc import (doc_loaders_cache, html_to_text, hasher,
                    file_hasher, get_splitter, check_docs_tkn_length,
@@ -72,15 +73,6 @@
 from .errors import TimeoutPdfLoaderError
 from .env import WDOC_MAX_PDF_LOADER_TIMEOUT, WDOC_EMPTY_LOADER, WDOC_PRIVATE_MODE
 
-# try:
-#     import yt_dlp as youtube_dl
-# except ImportError as e :
-#     red("Failed to import yt_dlp, a better fork of youtube_dl.\n"
-#         "Consider installing it from pip with pip install git+https://github.com/ytdl-patched/ytdl-patched\n"
-#         "Will fallback to using youtube_dl")
-#     import youtube_dl
-import youtube_dl
-
 try:
     import pdftotext
 except Exception as err: