Skip to content
Merged
Changes from 28 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
d3c9780
Add WTF Python Command
Shivansh-007 Mar 2, 2021
56ea9e2
Fix grammar in docstrings, remove redundant variable, remove the use …
Shivansh-007 Mar 5, 2021
d23cc4d
Fix indentation issues and make use of triple quotes
Shivansh-007 Mar 5, 2021
4dcc2fe
Update docstrings and remove redundant list()
Shivansh-007 Mar 6, 2021
415b7a4
Change minimum certainty to 75.
Shivansh-007 Mar 8, 2021
3b8eadd
Make 'make_embed' function a non async function
Shivansh-007 Mar 12, 2021
6465862
Try to unload WTFPython Extension if max fetch requests hit i.e. 3 el…
Shivansh-007 Mar 14, 2021
a2cdfe9
Correct log messages.
Shivansh-007 Mar 14, 2021
3ae3aad
Make flake8 happy :D
Shivansh-007 Mar 14, 2021
4a18a2d
Remove redundant class attributes and async functions.
Shivansh-007 Mar 15, 2021
2a518f9
Apply requested grammar and style changes.
Shivansh-007 Mar 15, 2021
a4c494f
Fix unload and load extension logic.
Shivansh-007 Mar 15, 2021
f82169f
Fix typo in `WTF_PYTHON_RAW_URL`
Shivansh-007 Apr 7, 2021
d893d42
Changed fuzzy_wuzzy to rapidfuzz
brad90four Sep 3, 2021
ba60b8a
Merge branch 'main' of https://github.com/python-discord/sir-lancebot…
brad90four Sep 7, 2021
a702969
Merge main
brad90four Sep 7, 2021
1e7b581
Move wtf_python.py to bot/exts/utilities, flake8
brad90four Sep 7, 2021
4c0ae39
Fix trailing commas and long lines
brad90four Sep 7, 2021
62d3a81
# This is a combination of 3 commits.
brad90four Sep 7, 2021
9d59b52
Squashing commits
brad90four Sep 8, 2021
4eb7c67
Pulling from main
brad90four Sep 10, 2021
2ca1691
Add debug logs, and send embed
brad90four Sep 11, 2021
f6e9d28
Add markdown file creation
brad90four Sep 12, 2021
0eb3670
Move the list(map(str.strip , ...) to for loop
brad90four Sep 12, 2021
cc55239
Remove line
brad90four Sep 12, 2021
5dcc43f
Use StringIO for file creation
brad90four Sep 12, 2021
917de69
Update file creation with StringIO
brad90four Sep 12, 2021
d74c8f3
Remove embed file preview
brad90four Sep 17, 2021
f49c9e0
chore: update wtf_python docstring
brad90four Sep 23, 2021
3adc085
chore: change regex to search, remove file preview
brad90four Sep 24, 2021
8c1f3ec
feat: update caching as recommended
brad90four Sep 27, 2021
f3d752a
chore: remove logging statements
brad90four Sep 28, 2021
a1832be
feat: scheduled task for fetch_readme
brad90four Sep 28, 2021
761a3f7
chore: fix hyperlink, remove dead code
brad90four Sep 28, 2021
a26ca12
fix: capitalization clean up
brad90four Sep 28, 2021
c942f32
chore: remove unused code
brad90four Sep 29, 2021
170676c
chore: remove more unused code
brad90four Sep 29, 2021
32a0f9a
feat: add light grey logo image in embed
brad90four Sep 30, 2021
04d71f8
feat: add light grey image
brad90four Sep 30, 2021
bc64aa6
chore: remove debug log message
brad90four Oct 4, 2021
2197fc6
feat: add found search result header
brad90four Oct 4, 2021
f60b724
feat: limit user query to 50 characters
brad90four Oct 6, 2021
14e9d7b
cleanup: remove debug logging
brad90four Oct 7, 2021
cd58665
fix: restructure if not match statement
brad90four Oct 7, 2021
ca1e447
Merge branch 'main' into feature/wtf-python
Xithrius Oct 22, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 154 additions & 0 deletions bot/exts/utilities/wtf_python.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import logging
import random
import re
from enum import Enum
from functools import partial
from typing import Optional

from discord import Embed
from discord.ext import commands, tasks
from rapidfuzz import process

from bot import constants
from bot.bot import Bot

log = logging.getLogger(__name__)

WTF_PYTHON_RAW_URL = "http://raw.githubusercontent.com/satwikkansal/wtfpython/master/"
BASE_URL = "https://github.com/satwikkansal/wtfpython"
FETCH_TRIES = 3

ERROR_MESSAGE = f"""
Unknown WTF Python Query. Please try to reformulate your query.

**Examples**:
```md
{constants.Client.prefix}wtf wild imports
{constants.Client.prefix}wtf subclass
{constants.Client.prefix}wtf del
```
If the problem persists send a message in <#{constants.Channels.dev_contrib}>
"""

MINIMUM_CERTAINTY = 50


class Action(Enum):
"""Represents an action to perform on an extension."""

# Need to be partial otherwise they are considered to be function definitions.
LOAD = partial(Bot.load_extension)
UNLOAD = partial(Bot.unload_extension)


class WTFPython(commands.Cog):
"""Cog that allows getting WTF Python entries from the WTF Python repository."""

def __init__(self, bot: Bot):
self.bot = bot
self.fetch_readme.start()

self.headers: dict[str, str] = dict()

@tasks.loop(hours=1)
async def fetch_readme(self) -> None:
"""Gets the content of README.md from the WTF Python Repository."""
failed_tries = 0

for x in range(FETCH_TRIES):
async with self.bot.http_session.get(f"{WTF_PYTHON_RAW_URL}README.md") as resp:
log.trace("Fetching the latest WTF Python README.md")

if resp.status == 200:
raw = await resp.text()
self.parse_readme(raw)
log.debug(
"Successfully fetched the latest WTF Python README.md, "
"breaking out of retry loop"
)
break

else:
failed_tries += 1
log.debug(
"Failed to get latest WTF Python README.md on try "
f"{x}/{FETCH_TRIES}. Status code {resp.status}"
)

if failed_tries == 3:
log.error("Couldn't fetch WTF Python README.md after 3 tries, unloading extension.")
action = Action.UNLOAD
else:
action = Action.LOAD

verb = action.name.lower()
ext = "bot.exts.utilities.wtf_python"

try:
action.value(self.bot, ext)
except (commands.ExtensionAlreadyLoaded, commands.ExtensionNotLoaded):
log.debug(f"Extension `{ext}` is already {verb}ed.")
else:
log.debug(f"Extension {verb}ed: `{ext}`.")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think any discussion spawned from the last one-

You should add an attribute to the cog: last_fetched which will be a datetime of when this was last updated.
Then, when you use the command and self.last_fetched + datetime.timedelta(hour=1) < datetime.datetime.now() you can use asyncio.create_task(self.fetch_readme) (don't await it) to start a task fetching it in the background.

We still want to keep the response time good, so by using a task we don't need to wait for its completion which means we can return what's cached so far.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I'm getting stuck on this one. I haven't used asyncio yet so maybe that is what is tripping me up.
Also, how can I get the "last" time the cog was loaded instead of just calling it immediately?

Here is what I have so far:

class WTFPython(commands.Cog):
    """Cog that allows getting WTF Python entries from the WTF Python repository."""

    def __init__(self, bot: Bot):
        self.bot = bot
        self.headers: dict[str, str] = dict()
        self.last_fetched = datetime.datetime.now()  # this is just the current time, how to access last load?
        log.debug(f"{self.last_fetched = }")

        if self.last_fetched + datetime.timedelta(seconds=10) < datetime.datetime.now():  # this will never be true currently
            log.debug("Last loading time was within x amount of time.")
            asyncio.create_task(self.fetch_readme)  # is this correct?

    # cut out most everything from the function, this is what is left
    async def fetch_readme(self) -> None:
        """Gets the content of README.md from the WTF Python Repository."""
        async with self.bot.http_session.get(f"{WTF_PYTHON_RAW_URL}README.md") as resp:
            log.trace("Fetching the latest WTF Python README.md")
            if resp.status == 200:
                raw = await resp.text()
                self.parse_readme(raw)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am thinking something more like this:

README_REFRESH = 60  # Amount of minutes

class WTFPython(commands.Cog):
    """Cog that allows getting WTF Python entries from the WTF Python repository."""

    def __init__(self, bot: Bot):
        self.bot = bot
        self.headers: dict[str, str] = dict()
        self.last_fetched = datetime.datetime.now()
        log.debug(f"{self.last_fetched = }")

        asyncio.create_task(self.fetch_readme)  # Start a first task

    async def fetch_readme(self) -> None:
        """Gets the content of README.md from the WTF Python Repository."""
        refresh = self.last_fetched + datetime.timedelta(minutes=README_REFRESH)

        if refresh > datetime.datetime.now() and self.headers:  # We also want to fetch if we haven't previously
            return  # Our cache should be up-to-date

        async with self.bot.http_session.get(f"{WTF_PYTHON_RAW_URL}README.md") as resp:
            log.trace("Fetching the latest WTF Python README.md")
            if resp.status == 200:
                raw = await resp.text()
                self.parse_readme(raw)

You then call asyncio.create_task(self.fetch_readme) every time someone runs the command.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whoa that is awesome! Thanks for the super clear example.


def parse_readme(self, data: str) -> None:
"""
Parses the README.md into a dict.

It parses the readme into the `self.headers` dict,
where the key is the heading and the value is the
link to the heading.
"""
# Match the start of examples, until the end of the table of contents (toc)
table_of_contents = re.search(
r"\[👀 Examples\]\(#-examples\)\n([\w\W]*)<!-- tocstop -->", data
)[0].split("\n")

for header in list(map(str.strip, table_of_contents)):
match = re.findall(r"\[▶ (.*)\]\((.*)\)", header)
if match:
self.headers[match[0][0]] = f"{BASE_URL}{match[0][1]}"

def fuzzy_match_header(self, query: str) -> Optional[str]:
"""
Returns the fuzzy match of a query if its ratio is above "MINIMUM_CERTAINTY" else returns None.

"MINIMUM_CERTAINTY" is the lowest score at which the fuzzy match will return a result.
The certainty returned by rapidfuzz.process.extractOne is a score between 0 and 100,
with 100 being a perfect match.
"""
match, certainty, _ = process.extractOne(query, self.headers.keys())
log.debug(f"{match = }, {certainty = }")
return match if certainty > MINIMUM_CERTAINTY else None

@commands.command(aliases=("wtf", "WTF"))
async def wtf_python(self, ctx: commands.Context, *, query: str) -> None:
"""
Search WTF python.

Gets the link of the fuzzy matched query from https://github.com/satwikkansal/wtfpython.
Usage:
--> .wtf wild imports
"""
match = self.fuzzy_match_header(query)
if match:
embed = Embed(
title=f"WTF Python Search Result For {query}",
colour=constants.Colours.dark_green,
description=f"[Go to Repository Section]({self.headers[match]})",
)
embed.set_thumbnail(url=f"{WTF_PYTHON_RAW_URL}images/logo.png")
await ctx.send(embed=embed)
return
else:
embed = Embed(
title=random.choice(constants.ERROR_REPLIES),
description=ERROR_MESSAGE,
colour=constants.Colours.soft_red,
)
await ctx.send(embed=embed)


def setup(bot: Bot) -> None:
"""Load the WTFPython Cog."""
bot.add_cog(WTFPython(bot))