diff --git a/lab-web-scraping.ipynb b/lab-web-scraping.ipynb index e552783..e4a626c 100644 --- a/lab-web-scraping.ipynb +++ b/lab-web-scraping.ipynb @@ -110,14 +110,305 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "40359eee-9cd7-4884-bfa4-83344c222305", - "metadata": { - "id": "40359eee-9cd7-4884-bfa4-83344c222305" - }, + "execution_count": 299, + "id": "5b2a720f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "import time\n", + "import re\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 300, + "id": "68e84e34", + "metadata": {}, "outputs": [], "source": [ - "# Your solution goes here" + "def get_book_url():\n", + "\n", + " all_book_url = []\n", + "\n", + " for number in range(1,51):\n", + " base_url = f\"https://books.toscrape.com/catalogue/page-{number}.html\"\n", + " response = requests.get(base_url)\n", + " if response.status_code != 200:\n", + " print(f\"page not found {base_url}\")\n", + "\n", + " soup = BeautifulSoup(response.content)\n", + " \n", + " for i in soup.find_all('h3'):\n", + " book_url = 'https://books.toscrape.com/catalogue/' + i.find('a')['href']\n", + " all_book_url.append(book_url)\n", + " \n", + " return all_book_url\n" + ] + }, + { + "cell_type": "code", + "execution_count": 302, + "id": "8c6751dd", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def get_book_data(min_rating,max_price):\n", + "\n", + " all_book_url = get_book_url() # a list of urls\n", + " word_to_num = {\"One\": 1,\"Two\": 2,\"Three\": 3,\"Four\": 4,\"Five\": 5}\n", + " book_dic = {\n", + " 'title':[],\n", + " 'upc':[],\n", + " 'price_taxIncluded':[],\n", + " 'genre':[],\n", + " 'availability':[],\n", + " 'rating':[],\n", + " 'description':[],\n", + " 'url': []\n", + " }\n", + "\n", + " df = pd.DataFrame(book_dic)\n", + "\n", + " df['url'] = all_book_url\n", + "\n", + " df['soup'] = df['url'].apply(lambda x: BeautifulSoup(requests.get(x).content))\n", + "\n", + " df['price'] = df['soup'].apply(lambda x: \n", + " int(float(re.sub(r'[^\\d.]', '', x.find_all('tr')[3].find('td').get_text()))))\n", + " df['rating'] = df['soup'].apply(lambda x: x.find('div',attrs={'class':'col-sm-6 product_main'}).find('p', class_='star-rating')['class'][1])\n", + " \n", + " df['rating_num'] = df['rating'].apply(lambda x:\n", + " word_to_num.get(x,0))\n", + "\n", + " df_filtered = df[(df['price'] <= max_price) & (df['rating_num'] >= min_rating)].copy()\n", + "\n", + " df_filtered['title'] = df_filtered['soup'].apply(lambda x: x.find('h1').get_text())\n", + "\n", + " df_filtered['upc'] = df_filtered['soup'].apply(lambda x: x.find_all('tr')[0].find('td').get_text())\n", + "\n", + " df_filtered['price_taxIncluded'] = df_filtered['soup'].apply(lambda x: x.find_all('tr')[3].find('td').get_text())\n", + "\n", + " df_filtered['genre'] = df_filtered['soup'].apply(lambda x: x.find_all('a')[3].get_text())\n", + "\n", + " df_filtered['availability'] = df_filtered['soup'].apply(lambda x: x.find_all('tr')[5].find('td').get_text())\n", + "\n", + " df_filtered['description'] = df_filtered['soup'].apply(lambda x: x.find_all('p')[3].get_text())\n", + "\n", + " columns_needed = ['title','upc','price_taxIncluded','rating','genre','availability','description']\n", + "\n", + " return df_filtered[columns_needed]\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 303, + "id": "7176429a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titleupcprice_taxIncludedratinggenreavailabilitydescription
12Set Me Freece6396b0f23f6ecc£17.46FiveYoung AdultIn stock (19 available)Aaron Ledbetter’s future had been planned out ...
30The Four Agreements: A Practical Guide to Pers...6258a1f6a6dcfe50£17.66FiveSpiritualityIn stock (18 available)In The Four Agreements, don Miguel Ruiz reveal...
34Sophie's World6be3beb0793a53e7£15.94FivePhilosophyIn stock (18 available)A page-turning novel that is also an explorati...
47Untitled Collection: Sabbath Poems 2014657fe5ead67a7767£14.27FourPoetryIn stock (16 available)More than thirty-five years ago, when the weat...
53This One Summer51653ef291ab7ddc£19.49FourSequential ArtIn stock (16 available)Every summer, Rose goes with her mom and dad t...
........................
913The Zombie Room9c96cd1329fbd82d£19.69FiveDefaultIn stock (1 available)An unlikely bond is forged between three men f...
917The Silent Wifeb78deb463531d078£12.34FiveFictionIn stock (1 available)A chilling psychological thriller about a marr...
934The Girl You Lost4280ac3eab57aa5d£12.29FiveMysteryIn stock (1 available)Eighteen years ago your baby daughter was snat...
937The Edge of Reason (Bridget Jones #2)29fc016c459aeb14£19.18FourWomens FictionIn stock (1 available)Monday 27 January“7:15 a.m. Hurrah! The wilder...
997A Spy's Devotion (The Regency Spies of London #1)19fec36a1dfb4c16£16.97FiveHistorical FictionIn stock (1 available)In England’s Regency era, manners and elegance...
\n", + "

75 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " title upc \\\n", + "12 Set Me Free ce6396b0f23f6ecc \n", + "30 The Four Agreements: A Practical Guide to Pers... 6258a1f6a6dcfe50 \n", + "34 Sophie's World 6be3beb0793a53e7 \n", + "47 Untitled Collection: Sabbath Poems 2014 657fe5ead67a7767 \n", + "53 This One Summer 51653ef291ab7ddc \n", + ".. ... ... \n", + "913 The Zombie Room 9c96cd1329fbd82d \n", + "917 The Silent Wife b78deb463531d078 \n", + "934 The Girl You Lost 4280ac3eab57aa5d \n", + "937 The Edge of Reason (Bridget Jones #2) 29fc016c459aeb14 \n", + "997 A Spy's Devotion (The Regency Spies of London #1) 19fec36a1dfb4c16 \n", + "\n", + " price_taxIncluded rating genre availability \\\n", + "12 £17.46 Five Young Adult In stock (19 available) \n", + "30 £17.66 Five Spirituality In stock (18 available) \n", + "34 £15.94 Five Philosophy In stock (18 available) \n", + "47 £14.27 Four Poetry In stock (16 available) \n", + "53 £19.49 Four Sequential Art In stock (16 available) \n", + ".. ... ... ... ... \n", + "913 £19.69 Five Default In stock (1 available) \n", + "917 £12.34 Five Fiction In stock (1 available) \n", + "934 £12.29 Five Mystery In stock (1 available) \n", + "937 £19.18 Four Womens Fiction In stock (1 available) \n", + "997 £16.97 Five Historical Fiction In stock (1 available) \n", + "\n", + " description \n", + "12 Aaron Ledbetter’s future had been planned out ... \n", + "30 In The Four Agreements, don Miguel Ruiz reveal... \n", + "34 A page-turning novel that is also an explorati... \n", + "47 More than thirty-five years ago, when the weat... \n", + "53 Every summer, Rose goes with her mom and dad t... \n", + ".. ... \n", + "913 An unlikely bond is forged between three men f... \n", + "917 A chilling psychological thriller about a marr... \n", + "934 Eighteen years ago your baby daughter was snat... \n", + "937 Monday 27 January“7:15 a.m. Hurrah! The wilder... \n", + "997 In England’s Regency era, manners and elegance... \n", + "\n", + "[75 rows x 7 columns]" + ] + }, + "execution_count": 303, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "df_books_lowPrice_highRating = get_book_data(min_rating= 4,max_price = 19)\n", + "df_books_lowPrice_highRating" ] } ], @@ -126,7 +417,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "3.10.10", "language": "python", "name": "python3" }, @@ -140,7 +431,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.10.10" } }, "nbformat": 4,