naver_movie_crawling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reviews with ratings saved to ver3_reviews_data.csv\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "df = pd.read_csv('0530_dataset\\cleaned_preprosessing_ver3.csv')\n",
    "\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument(\"chrome-win64/chromedriver.exe\")\n",
    "driver = webdriver.Chrome(options=options)\n",
    "\n",
    "all_reviews_df=[]\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "        movie_title = row['Movie_Title']\n",
    "        release_date = datetime.strptime(row['Release_Date'], '%Y-%m-%d')\n",
    "    \n",
    "        # 개봉일 전후 1주일 기간 설정\n",
    "    \n",
    "        week1_end_date = (release_date + timedelta(days=6))\n",
    "        week2_end_date = (week1_end_date + timedelta(days=7))\n",
    "        week3_end_date = (week2_end_date + timedelta(days=7))\n",
    "        start_date = release_date\n",
    "\n",
    "        url = f\"https://search.naver.com/search.naver?sm=tab_sug.top&where=nexearch&ssc=tab.nx.all&query=영화+{movie_title}+관람평\"\n",
    "\n",
    "\n",
    "\n",
    "        driver.get(url)\n",
    "        time.sleep(5)\n",
    "        try:\n",
    "            # Click on the area to load reviews\n",
    "            area_element = driver.find_element(By.CLASS_NAME, 'area_card_outer')\n",
    "            area_element.click()\n",
    "            time.sleep(2)  # Wait for content to load (adjust as needed)\n",
    "            \n",
    "            # Find the specific element to scroll within\n",
    "            scrollable_element = driver.find_element(By.CLASS_NAME, 'lego_review_list')\n",
    "            \n",
    "            # Scroll down within the specific element to load all reviews\n",
    "            last_height = driver.execute_script(\"return arguments[0].scrollHeight;\", scrollable_element)\n",
    "            while True:\n",
    "                driver.execute_script(\"arguments[0].scrollTo(0, arguments[0].scrollHeight);\", scrollable_element)\n",
    "                time.sleep(1)  # Wait for content to load (adjust as needed)\n",
    "                \n",
    "                new_height = driver.execute_script(\"return arguments[0].scrollHeight;\", scrollable_element)\n",
    "                if new_height == last_height:\n",
    "                    break\n",
    "                last_height = new_height\n",
    "    \n",
    "            page_source = driver.page_source\n",
    "    \n",
    "    \n",
    "            soup = BeautifulSoup(page_source, 'html.parser')\n",
    "            \n",
    "            review_items = soup.find_all('li', class_='area_card _item')\n",
    "            \n",
    "            # Create a list to store review data\n",
    "            reviews_data = []\n",
    "            \n",
    "            for review in review_items:\n",
    "                rating_id = review['data-rating-id']\n",
    "                movie_code = review['data-movie-code']\n",
    "                rating_point_type = review['data-rating-point-type']\n",
    "                writer_id = review['data-report-writer-id']\n",
    "                report_title = review['data-report-title']\n",
    "                report_time = review['data-report-time']\n",
    "                \n",
    "                # Extract the last character as the rating value\n",
    "                rating_value = review.find('div', class_='area_text_box').text.strip()[-1]\n",
    "                \n",
    "                if rating_value =='0':\n",
    "                    rating_value='10'\n",
    "                report_time=report_time[:8]\n",
    "                report_time= datetime.strptime(report_time, '%Y%m%d')\n",
    "            \n",
    "                \n",
    "                reviews_data.append({\n",
    "                    \"Rating ID\": rating_id,\n",
    "                    \"Movie Code\": movie_code,\n",
    "                    \"Rating Point Type\": rating_point_type,\n",
    "                    \"Writer ID\": writer_id,\n",
    "                    \"Report Title\": report_title,\n",
    "                    \"Report Time\": report_time,\n",
    "                    \"Rating\": rating_value\n",
    "                })\n",
    "                \n",
    "                # Create a DataFrame from the reviews data\n",
    "            reviews_df = pd.DataFrame(reviews_data)\n",
    "        \n",
    "            # Calculate average ratings for each week\n",
    "            week1_avg = reviews_df[(reviews_df['Report Time'] >= start_date) & (reviews_df['Report Time'] <= week1_end_date)]['Rating'].astype(float).mean()\n",
    "            week2_avg = reviews_df[(reviews_df['Report Time'] >= start_date) & (reviews_df['Report Time'] <= week2_end_date)]['Rating'].astype(float).mean()\n",
    "            week3_avg = reviews_df[(reviews_df['Report Time'] >= start_date) & (reviews_df['Report Time'] <= week3_end_date)]['Rating'].astype(float).mean()\n",
    "        \n",
    "            # Append the results to the all_reviews_df\n",
    "            all_reviews_df.append({\n",
    "                'Movie_Title': movie_title,\n",
    "                'Week1_Avg': week1_avg,\n",
    "                'Week2_Avg': week2_avg,\n",
    "                'Week3_Avg': week3_avg\n",
    "            })\n",
    "        except:\n",
    "            all_reviews_df.append({\n",
    "                'Movie_Title': movie_title,\n",
    "                'Week1_Avg': 0,\n",
    "                'Week2_Avg': 0,\n",
    "                'Week3_Avg': 0\n",
    "            })\n",
    "        \n",
    "        \n",
    "    \n",
    "driver.quit()\n",
    "all_reviews_df_good = pd.DataFrame(all_reviews_df)\n",
    "# Save the DataFrame to an Excel file\n",
    "excel_filename = \"ver3_reviews_data.csv\"\n",
    "all_reviews_df_good.to_csv(excel_filename, index=False)\n",
    "\n",
    "print(f\"Reviews with ratings saved to {excel_filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# CSV 파일 불러오기\n",
    "data = pd.read_csv(\"ver3_reviews_data.csv\")\n",
    "\n",
    "# Week1_Avg, Week2_Avg, Week3_Avg가 모두 0 또는 NaN인 경우의 Movie_Title 뽑기\n",
    "empty_weeks = data[(data['Week1_Avg'].fillna(0) == 0) & (data['Week2_Avg'].fillna(0) == 0) & (data['Week3_Avg'].fillna(0) == 0)]\n",
    "\n",
    "# 결과 출력\n",
    "empty_weeks.to_csv(\"empty_movie.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# 기존 CSV 파일 읽기\n",
    "data = pd.read_csv(\"empty_movie.csv\")\n",
    "# Release_Date를 포함한 두 번째 CSV 파일 읽기\n",
    "release_dates = pd.read_csv(\"updated_preprosessing_ver7_r11.csv\")\n",
    "\n",
    "# Release_Date 열을 기존 데이터에 추가\n",
    "data['Release_Date'] = release_dates['Release_Date']\n",
    "\n",
    "# 결과를 새로운 CSV 파일로 저장\n",
    "data.to_csv(\"empty_movie.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}