FM-100k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>20000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>20000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>328\\t519\\t5\\t885046420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             0\n",
       "count                    20000\n",
       "unique                   20000\n",
       "top     328\\t519\\t5\\t885046420\n",
       "freq                         1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_path = './ml-100k'\n",
    "base1 = pd.read_csv(os.path.join(dataset_path,'u1.base'), header=None)\n",
    "base1.describe()\n",
    "test1 = pd.read_csv(os.path.join(dataset_path,'u1.test'), header=None)\n",
    "test1.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>80000.00000</td>\n",
       "      <td>80000.000000</td>\n",
       "      <td>80000.000000</td>\n",
       "      <td>8.000000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>525.16570</td>\n",
       "      <td>425.705200</td>\n",
       "      <td>3.528350</td>\n",
       "      <td>8.835645e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>255.94956</td>\n",
       "      <td>331.383936</td>\n",
       "      <td>1.118565</td>\n",
       "      <td>5.318611e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.747247e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>334.00000</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>8.794556e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>535.00000</td>\n",
       "      <td>321.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.828441e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>744.00000</td>\n",
       "      <td>631.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.882676e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>943.00000</td>\n",
       "      <td>1682.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>8.932866e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           user_id      movie_id        rating     timestamp\n",
       "count  80000.00000  80000.000000  80000.000000  8.000000e+04\n",
       "mean     525.16570    425.705200      3.528350  8.835645e+08\n",
       "std      255.94956    331.383936      1.118565  5.318611e+06\n",
       "min        1.00000      1.000000      1.000000  8.747247e+08\n",
       "25%      334.00000    175.000000      3.000000  8.794556e+08\n",
       "50%      535.00000    321.000000      4.000000  8.828441e+08\n",
       "75%      744.00000    631.000000      4.000000  8.882676e+08\n",
       "max      943.00000   1682.000000      5.000000  8.932866e+08"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.zeros([80000,4])\n",
    "\n",
    "iloc = 0\n",
    "with open(os.path.join(dataset_path,'u1.base')) as f:\n",
    "    for line in f:\n",
    "        data[iloc,:] = line.split('\\t')\n",
    "        iloc += 1\n",
    "        \n",
    "ratings_train = pd.DataFrame(data,columns=['user_id','movie_id','rating','timestamp'])\n",
    "ratings_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>20000.00000</td>\n",
       "      <td>20000.000000</td>\n",
       "      <td>20000.00000</td>\n",
       "      <td>2.000000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>211.76095</td>\n",
       "      <td>424.829850</td>\n",
       "      <td>3.53590</td>\n",
       "      <td>8.833864e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>121.64690</td>\n",
       "      <td>328.452907</td>\n",
       "      <td>1.15368</td>\n",
       "      <td>5.441471e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>8.747247e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>99.00000</td>\n",
       "      <td>174.000000</td>\n",
       "      <td>3.00000</td>\n",
       "      <td>8.793730e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>222.00000</td>\n",
       "      <td>322.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>8.824747e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>308.00000</td>\n",
       "      <td>633.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>8.882045e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>462.00000</td>\n",
       "      <td>1591.000000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>8.932777e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           user_id      movie_id       rating     timestamp\n",
       "count  20000.00000  20000.000000  20000.00000  2.000000e+04\n",
       "mean     211.76095    424.829850      3.53590  8.833864e+08\n",
       "std      121.64690    328.452907      1.15368  5.441471e+06\n",
       "min        1.00000      1.000000      1.00000  8.747247e+08\n",
       "25%       99.00000    174.000000      3.00000  8.793730e+08\n",
       "50%      222.00000    322.000000      4.00000  8.824747e+08\n",
       "75%      308.00000    633.000000      4.00000  8.882045e+08\n",
       "max      462.00000   1591.000000      5.00000  8.932777e+08"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.zeros([20000,4])\n",
    "\n",
    "iloc = 0\n",
    "with open(os.path.join(dataset_path,'u1.test')) as f:\n",
    "    for line in f:\n",
    "        data[iloc,:] = line.split('\\t')\n",
    "        iloc += 1\n",
    "        \n",
    "ratings_test = pd.DataFrame(data,columns=['user_id','movie_id','rating','timestamp'])\n",
    "ratings_test.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100000.00000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>1.000000e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>462.48475</td>\n",
       "      <td>425.530130</td>\n",
       "      <td>3.529860</td>\n",
       "      <td>8.835289e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>266.61442</td>\n",
       "      <td>330.798356</td>\n",
       "      <td>1.125674</td>\n",
       "      <td>5.343856e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.747247e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>254.00000</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>8.794487e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>447.00000</td>\n",
       "      <td>322.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.828269e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>682.00000</td>\n",
       "      <td>631.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.882600e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>943.00000</td>\n",
       "      <td>1682.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>8.932866e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            user_id       movie_id         rating     timestamp\n",
       "count  100000.00000  100000.000000  100000.000000  1.000000e+05\n",
       "mean      462.48475     425.530130       3.529860  8.835289e+08\n",
       "std       266.61442     330.798356       1.125674  5.343856e+06\n",
       "min         1.00000       1.000000       1.000000  8.747247e+08\n",
       "25%       254.00000     175.000000       3.000000  8.794487e+08\n",
       "50%       447.00000     322.000000       4.000000  8.828269e+08\n",
       "75%       682.00000     631.000000       4.000000  8.882600e+08\n",
       "max       943.00000    1682.000000       5.000000  8.932866e+08"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.zeros([100000,4])\n",
    "\n",
    "iloc = 0\n",
    "with open(os.path.join(dataset_path,'u.data')) as f:\n",
    "    for line in f:\n",
    "        data[iloc,:] = line.split('\\t')\n",
    "        iloc += 1\n",
    "        \n",
    "ratings = pd.DataFrame(data,columns=['user_id','movie_id','rating','timestamp'])\n",
    "ratings.head(10)\n",
    "ratings.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['unknown', 'Action', 'Adventure', 'Animation', \"Children's\", 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']\n"
     ]
    }
   ],
   "source": [
    "genres_info = pd.read_csv(os.path.join(dataset_path,'u.genre'),header=None,delimiter=\"|\")\n",
    "genres_info = genres_info.rename(columns={0:'genre'}).drop(columns=[1])\n",
    "genres = list(genres_info['genre'])\n",
    "print(genres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unknown</th>\n",
       "      <th>action</th>\n",
       "      <th>adventure</th>\n",
       "      <th>animation</th>\n",
       "      <th>children's</th>\n",
       "      <th>comedy</th>\n",
       "      <th>crime</th>\n",
       "      <th>documentary</th>\n",
       "      <th>drama</th>\n",
       "      <th>fantasy</th>\n",
       "      <th>film-noir</th>\n",
       "      <th>horror</th>\n",
       "      <th>musical</th>\n",
       "      <th>mystery</th>\n",
       "      <th>romance</th>\n",
       "      <th>sci-fi</th>\n",
       "      <th>thriller</th>\n",
       "      <th>war</th>\n",
       "      <th>western</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movie_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          unknown  action  adventure  animation  children's  comedy  crime  \\\n",
       "movie_id                                                                     \n",
       "1               0       0          0          1           1       1      0   \n",
       "2               0       1          1          0           0       0      0   \n",
       "3               0       0          0          0           0       0      0   \n",
       "4               0       1          0          0           0       1      0   \n",
       "5               0       0          0          0           0       0      1   \n",
       "6               0       0          0          0           0       0      0   \n",
       "7               0       0          0          0           0       0      0   \n",
       "8               0       0          0          0           1       1      0   \n",
       "9               0       0          0          0           0       0      0   \n",
       "10              0       0          0          0           0       0      0   \n",
       "\n",
       "          documentary  drama  fantasy  film-noir  horror  musical  mystery  \\\n",
       "movie_id                                                                     \n",
       "1                   0      0        0          0       0        0        0   \n",
       "2                   0      0        0          0       0        0        0   \n",
       "3                   0      0        0          0       0        0        0   \n",
       "4                   0      1        0          0       0        0        0   \n",
       "5                   0      1        0          0       0        0        0   \n",
       "6                   0      1        0          0       0        0        0   \n",
       "7                   0      1        0          0       0        0        0   \n",
       "8                   0      1        0          0       0        0        0   \n",
       "9                   0      1        0          0       0        0        0   \n",
       "10                  0      1        0          0       0        0        0   \n",
       "\n",
       "          romance  sci-fi  thriller  war  western  \n",
       "movie_id                                           \n",
       "1               0       0         0    0        0  \n",
       "2               0       0         1    0        0  \n",
       "3               0       0         1    0        0  \n",
       "4               0       0         0    0        0  \n",
       "5               0       0         1    0        0  \n",
       "6               0       0         0    0        0  \n",
       "7               0       1         0    0        0  \n",
       "8               0       0         0    0        0  \n",
       "9               0       0         0    0        0  \n",
       "10              0       0         0    1        0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = pd.read_csv(os.path.join(dataset_path,'u.item'),header=None,encoding='ISO-8859-1',delimiter=\"|\")\n",
    "movies = movies.rename(columns={0:'movie_id',1:'title'}).drop(columns=[2,3,4])\n",
    "genres_names = {}\n",
    "for i in range(5,24):\n",
    "    genre = genres[i-5].lower()\n",
    "    #genre = re.sub(\"[^a-zA-Z0-9 ]\", \" \", genre)\n",
    "    #genre = genre.strip()\n",
    "    #genre = re.sub(\"\\s+\", \" \", genre)\n",
    "    genres_names[i] = genre\n",
    "    \n",
    "movies = movies.rename(columns=genres_names).set_index(['movie_id']).drop(columns=['title'])\n",
    "#movies.describe()\n",
    "movies.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:19: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "/usr/local/Cellar/python/3.7.0/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:21: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scipy.sparse.csr.csr_matrix'>\n"
     ]
    }
   ],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "from scipy import sparse\n",
    "\n",
    "num_ratings = 100000\n",
    "num_users = 943\n",
    "num_movies = 1682\n",
    "\n",
    "# userId matrix with shape[100000,943]\n",
    "uid_mat = csr_matrix(([], ([], [])), shape=(num_ratings, num_users))\n",
    "\n",
    "# movieId matrix with shape[100000,1682]\n",
    "mid_mat = csr_matrix(([], ([], [])), shape=(num_ratings, num_movies))\n",
    "\n",
    "# uids contains a sequence of user_id in the order of ratings\n",
    "uids = list(ratings['user_id']-1)\n",
    "mids = list(ratings['movie_id']-1)\n",
    "\n",
    "# In each rating, the user who rated the movie is set to one\n",
    "uid_mat[range(num_ratings), uids] = 1\n",
    "# In each rating, the movie which is rated is set to one\n",
    "mid_mat[range(num_ratings), mids] = 1\n",
    "\n",
    "print(type(uid_mat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1682, 21)\n",
      "[[0.         0.         0.74066017 ... 0.         0.         0.        ]\n",
      " [0.53676706 0.65097024 0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " ...\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]\n",
      " [0.         0.         0.         ... 0.         0.         0.        ]]\n",
      "[0.         0.         0.74066017 0.57387209 0.34941857 0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "# stop_words = set(stopwords.words('english'))   # have to download\n",
    "movie_id = list(ratings['movie_id'])\n",
    "genres = movies.columns\n",
    "#print(movie_id)\n",
    "\n",
    "movie_genres = []\n",
    "for iloc in range(num_movies):\n",
    "    judge = list(movies.iloc[iloc]==1)\n",
    "    judge_index = [i for i, x in enumerate(judge) if x]\n",
    "    movie_genres.append(' '.join([genres[i] for i in judge_index]))\n",
    "\n",
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(movie_genres)\n",
    "movie_genres_map = X.toarray()\n",
    "#print(vectorizer.get_feature_names())\n",
    "print(X.shape)\n",
    "print(X.toarray()) # X.toarray() is numpy.ndarray\n",
    "print(movie_genres_map[0])\n",
    "#print(movie_genres)\n",
    "    \n",
    "    \n",
    "    # find genres of each movie being rated. construct tf-idf of genres\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres_mat = np.zeros([num_ratings, 21])\n",
    "\n",
    "for i in range(num_ratings):\n",
    "    genres_mat[i] = movie_genres_map[int(mids[i])]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scipy.sparse.csr.csr_matrix'>\n",
      "(100000, 21)\n",
      "  (0, 4)\t1.0\n",
      "  (1, 5)\t0.38231324554854784\n",
      "  (1, 10)\t0.5342605102970769\n",
      "  (1, 13)\t0.4411132499510552\n",
      "  (1, 14)\t0.5342605102970769\n",
      "  (1, 17)\t0.29729967586589695\n",
      "  (2, 3)\t0.8541290539419342\n",
      "  (2, 4)\t0.5200611110362477\n",
      "  (3, 7)\t0.2480037786742468\n",
      "  (3, 15)\t0.39271725518143463\n",
      "  (3, 19)\t0.5593429655280826\n",
      "  (3, 20)\t0.6865877439630753\n",
      "  (4, 5)\t0.8966427178409158\n",
      "  (4, 7)\t0.44275482667369725\n",
      "  (5, 9)\t0.5597570628006706\n",
      "  (5, 16)\t0.5597570628006706\n",
      "  (5, 19)\t0.611018871467594\n",
      "  (6, 0)\t0.7071067811865476\n",
      "  (6, 17)\t0.7071067811865476\n",
      "  (7, 1)\t0.6035270741421865\n",
      "  (7, 3)\t0.6207745808557714\n",
      "  (7, 15)\t0.5003938354343638\n",
      "  (8, 4)\t0.3857741712122011\n",
      "  (8, 12)\t0.7683400720900841\n",
      "  (8, 15)\t0.5107169690211435\n",
      "  :\t:\n",
      "  (99988, 19)\t0.6088519546192082\n",
      "  (99989, 1)\t0.43128439456419654\n",
      "  (99989, 2)\t0.5725386362198002\n",
      "  (99989, 3)\t0.4436095756694238\n",
      "  (99989, 12)\t0.5379626617297519\n",
      "  (99990, 7)\t0.5339496321125392\n",
      "  (99990, 15)\t0.8455162862812781\n",
      "  (99991, 2)\t0.6345915113407754\n",
      "  (99991, 3)\t0.4916888630049177\n",
      "  (99991, 12)\t0.5962681240274058\n",
      "  (99992, 7)\t1.0\n",
      "  (99993, 4)\t0.4201175950444482\n",
      "  (99993, 7)\t0.3512338190957366\n",
      "  (99993, 12)\t0.8367413045006755\n",
      "  (99994, 1)\t0.6548237202125763\n",
      "  (99994, 3)\t0.6735371748270751\n",
      "  (99994, 7)\t0.342860860368931\n",
      "  (99995, 4)\t1.0\n",
      "  (99996, 4)\t0.3788381736710695\n",
      "  (99996, 9)\t0.6544011148254443\n",
      "  (99996, 16)\t0.6544011148254443\n",
      "  (99997, 17)\t1.0\n",
      "  (99998, 3)\t0.8541290539419342\n",
      "  (99998, 4)\t0.5200611110362477\n",
      "  (99999, 20)\t1.0\n"
     ]
    }
   ],
   "source": [
    "genres_mat = sparse.csr_matrix(genres_mat)\n",
    "print(type(genres_mat))\n",
    "print(genres_mat.shape)\n",
    "print(genres_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3. 3. 1. ... 1. 2. 3.]\n"
     ]
    }
   ],
   "source": [
    "ratings_mat = np.array(list(ratings['rating']))#.reshape((100000,1))\n",
    "print(ratings_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import hstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mat = hstack((uid_mat, mid_mat, genres_mat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 2646)\n",
      "<class 'scipy.sparse.coo.coo_matrix'>\n"
     ]
    }
   ],
   "source": [
    "print(mat.shape)\n",
    "print(type(mat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(mat, ratings_mat, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80000, 2646)\n",
      "<class 'scipy.sparse.csr.csr_matrix'>\n",
      "[2. 5. 2. ... 4. 4. 4.]\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(type(X_train))\n",
    "print(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating validation dataset of 0.01 of training for adaptive regularization\n",
      "-- Epoch 1\n",
      "Training MSE: 0.49700\n",
      "-- Epoch 2\n",
      "Training MSE: 0.40545\n",
      "-- Epoch 3\n",
      "Training MSE: 0.35863\n",
      "-- Epoch 4\n",
      "Training MSE: 0.32223\n",
      "-- Epoch 5\n",
      "Training MSE: 0.29047\n",
      "-- Epoch 6\n",
      "Training MSE: 0.26660\n",
      "-- Epoch 7\n",
      "Training MSE: 0.24706\n",
      "-- Epoch 8\n",
      "Training MSE: 0.23369\n",
      "-- Epoch 9\n",
      "Training MSE: 0.22487\n",
      "-- Epoch 10\n",
      "Training MSE: 0.21944\n"
     ]
    }
   ],
   "source": [
    "from pyfm import pylibfm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "fm = pylibfm.FM(num_factors=10, num_iter=10, verbose=True, task=\"regression\", initial_learning_rate=0.01,learning_rate_schedule=\"optimal\")\n",
    "fm.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000,)\n"
     ]
    }
   ],
   "source": [
    "y_pred = fm.predict(X_test)\n",
    "print(y_pred.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.892142971065922\n",
      "[3. 5. 3. 4. 5. 2. 5. 1. 4. 3. 4. 4. 4. 2. 5. 3. 3. 4. 5. 4. 5. 5. 4. 2.\n",
      " 4. 3. 4. 2. 3. 3. 2. 1. 4. 3. 2. 2. 5. 3. 4. 1. 3. 4. 5. 3. 5. 4. 4. 3.\n",
      " 5. 5. 1. 3. 4. 5. 3. 4. 3. 5. 5. 5. 4. 1. 5. 4. 3. 1. 5. 3. 3. 3. 3. 3.\n",
      " 3. 2. 5. 4. 3. 4. 4. 3. 4. 3. 3. 4. 5. 5. 4. 5. 2. 3. 1. 3. 3. 4. 5. 3.\n",
      " 3. 3. 4. 5.]\n",
      "[2.94921001 4.37496084 3.8657114  2.97592594 3.26181406 3.31476648\n",
      " 4.41896293 3.63209577 3.70890477 3.05973414 3.50520487 3.89476743\n",
      " 4.1627275  2.46182869 3.99209345 3.28852833 3.04708811 4.16556101\n",
      " 5.         3.04917458 4.3773367  3.84742342 4.05157636 4.37698798\n",
      " 4.85511533 3.27899954 3.98278593 2.51553428 4.51890901 3.33887982\n",
      " 2.61439141 1.         3.31176992 2.89112397 3.23881256 3.15427657\n",
      " 3.03928193 3.79171704 4.06015165 1.73906179 3.38954273 3.36427961\n",
      " 4.91987195 2.7478592  3.63521289 3.50586002 4.05261537 2.82628284\n",
      " 4.45836013 4.44045734 3.22277086 3.40514061 3.1873767  4.35392087\n",
      " 4.01720199 3.99805067 2.62420879 4.55532768 4.985836   4.83482209\n",
      " 2.64371967 3.54800983 4.61321057 4.01041937 3.10410013 3.49048454\n",
      " 3.96380808 2.46311218 4.19675027 4.38497472 3.08294661 4.15112661\n",
      " 2.59503861 3.97622786 3.77648203 3.69112093 3.30709484 3.2834924\n",
      " 4.75866576 3.80712967 3.64507986 2.72731258 3.0487426  4.42713412\n",
      " 4.34120139 4.59844123 4.14588616 3.87595222 2.71240488 4.17433887\n",
      " 2.60100274 3.95441331 3.36249109 3.64066306 4.37885933 3.96929898\n",
      " 3.63160225 3.50092892 4.02356304 4.81226897]\n"
     ]
    }
   ],
   "source": [
    "print(mean_squared_error(y_test, y_pred))\n",
    "print(y_test[:100])\n",
    "print(y_pred[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow3",
   "language": "python",
   "name": "tensorflow3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}