semi-supervised.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "\n",
    "# Read The data\n",
    "training_set = pd.read_json('./data/train_set.json')\n",
    "test_set = pd.read_json('./data/test_set.json')\n",
    "\n",
    "# Use logistic regression to predict the class\n",
    "\n",
    "# 1. word2vec\n",
    "vectorizer = TfidfVectorizer(max_features=10000)\n",
    "X = vectorizer.fit_transform(training_set['text'])\n",
    "\n",
    "# 2. classification\n",
    "clf = LogisticRegression()\n",
    "clf.fit(X[500:], training_set['label'][500:])\n",
    "X_test = vectorizer.transform(test_set['text'])\n",
    "predictions = clf.predict(X_test)\n",
    "predict_probas = clf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [],
   "source": [
    "valid = clf.predict(X[0:500])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [
    {
     "data": {
      "text/plain": "74"
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "while np.sum(np.max(predict_probas, axis = 1) > 0.8) >0:\n",
    "    for i in range(predictions):\n",
    "        if predict_probas > 0.8:\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "data": {
      "text/plain": "0.606"
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.mean(valid==training_set['label'][0:500])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [
    "from scipy.sparse import vstack\n",
    "new_X = vstack((X[500:], X_test))\n",
    "new_label = np.hstack((training_set['label'][500:].to_numpy(), predictions))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "data": {
      "text/plain": "0.578"
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. classification\n",
    "clf = LogisticRegression()\n",
    "clf.fit(new_X, new_label)\n",
    "valid = clf.predict(X[0:500])\n",
    "np.mean(valid==training_set['label'][0:500])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8302857142857143"
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(clf.predict(X[500:])==training_set['label'][500:])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "outputs": [],
   "source": [
    "predictions = clf.predict(X_test)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "test_label_gpt = []\n",
    "with open(\"submission_gpt.csv\", \"r\") as pred:\n",
    "    csv_in = csv.reader(pred)\n",
    "    for i, row in enumerate(csv_in):\n",
    "        if i>=1:\n",
    "            test_label_gpt.append(eval(row[1]))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "outputs": [],
   "source": [
    "\n",
    "test_label_xgboost = []\n",
    "with open(\"submission_xgboost_83.csv\", \"r\") as pred:\n",
    "    csv_in = csv.reader(pred)\n",
    "    for i, row in enumerate(csv_in):\n",
    "        if i>=1:\n",
    "            test_label_xgboost.append(eval(row[1]))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "outputs": [],
   "source": [
    "test_label_large = []\n",
    "with open(\"submission_large.csv\", \"r\") as pred:\n",
    "    csv_in = csv.reader(pred)\n",
    "    for i, row in enumerate(csv_in):\n",
    "        if i>=1:\n",
    "            test_label_large.append(eval(row[1]))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "outputs": [
    {
     "data": {
      "text/plain": "(array([   2,    3,    9, ..., 3993, 3994, 3996]),)"
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(np.array(test_label_xgboost) + np.array(test_label_gpt) + np.array(test_label_large)==3)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "outputs": [
    {
     "data": {
      "text/plain": "1382"
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.array(test_label_xgboost) + np.array(test_label_large)==2)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "outputs": [
    {
     "data": {
      "text/plain": "644"
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(np.array(test_label_xgboost) + np.array(test_label_gpt) + np.array(test_label_large)==0)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "outputs": [
    {
     "data": {
      "text/plain": "1781"
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.array(test_label_xgboost)  + np.array(test_label_large)==0)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "outputs": [
    {
     "data": {
      "text/plain": "4000"
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = []\n",
    "vote = np.array(test_label_xgboost) + np.array(test_label_gpt) + np.array(test_label_large)\n",
    "for i in range(len(test_label_large)):\n",
    "    if vote[i] > 1.5:\n",
    "        predictions.append(1)\n",
    "    else:\n",
    "        predictions.append(0)\n",
    "len(predictions)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "outputs": [],
   "source": [
    "# Write predictions to a file\n",
    "with open(\"submission_merge.csv\", \"w\") as pred:\n",
    "    csv_out = csv.writer(pred)\n",
    "    csv_out.writerow(['id', 'label'])\n",
    "    for i, row in enumerate(predictions):\n",
    "        csv_out.writerow([i, row])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}