a9a.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_a9a(path = './'):\n",
    "    \"\"\"\n",
    "        input: path\n",
    "        return: lib svm files\n",
    "    \"\"\"\n",
    "    X_train, y_train = load_svmlight_file(path+'a9a')\n",
    "    X_test, y_test = load_svmlight_file(path+'a9a.t')\n",
    "\n",
    "    y_test[y_test==-1] = 0\n",
    "    y_train[y_train==-1] = 0\n",
    "    \n",
    "    \n",
    "    return {\"Xtr\":X_train, \"ytr\":y_train, \"Xtst\":X_test, \"ytst\":y_test}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_data_a9a()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xgb_classifier(data, xgb_params, K=5):\n",
    "    \"\"\"\n",
    "        input: hyperparameters for the model\n",
    "        return: predictions\n",
    "    \"\"\"\n",
    "    kf = KFold(n_splits = K, random_state = 3228, shuffle = True)\n",
    "    xgb_preds = []\n",
    "    cv_preds = []\n",
    "    cv_labels = []\n",
    "    train_preds = []\n",
    "    train_labels = []\n",
    "    \n",
    "    for train_index, cv_index in kf.split(data['Xtr']):\n",
    "        train_X, valid_X = data['Xtr'][train_index], data['Xtr'][cv_index]\n",
    "        train_y, valid_y = data['ytr'][train_index], data['ytr'][cv_index]\n",
    "\n",
    "        dtrain = xgb.DMatrix(train_X, train_y)\n",
    "        dvalid = xgb.DMatrix(valid_X, valid_y)\n",
    "        dtest = xgb.DMatrix(data['Xtst'])\n",
    "\n",
    "        watchlist = [(dtrain, 'train'), (dvalid, 'valid')]\n",
    "\n",
    "        model = xgb.train(xgb_params, dtrain, 1000, watchlist, early_stopping_rounds=100, maximize=False, verbose_eval=50)\n",
    "        \n",
    "        #train\n",
    "        dtrain = xgb.DMatrix(train_X)\n",
    "        train_preds.append(list(model.predict(dtrain)))\n",
    "        train_labels.append(train_y) \n",
    "        \n",
    "        #cross validation\n",
    "        dcv = xgb.DMatrix(valid_X)\n",
    "        cv_preds.append(list(model.predict(dcv)))\n",
    "        cv_labels.append(valid_y)\n",
    "        \n",
    "        #test\n",
    "        xgb_pred = model.predict(dtest)\n",
    "        xgb_preds.append(list(xgb_pred))\n",
    "        \n",
    "    return xgb_preds, cv_preds, cv_labels, train_preds, train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svm_classifier(data, params, K=5):\n",
    "    \"\"\"\n",
    "       input: hyper params, and data\n",
    "       return: predictions\n",
    "    \"\"\"\n",
    "    kf = KFold(n_splits = K, random_state = 3228, shuffle = True)\n",
    "    svc_preds = []\n",
    "    cv_preds = []\n",
    "    cv_labels = []\n",
    "    train_preds = []\n",
    "    train_labels = []\n",
    "    for train_index, cv_index in kf.split(data['Xtr']):\n",
    "        train_X, valid_X = data['Xtr'][train_index], data['Xtr'][cv_index]\n",
    "        train_y, valid_y = data['ytr'][train_index], data['ytr'][cv_index]\n",
    "\n",
    "        model = SVC(**params)\n",
    "        model.fit(train_X, train_y)\n",
    "        \n",
    "        #train predictions\n",
    "        train_preds.append(list(model.predict(train_X)))\n",
    "        train_labels.append(train_y)\n",
    "        \n",
    "        #cv predictions\n",
    "        cv_preds.append(list(model.predict(valid_X)))\n",
    "        cv_labels.append(valid_y)\n",
    "        \n",
    "        #test predictions\n",
    "        svc_pred = model.predict(data['Xtst'])\n",
    "        svc_preds.append(list(svc_pred))\n",
    "\n",
    "    return svc_preds, cv_preds, cv_labels, train_preds, train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_pars = {'eta': 0.1, 'colsample_bytree': 0.3, 'max_depth': 6, \"min_child_weight\":3, \"lambda\": .001, \"alpha\": .5,\n",
    "            'subsample':.6,'nthread': 4, 'objective': 'binary:logistic'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-error:0.184314\tvalid-error:0.193766\n",
      "Multiple eval metrics have been passed: 'valid-error' will be used for early stopping.\n",
      "\n",
      "Will train until valid-error hasn't improved in 100 rounds.\n",
      "[50]\ttrain-error:0.146537\tvalid-error:0.161523\n",
      "[100]\ttrain-error:0.140625\tvalid-error:0.157992\n",
      "[150]\ttrain-error:0.135826\tvalid-error:0.157838\n",
      "Stopping. Best iteration:\n",
      "[85]\ttrain-error:0.141431\tvalid-error:0.156149\n",
      "\n",
      "[0]\ttrain-error:0.203002\tvalid-error:0.212224\n",
      "Multiple eval metrics have been passed: 'valid-error' will be used for early stopping.\n",
      "\n",
      "Will train until valid-error hasn't improved in 100 rounds.\n",
      "[50]\ttrain-error:0.14634\tvalid-error:0.160934\n",
      "[100]\ttrain-error:0.138854\tvalid-error:0.154484\n",
      "[150]\ttrain-error:0.135744\tvalid-error:0.155405\n",
      "[200]\ttrain-error:0.133057\tvalid-error:0.155098\n",
      "Stopping. Best iteration:\n",
      "[127]\ttrain-error:0.137049\tvalid-error:0.15387\n",
      "\n",
      "[0]\ttrain-error:0.204192\tvalid-error:0.209613\n",
      "Multiple eval metrics have been passed: 'valid-error' will be used for early stopping.\n",
      "\n",
      "Will train until valid-error hasn't improved in 100 rounds.\n",
      "[50]\ttrain-error:0.149871\tvalid-error:0.150184\n",
      "[100]\ttrain-error:0.143499\tvalid-error:0.146499\n",
      "[150]\ttrain-error:0.139698\tvalid-error:0.145577\n",
      "[200]\ttrain-error:0.135092\tvalid-error:0.147574\n",
      "Stopping. Best iteration:\n",
      "[128]\ttrain-error:0.140389\tvalid-error:0.144042\n",
      "\n",
      "[0]\ttrain-error:0.20519\tvalid-error:0.209152\n",
      "Multiple eval metrics have been passed: 'valid-error' will be used for early stopping.\n",
      "\n",
      "Will train until valid-error hasn't improved in 100 rounds.\n",
      "[50]\ttrain-error:0.14918\tvalid-error:0.150338\n",
      "[100]\ttrain-error:0.142347\tvalid-error:0.146806\n",
      "[150]\ttrain-error:0.138201\tvalid-error:0.146959\n",
      "[200]\ttrain-error:0.13513\tvalid-error:0.146345\n",
      "Stopping. Best iteration:\n",
      "[123]\ttrain-error:0.140504\tvalid-error:0.144963\n",
      "\n",
      "[0]\ttrain-error:0.206534\tvalid-error:0.203471\n",
      "Multiple eval metrics have been passed: 'valid-error' will be used for early stopping.\n",
      "\n",
      "Will train until valid-error hasn't improved in 100 rounds.\n",
      "[50]\ttrain-error:0.150063\tvalid-error:0.145424\n",
      "[100]\ttrain-error:0.143768\tvalid-error:0.142967\n",
      "[150]\ttrain-error:0.140658\tvalid-error:0.140203\n",
      "[200]\ttrain-error:0.137587\tvalid-error:0.139588\n",
      "[250]\ttrain-error:0.134938\tvalid-error:0.141738\n",
      "Stopping. Best iteration:\n",
      "[155]\ttrain-error:0.140197\tvalid-error:0.139435\n",
      "\n"
     ]
    }
   ],
   "source": [
    "K=5\n",
    "test_preds, cv_preds, cv_labels, train_preds, train_labels = xgb_classifier(data, xgb_pars, K = K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge(preds):\n",
    "    total_len = 0\n",
    "    predictions = np.array([])\n",
    "    for k in range(len(preds)):\n",
    "        total_len+=len(preds[k])\n",
    "        predictions = np.concatenate((predictions, np.array(preds[k])), axis = 0)\n",
    "    predictions.reshape((total_len,))\n",
    "    predictions[predictions>=.5] = 1\n",
    "    predictions[predictions<.5] = 0\n",
    "    return predictions\n",
    "def mergeKFoldPredictions(preds, K = 5):\n",
    "    final_preds = []\n",
    "    for i in range(len(preds[0])):\n",
    "        res = 0\n",
    "        for j in range(len(preds)):\n",
    "            res += preds[j][i]\n",
    "        final_preds.append(res*(1/len(preds)))\n",
    "    predictions = np.array(final_preds)\n",
    "    predictions[predictions>=.5] = 1\n",
    "    predictions[predictions<.5] = 0\n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of train Set: 86.59%\n",
      "Error of train Set: 13.41%\n"
     ]
    }
   ],
   "source": [
    "train_predictions = merge(train_preds)\n",
    "train_labels = merge(train_labels)\n",
    "accuracy = accuracy_score(train_labels, train_predictions)\n",
    "print(\"Accuracy of train Set: %.2f%%\" % (accuracy * 100.0))\n",
    "print(\"Error of train Set: %.2f%%\" % (100 - accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of cv Set: 84.99%\n",
      "Error of cv Set: 15.01%\n"
     ]
    }
   ],
   "source": [
    "cv_predictions = merge(cv_preds)\n",
    "cv_labels = merge(cv_labels)\n",
    "accuracy = accuracy_score(cv_labels, cv_predictions)\n",
    "print(\"Accuracy of cv Set: %.2f%%\" % (accuracy * 100.0))\n",
    "print(\"Error of cv Set: %.2f%%\" % (100 - accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of test Set: 85.28%\n",
      "Error of test Set: 14.72%\n"
     ]
    }
   ],
   "source": [
    "test_predictions = mergeKFoldPredictions(test_preds, K)\n",
    "accuracy = accuracy_score(data['ytst'], test_predictions)\n",
    "print(\"Accuracy of test Set: %.2f%%\" % (accuracy * 100.0))\n",
    "print(\"Error of test Set: %.2f%%\" % (100 - accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM][LibSVM]"
     ]
    }
   ],
   "source": [
    "K=2\n",
    "svm_pars = {\"probability\":True, \"C\":1, \"gamma\":.005, \"kernel\":'rbf', 'verbose':True}\n",
    "test_preds2, cv_preds2, cv_labels2, train_preds2, train_labels2 = svm_classifier(data, svm_pars, K = K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of train Set: 84.44%\n",
      "Error of train Set: 15.56%\n"
     ]
    }
   ],
   "source": [
    "train_predictions = merge(train_preds2)\n",
    "train_labels = merge(train_labels2)\n",
    "accuracy = accuracy_score(train_labels, train_predictions)\n",
    "print(\"Accuracy of train Set: %.2f%%\" % (accuracy * 100.0))\n",
    "print(\"Error of train Set: %.2f%%\" % (100 - accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of cv Set: 84.32%\n",
      "Error of cv Set: 15.68%\n"
     ]
    }
   ],
   "source": [
    "cv_predictions = merge(cv_preds2)\n",
    "cv_labels = merge(cv_labels2)\n",
    "accuracy = accuracy_score(cv_labels, cv_predictions)\n",
    "print(\"Accuracy of cv Set: %.2f%%\" % (accuracy * 100.0))\n",
    "print(\"Error of cv Set: %.2f%%\" % (100 - accuracy * 100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of test Set: 84.88%\n",
      "Error of test Set: 15.12%\n"
     ]
    }
   ],
   "source": [
    "test_predictions = mergeKFoldPredictions(test_preds2, K)\n",
    "accuracy = accuracy_score(data['ytst'], test_predictions)\n",
    "print(\"Accuracy of test Set: %.2f%%\" % (accuracy * 100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}