DataManagement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Charger le fichier original\n",
    "df = pd.read_csv('data/EDdA_dataframe_withContent.tsv', sep='\\t')\n",
    "\n",
    "# Nettoyage des colonnes non nécessaires\n",
    "columns_to_drop = ['firstParagraph', 'contentWithoutClass', 'ensemble_domaine_enccre', 'domaine_enccre', 'normClass']\n",
    "df_cleaned = df.drop(columns=columns_to_drop)\n",
    "\n",
    "# Enlever les trois premières lignes\n",
    "df_cleaned = df_cleaned.iloc[3:]\n",
    "\n",
    "# Sauvegarder les données nettoyées\n",
    "df_cleaned.to_csv('data/EDdA_dataframe_cleaned.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volume</th>\n",
       "      <th>numero</th>\n",
       "      <th>head</th>\n",
       "      <th>classEDdA</th>\n",
       "      <th>author</th>\n",
       "      <th>id_enccre</th>\n",
       "      <th>content</th>\n",
       "      <th>nb_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>A, a &amp; a</td>\n",
       "      <td>ordre Encyclopéd. Entend. Science de l'homme, ...</td>\n",
       "      <td>Dumarsais5</td>\n",
       "      <td>v1-1-0</td>\n",
       "      <td>A, a &amp; a s.m. (ordre Encyclopéd.\\nEntend. Scie...</td>\n",
       "      <td>1092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>A</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>Dumarsais5</td>\n",
       "      <td>v1-1-1</td>\n",
       "      <td>A, mot, est 1. la troisieme personne du présen...</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>A</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>Dumarsais</td>\n",
       "      <td>v1-1-2</td>\n",
       "      <td>A, préposition vient du latin à, à dextris, à ...</td>\n",
       "      <td>3067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>A</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>Mallet</td>\n",
       "      <td>v1-1-3</td>\n",
       "      <td>A, étoit une lettre numérale parmi les Anciens...</td>\n",
       "      <td>285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>A, lettre symbolique</td>\n",
       "      <td>unclassified</td>\n",
       "      <td>Mallet</td>\n",
       "      <td>v1-1-4</td>\n",
       "      <td>A, lettre symbolique, étoit un hiéroglyphe che...</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74185</th>\n",
       "      <td>17</td>\n",
       "      <td>3204</td>\n",
       "      <td>TRANSFUGE</td>\n",
       "      <td>Art. milit.</td>\n",
       "      <td>Saint-Lambert5</td>\n",
       "      <td>v17-2197-0</td>\n",
       "      <td>TRANSFUGE, s. m. (Art. milit.) La plus grande\\...</td>\n",
       "      <td>9576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74186</th>\n",
       "      <td>17</td>\n",
       "      <td>3205</td>\n",
       "      <td>VÉNUS</td>\n",
       "      <td>Astronom.</td>\n",
       "      <td>unsigned</td>\n",
       "      <td>v17-2198-0</td>\n",
       "      <td>VÉNUS, (Astronom.) satellites de Vénus. Depuis...</td>\n",
       "      <td>3754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74187</th>\n",
       "      <td>17</td>\n",
       "      <td>3206</td>\n",
       "      <td>VERS falisque</td>\n",
       "      <td>Poésie latine.</td>\n",
       "      <td>Jaucourt</td>\n",
       "      <td>v17-2199-0</td>\n",
       "      <td>VERS falisque, (Poésie latine.) vers latin de\\...</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74188</th>\n",
       "      <td>17</td>\n",
       "      <td>3207</td>\n",
       "      <td>VIBRATION, ou OSCILLATION</td>\n",
       "      <td>Horlog.</td>\n",
       "      <td>Romilly5</td>\n",
       "      <td>v17-2200-0</td>\n",
       "      <td>VIBRATION, ou OSCILLATION, s. f. (Horlog.) ter...</td>\n",
       "      <td>4945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74189</th>\n",
       "      <td>17</td>\n",
       "      <td>3208</td>\n",
       "      <td>VINGTIEME, imposition</td>\n",
       "      <td>Econ. pol.</td>\n",
       "      <td>Boulanger</td>\n",
       "      <td>v17-2201-0</td>\n",
       "      <td>VINGTIEME, imposition, s. m. (Econ. pol.)\\ndan...</td>\n",
       "      <td>56857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74187 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       volume  numero                       head  \\\n",
       "3           1       5                   A, a & a   \n",
       "4           1       6                          A   \n",
       "5           1       7                          A   \n",
       "6           1       8                          A   \n",
       "7           1       9       A, lettre symbolique   \n",
       "...       ...     ...                        ...   \n",
       "74185      17    3204                  TRANSFUGE   \n",
       "74186      17    3205                      VÉNUS   \n",
       "74187      17    3206              VERS falisque   \n",
       "74188      17    3207  VIBRATION, ou OSCILLATION   \n",
       "74189      17    3208      VINGTIEME, imposition   \n",
       "\n",
       "                                               classEDdA          author  \\\n",
       "3      ordre Encyclopéd. Entend. Science de l'homme, ...      Dumarsais5   \n",
       "4                                           unclassified      Dumarsais5   \n",
       "5                                           unclassified       Dumarsais   \n",
       "6                                           unclassified          Mallet   \n",
       "7                                           unclassified          Mallet   \n",
       "...                                                  ...             ...   \n",
       "74185                                        Art. milit.  Saint-Lambert5   \n",
       "74186                                          Astronom.        unsigned   \n",
       "74187                                     Poésie latine.        Jaucourt   \n",
       "74188                                            Horlog.        Romilly5   \n",
       "74189                                         Econ. pol.       Boulanger   \n",
       "\n",
       "        id_enccre                                            content  nb_words  \n",
       "3          v1-1-0  A, a & a s.m. (ordre Encyclopéd.\\nEntend. Scie...      1092  \n",
       "4          v1-1-1  A, mot, est 1. la troisieme personne du présen...       381  \n",
       "5          v1-1-2  A, préposition vient du latin à, à dextris, à ...      3067  \n",
       "6          v1-1-3  A, étoit une lettre numérale parmi les Anciens...       285  \n",
       "7          v1-1-4  A, lettre symbolique, étoit un hiéroglyphe che...       118  \n",
       "...           ...                                                ...       ...  \n",
       "74185  v17-2197-0  TRANSFUGE, s. m. (Art. milit.) La plus grande\\...      9576  \n",
       "74186  v17-2198-0  VÉNUS, (Astronom.) satellites de Vénus. Depuis...      3754  \n",
       "74187  v17-2199-0  VERS falisque, (Poésie latine.) vers latin de\\...        97  \n",
       "74188  v17-2200-0  VIBRATION, ou OSCILLATION, s. f. (Horlog.) ter...      4945  \n",
       "74189  v17-2201-0  VINGTIEME, imposition, s. m. (Econ. pol.)\\ndan...     56857  \n",
       "\n",
       "[74187 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_cleaned' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Afficher la distribution de nb_words\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mdf_cleaned\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnb_words\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mdescribe()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df_cleaned' is not defined"
     ]
    }
   ],
   "source": [
    "# Afficher la distribution de nb_words\n",
    "df_cleaned['nb_words'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Faire un sample de 100 lignes\n",
    "df_sample = df_cleaned.sample(3333)\n",
    "df_sample.to_csv('data/EDdA_dataframe_sample.tsv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nombre total de tokens : 36356811\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tiktoken\n",
    "\n",
    "# Charger le fichier nettoyé\n",
    "df_cleaned = pd.read_csv('data/EDdA_dataframe_cleaned.tsv', sep='\\t')\n",
    "\n",
    "# Instancier le tokenizer pour l'encodage des tokens\n",
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "# Convertir la colonne 'content' en chaîne de caractères pour éviter les erreurs\n",
    "df_cleaned['content'] = df_cleaned['content'].astype(str)\n",
    "\n",
    "# Calculer le nombre total de tokens pour chaque texte de la colonne 'content'\n",
    "df_cleaned['token_count'] = df_cleaned['content'].apply(lambda x: len(tokenizer.encode(x)))\n",
    "\n",
    "# Calculer le nombre total de tokens dans le fichier\n",
    "total_tokens = df_cleaned['token_count'].sum()\n",
    "\n",
    "print(f\"Nombre total de tokens : {total_tokens}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(np.float64(0.72713622),\n",
       " np.float64(0.36356811),\n",
       " np.float64(4.7263854300000006),\n",
       " np.float64(2.3631927150000003),\n",
       " np.float64(3.6356811),\n",
       " np.float64(1.81784055))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Modèle text-embedding-3-small\n",
    "cost_small = (total_tokens / 1_000_000) * 0.020\n",
    "batch_cost_small = (total_tokens / 1_000_000) * 0.010\n",
    "\n",
    "# Modèle text-embedding-3-large\n",
    "cost_large = (total_tokens / 1_000_000) * 0.130\n",
    "batch_cost_large = (total_tokens / 1_000_000) * 0.065\n",
    "\n",
    "# Modèle ada v2\n",
    "cost_ada = (total_tokens / 1_000_000) * 0.100\n",
    "batch_cost_ada = (total_tokens / 1_000_000) * 0.050\n",
    "\n",
    "cost_small, batch_cost_small, cost_large, batch_cost_large, cost_ada, batch_cost_ada"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# On prend le fichier EDdA_knowledge_graph_sample.jsonld pour le transformer en json\n",
    "import json\n",
    "\n",
    "# Charger le fichier jsonld\n",
    "with open('data/EDdA_knowledge_graph_sample.jsonld', 'r', encoding='utf-8') as f: \n",
    "    data = json.load(f)\n",
    "\n",
    "# Sauvegarder le fichier en json\n",
    "with open('data/EDdA_knowledge_graph_sample.json', 'w') as f:\n",
    "    json.dump(data, f, indent=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}