{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "051c5726",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import KFold, StratifiedShuffleSplit\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "\n",
    "import re\n",
    "import string\n",
    "from stop_words import get_stop_words\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "import spacy\n",
    "\n",
    "from nltk import ngrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "310c031d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_colwidth\", 255)\n",
    "pd.set_option('display.max_rows', 255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "cc6821d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel(\"training-ner-mapping3.xlsx\")\n",
    "cnt = df.groupby(\"CASISTICA_MOTIVAZIONE\").count() \\\n",
    ".rename(columns={\"Unnamed: 0\": \"count\"})[\"count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4b8cbd26",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_infrequent_classes(input_df: pd.DataFrame):\n",
    "    card_thresh = 10\n",
    "    cnt = input_df[[\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE\"]].groupby([\"CASISTICA_MOTIVAZIONE\"]).count().rename({\"NOTE_OPERATORE\":\"count\"},axis=1)\n",
    "    res = input_df.merge(cnt, on=\"CASISTICA_MOTIVAZIONE\")\n",
    "    res = res[res[\"count\"] >= card_thresh]\n",
    "    res = res.drop(\"count\", axis=1)\n",
    "    return res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3903d9da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unique_str(l:list[str]):\n",
    "    return list(set(l))\n",
    "\n",
    "def apply_ngram(l:list[str], n:int):\n",
    "    return list(ngrams(l, n))\n",
    "\n",
    "def join_string_list(l:list[tuple[str]]):\n",
    "    return [\" \".join(list(i)) for i in l]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "c0d08201",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\3455914512.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_cp['NOTE_OPERATORE'] = df_cp['NOTE_OPERATORE'].apply(str)\n",
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\3455914512.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_cp[\"NOTE_OPERATORE_lower\"] = df_cp[\"NOTE_OPERATORE\"].str.lower()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASISTICA_MOTIVAZIONE</th>\n",
       "      <th>NOTE_OPERATORE_1gram_unique_words</th>\n",
       "      <th>ngram_occurrencies</th>\n",
       "      <th>record_per_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Generica</td>\n",
       "      <td>crs</td>\n",
       "      <td>19</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Generica</td>\n",
       "      <td>fallimento</td>\n",
       "      <td>19</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Generica</td>\n",
       "      <td>libretto</td>\n",
       "      <td>18</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Generica</td>\n",
       "      <td>mandato</td>\n",
       "      <td>16</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Generica</td>\n",
       "      <td>storica</td>\n",
       "      <td>15</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Generica</td>\n",
       "      <td>modalità</td>\n",
       "      <td>13</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ricevuta non valida</td>\n",
       "      <td>prescontrino</td>\n",
       "      <td>11</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Generica</td>\n",
       "      <td>carente</td>\n",
       "      <td>11</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Generica</td>\n",
       "      <td>dato</td>\n",
       "      <td>11</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Generica</td>\n",
       "      <td>immatricolazione</td>\n",
       "      <td>11</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Generica</td>\n",
       "      <td>intestazione</td>\n",
       "      <td>10</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Generica</td>\n",
       "      <td>autocarro</td>\n",
       "      <td>10</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Generica</td>\n",
       "      <td>confisca</td>\n",
       "      <td>10</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Generica</td>\n",
       "      <td>eredi</td>\n",
       "      <td>10</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Generica</td>\n",
       "      <td>sanzioni</td>\n",
       "      <td>10</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Generica</td>\n",
       "      <td>incompleto</td>\n",
       "      <td>9</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Generica</td>\n",
       "      <td>inserire</td>\n",
       "      <td>9</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Generica</td>\n",
       "      <td>perso</td>\n",
       "      <td>9</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Generica</td>\n",
       "      <td>richiede</td>\n",
       "      <td>9</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Generica</td>\n",
       "      <td>storici</td>\n",
       "      <td>9</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Periodo diverso</td>\n",
       "      <td>riferiti</td>\n",
       "      <td>9</td>\n",
       "      <td>794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Generica</td>\n",
       "      <td>verifiche</td>\n",
       "      <td>8</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Generica</td>\n",
       "      <td>versare</td>\n",
       "      <td>8</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Generica</td>\n",
       "      <td>ciclo</td>\n",
       "      <td>8</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Generica</td>\n",
       "      <td>aggiornato</td>\n",
       "      <td>8</td>\n",
       "      <td>2889</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CASISTICA_MOTIVAZIONE NOTE_OPERATORE_1gram_unique_words  \\\n",
       "0               Generica                               crs   \n",
       "1               Generica                        fallimento   \n",
       "2               Generica                          libretto   \n",
       "3               Generica                           mandato   \n",
       "4               Generica                           storica   \n",
       "5               Generica                          modalità   \n",
       "6    Ricevuta non valida                      prescontrino   \n",
       "7               Generica                           carente   \n",
       "8               Generica                              dato   \n",
       "9               Generica                  immatricolazione   \n",
       "10              Generica                      intestazione   \n",
       "11              Generica                         autocarro   \n",
       "12              Generica                          confisca   \n",
       "13              Generica                             eredi   \n",
       "14              Generica                          sanzioni   \n",
       "15              Generica                        incompleto   \n",
       "16              Generica                          inserire   \n",
       "17              Generica                             perso   \n",
       "18              Generica                          richiede   \n",
       "19              Generica                           storici   \n",
       "20       Periodo diverso                          riferiti   \n",
       "21              Generica                         verifiche   \n",
       "22              Generica                           versare   \n",
       "23              Generica                             ciclo   \n",
       "24              Generica                        aggiornato   \n",
       "\n",
       "    ngram_occurrencies  record_per_class  \n",
       "0                   19              2889  \n",
       "1                   19              2889  \n",
       "2                   18              2889  \n",
       "3                   16              2889  \n",
       "4                   15              2889  \n",
       "5                   13              2889  \n",
       "6                   11                69  \n",
       "7                   11              2889  \n",
       "8                   11              2889  \n",
       "9                   11              2889  \n",
       "10                  10              2889  \n",
       "11                  10              2889  \n",
       "12                  10              2889  \n",
       "13                  10              2889  \n",
       "14                  10              2889  \n",
       "15                   9              2889  \n",
       "16                   9              2889  \n",
       "17                   9              2889  \n",
       "18                   9              2889  \n",
       "19                   9              2889  \n",
       "20                   9               794  \n",
       "21                   8              2889  \n",
       "22                   8              2889  \n",
       "23                   8              2889  \n",
       "24                   8              2889  "
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_unique_ngrams(df: pd.DataFrame, n:int):\n",
    "    df_cp = df[[\"CASISTICA_LAVORAZIONE\",\"NOTE_OPERATORE\",\"CASISTICA_MOTIVAZIONE\"]]\n",
    "    df_cp['NOTE_OPERATORE'] = df_cp['NOTE_OPERATORE'].apply(str)\n",
    "\n",
    "    #make all strings lowercase\n",
    "    df_cp[\"NOTE_OPERATORE_lower\"] = df_cp[\"NOTE_OPERATORE\"].str.lower()\n",
    "\n",
    "    #make column as a list of words (1-gram)\n",
    "    splitted_col_name = \"NOTE_OPERATORE_split_{}gram\".format(str(n))\n",
    "    df_cp[\"NOTE_OPERATORE_split_1gram\"] = df_cp[\"NOTE_OPERATORE_lower\"].str.split(\" \")\n",
    "\n",
    "    #if n!=1 make a column as a list of n-grams\n",
    "    if(n!=1):\n",
    "        df_cp[splitted_col_name] = df_cp[\"NOTE_OPERATORE_split_1gram\"].apply(lambda x: apply_ngram(x, n))\n",
    "\n",
    "    ngram_col_name = \"NOTE_OPERATORE_{}gram_unique_words\".format(str(n))\n",
    "\n",
    "    #make sure every n-gram in every row is not repeated\n",
    "    df_cp[ngram_col_name] = df_cp[splitted_col_name].apply(lambda x: unique_str(x))\n",
    "\n",
    "    #concat every word composing a n-gram, transforming the column from a list[tuple[str]] to list[str]\n",
    "    if(n!=1):\n",
    "        df_cp[ngram_col_name] = df_cp[ngram_col_name].apply(lambda x: join_string_list(x))\n",
    "\n",
    "\n",
    "    #explode the df, to make a row for every n-gram - then groupBy [class, n-gram] to find how many records contain every n-gram\n",
    "    df_cp_ngram = df_cp[[ngram_col_name, \"CASISTICA_MOTIVAZIONE\", \"CASISTICA_LAVORAZIONE\"]].explode(ngram_col_name) \\\n",
    "        .groupby([\"CASISTICA_MOTIVAZIONE\",ngram_col_name]).count()\\\n",
    "        .sort_values(by=[\"CASISTICA_MOTIVAZIONE\", \"CASISTICA_LAVORAZIONE\"], ascending=[True,False])\\\n",
    "        .reset_index() \\\n",
    "        .rename(columns={\"CASISTICA_LAVORAZIONE\":\"count\"})\n",
    "\n",
    "\n",
    "    #self-join the df to find every n-gram occurring in just one class\n",
    "    df_cp_ngram_mrg = df_cp_ngram.merge(df_cp_ngram, how=\"left\", on=ngram_col_name)\n",
    "    df_cp_ngram_mrg = df_cp_ngram_mrg.rename(columns={\"count_x\":\"ngram_occurrencies\"})\n",
    "\n",
    "    df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE = df_cp_ngram_mrg.groupby([\"CASISTICA_MOTIVAZIONE_x\", ngram_col_name]).count()\n",
    "    \n",
    "    df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE = df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE.reset_index()\n",
    "    df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE = df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE[[\"CASISTICA_MOTIVAZIONE_x\", ngram_col_name, \"count_y\"]] \\\n",
    "        .rename(columns={\"count_y\":\"ngram_occurrencies_in_CASISTICA_MOTIVAZIONE\", \"CASISTICA_MOTIVAZIONE_x\": \"CASISTICA_MOTIVAZIONE\"})\n",
    "    \n",
    "    # find every n-gram occurring in just one class (it was groupedBy + count by [class,n-gram])\n",
    "    df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE = df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE[df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE[\"ngram_occurrencies_in_CASISTICA_MOTIVAZIONE\"] == 1]\n",
    "\n",
    "    #merge the new DF to produce the final result\n",
    "    res = df_cp_ngram_mrg.merge(df_cp_ngram_mrg_uniqueCASISTICAMOTIVAZIONE, on=ngram_col_name)\n",
    "\n",
    "    res = res[[\"CASISTICA_MOTIVAZIONE_x\", ngram_col_name, \"ngram_occurrencies\"]] #\"ngram_occurrencies_in_CASISTICA_MOTIVAZIONE\"\n",
    "    res = res.rename(columns={\"CASISTICA_MOTIVAZIONE_x\":\"CASISTICA_MOTIVAZIONE\"})\n",
    "\n",
    "    res = res.sort_values(by = \"ngram_occurrencies\", ascending=False)\n",
    "\n",
    "    #groupBy and merge the original DF to create the column \"record_per_class\", indicating how many record occurr in every class\n",
    "    df_grp_class = df.groupby(\"CASISTICA_MOTIVAZIONE\").count().reset_index()\n",
    "    df_grp_class = df_grp_class.rename(columns={\"Unnamed: 0\":\"record_per_class\"})\n",
    "    df_grp_class = df_grp_class[[\"CASISTICA_MOTIVAZIONE\", \"record_per_class\"]]\n",
    "\n",
    "    res = res.merge(df_grp_class, on=\"CASISTICA_MOTIVAZIONE\")\n",
    "\n",
    "    return res\n",
    "\n",
    "\n",
    "gram_dim = 1\n",
    "res= get_unique_ngrams(df, gram_dim)\n",
    "\n",
    "res.iloc[:25]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "c749be4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<>:59: SyntaxWarning: invalid escape sequence '\\s'\n",
      "<>:59: SyntaxWarning: invalid escape sequence '\\s'\n",
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\2433266376.py:59: SyntaxWarning: invalid escape sequence '\\s'\n",
      "  res[\"cat\"] = res[\"cat\"].str.replace(r\"\\s+\", \" \", regex=True) \"\"\"\n"
     ]
    }
   ],
   "source": [
    "def replace_punct(text):\n",
    "    text = ''.join([char for char in text if char not in string.punctuation])\n",
    "    return text\n",
    "\n",
    "def remove_stopwords(text):\n",
    "    ita_stop_words = get_stop_words('italian')\n",
    "    pattern = r'\\b(?:' + '|'.join(re.escape(s) for s in ita_stop_words) + r')\\b'\n",
    "    return re.sub(pattern, '', text)\n",
    "\n",
    "\n",
    "\n",
    "def get_word_count_df(df):\n",
    "    df_cp=df.copy()\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].fillna(\"NULL\")\n",
    "\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"]\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"x000d\", \" \")\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"000d\", \" \")\n",
    "\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].apply(lambda x: remove_stopwords(x))\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].apply(lambda x: replace_punct(x))\n",
    "\n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"/\", \" \", regex=True) \n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(r\"\\s+\", \" \", regex=True) \n",
    "    #df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace({\"à\":\"a\", \"è\":\"e\", \"ì\":\"i\", \"ò\":\"o\", \"ù\":\"u\"}) \n",
    "    df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.lower()\n",
    "\n",
    "\n",
    "    vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "    X = vectorizer.fit_transform(df_cp[\"NOTE_OPERATORE\"])\n",
    "    word_count_df = pd.DataFrame({\"word\":vectorizer.get_feature_names_out(), \"word_count_per_doc\" : X.toarray().transpose().tolist() })\n",
    "\n",
    "    word_count_df[\"word_count\"] = word_count_df[\"word_count_per_doc\"].apply(lambda x: sum(x))\n",
    "    word_count_df[\"word_document_count\"] = word_count_df[\"word_count_per_doc\"].apply(lambda x: sum([min(1,i) for i in x]))\n",
    "\n",
    "    return word_count_df\n",
    "    #word_count_df[[\"word\",\"word_count\", \"word_document_count\"]].sort_values(\"word_count\", ascending=False).iloc[:25]\n",
    "    #word_count_df[word_count_df[\"word_count\"] / word_count_df[\"word_document_count\"] > 1.2][[\"word\",\"word_count\", \"word_document_count\"]]\n",
    "\n",
    "\n",
    "def remove_common_words(df, top_n = 5):\n",
    "    res = df.copy()\n",
    "    word_count_df = get_word_count_df(res)\n",
    "    word_count_df = word_count_df.sort_values(\"word_count\", ascending=False).iloc[:top_n]\n",
    "    remove_words = word_count_df[\"word\"].tolist()\n",
    "    print(remove_words)\n",
    "    pattern = r'\\b(?:' + '|'.join(map(re.escape, remove_words)) + r')\\b'\n",
    "    res['cat'] = res['cat'].apply(lambda x: re.sub(pattern, '', x).strip())\n",
    "    res[\"cat\"] = res[\"cat\"].str.replace(r\"\\s+\", \" \", regex=True) \n",
    "\n",
    "    \"\"\"for i in range(top_n):\n",
    "\n",
    "        remove_words = word_count_df[\"word\"].tolist()\n",
    "        pattern = r'\\b(?:' + '|'.join(map(re.escape, remove_words)) + r')\\b'\n",
    "        res['cat'] = res['cat'].apply(lambda x: re.sub(pattern, '', x).strip())\n",
    "\n",
    "        word = word_count_df[[\"word\",\"word_count\", \"word_document_count\"]].sort_values(\"word_count\", ascending=False).iloc[i][\"word\"]\n",
    "        res[\"cat\"] = res[\"cat\"].str.replace(word, \"\")\n",
    "        res[\"cat\"] = res[\"cat\"].str.replace(r\"\\s+\", \" \", regex=True) \"\"\"\n",
    "\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f05a5897",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data', 'allega', 'bollo', 'versamento', 'esenzione', 'veicolo', 'atto', 'pagamento', 'pra', 'luogo', 'vendita', 'periodo', 'anno', 'documentazione', 'dichiara']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\2368892003.py:50: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  a = unigrams.drop_duplicates([\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_1gram\"]).groupby(['CASISTICA_MOTIVAZIONE']).apply(lambda x: x.nlargest(10,['metric'])).reset_index(drop=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data', 'allega', 'bollo', 'versamento', 'esenzione', 'veicolo', 'atto', 'pagamento', 'pra', 'luogo', 'vendita', 'periodo', 'anno', 'documentazione', 'dichiara']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\2368892003.py:55: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  b = bigrams.drop_duplicates([\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_2gram\"]).groupby(['CASISTICA_MOTIVAZIONE']).apply(lambda x: x.nlargest(10,['metric'])).reset_index(drop=True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASISTICA_MOTIVAZIONE</th>\n",
       "      <th>n-gram</th>\n",
       "      <th>count_placeholder</th>\n",
       "      <th>sum_selfjoin</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(versamenti, allegato)</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.335375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(allegato,)</td>\n",
       "      <td>3</td>\n",
       "      <td>237</td>\n",
       "      <td>-5.927289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(dettaglio,)</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2.397895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(viene,)</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.704748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(annullamento,)</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1.193922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(annullato,)</td>\n",
       "      <td>6</td>\n",
       "      <td>27</td>\n",
       "      <td>0.985818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(effettuato,)</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>-0.528844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(versamenti,)</td>\n",
       "      <td>2</td>\n",
       "      <td>70</td>\n",
       "      <td>-1.532345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(risulta,)</td>\n",
       "      <td>4</td>\n",
       "      <td>214</td>\n",
       "      <td>-7.496522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(risulta, annullato)</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>8.222520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(versamenti, risulta)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.740840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>Versamento annullato</td>\n",
       "      <td>(allegato, risulta)</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>2.473525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(avvenuta,)</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>6.520907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(dovuto, venduto)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>10.963360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(dopo, scadenza)</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>14.900002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(vende,)</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6.202390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(giorno,)</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>6.520907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(autenticato,)</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>6.520907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(decorrenza, richiesta)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>10.963360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(lombardo,)</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>7.795698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(dopo,)</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "      <td>7.879765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(registrata,)</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>8.440853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(venduta,)</td>\n",
       "      <td>24</td>\n",
       "      <td>20</td>\n",
       "      <td>10.847643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(formato,)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>11.213442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(decorrenza,)</td>\n",
       "      <td>17</td>\n",
       "      <td>16</td>\n",
       "      <td>11.276002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(tributario, già)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>10.963360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(soggetto, lombardo)</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>14.012249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(venduta, mese)</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>14.012249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(decorrenza, scadenza)</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>18.245493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(venduta, dopo)</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>16.347624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(mese, dovuto)</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>16.420944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>Vendita tardiva</td>\n",
       "      <td>(venduto, mese)</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>18.682999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(versamenti, altra)</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4.670750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(targa, altra)</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>13.704200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(diversa,)</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>3.295837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(rimborso,)</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>2.387845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(tg,)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.110213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(motoveicolo,)</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1.704748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(effettuati,)</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1.550597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(moto,)</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1.550597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(vettura,)</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1.417066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(targa, diversa)</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>6.143079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(targa, file)</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4.670750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(targa, riferito)</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5.481680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(bic,)</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5.606721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(può, richiedere)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.740840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(controlli, effettuati)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.740840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(riferito, altra)</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>3.108276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(altra, targa)</td>\n",
       "      <td>18</td>\n",
       "      <td>24</td>\n",
       "      <td>3.872005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(riferita, altra)</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4.095386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(proprieta,)</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4.795791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>Veicolo diverso</td>\n",
       "      <td>(correzione,)</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1.887070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>Variazioni tecniche</td>\n",
       "      <td>(uso,)</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0.552069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>Variazioni tecniche</td>\n",
       "      <td>(cambio,)</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>0.062520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>Variazioni tecniche</td>\n",
       "      <td>(pv,)</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.803360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    CASISTICA_MOTIVAZIONE                   n-gram  count_placeholder  \\\n",
       "248  Versamento annullato   (versamenti, allegato)                  1   \n",
       "292  Versamento annullato              (allegato,)                  3   \n",
       "286  Versamento annullato             (dettaglio,)                  1   \n",
       "287  Versamento annullato                 (viene,)                  1   \n",
       "288  Versamento annullato          (annullamento,)                  1   \n",
       "289  Versamento annullato             (annullato,)                  6   \n",
       "290  Versamento annullato            (effettuato,)                  1   \n",
       "291  Versamento annullato            (versamenti,)                  2   \n",
       "293  Versamento annullato               (risulta,)                  4   \n",
       "245  Versamento annullato     (risulta, annullato)                  3   \n",
       "246  Versamento annullato    (versamenti, risulta)                  1   \n",
       "247  Versamento annullato      (allegato, risulta)                  2   \n",
       "283       Vendita tardiva              (avvenuta,)                  7   \n",
       "244       Vendita tardiva        (dovuto, venduto)                  4   \n",
       "239       Vendita tardiva         (dopo, scadenza)                 11   \n",
       "285       Vendita tardiva                 (vende,)                  4   \n",
       "284       Vendita tardiva                (giorno,)                  7   \n",
       "282       Vendita tardiva           (autenticato,)                  7   \n",
       "242       Vendita tardiva  (decorrenza, richiesta)                  4   \n",
       "281       Vendita tardiva              (lombardo,)                  6   \n",
       "280       Vendita tardiva                  (dopo,)                 13   \n",
       "279       Vendita tardiva            (registrata,)                  4   \n",
       "278       Vendita tardiva               (venduta,)                 24   \n",
       "277       Vendita tardiva               (formato,)                  4   \n",
       "276       Vendita tardiva            (decorrenza,)                 17   \n",
       "243       Vendita tardiva        (tributario, già)                  4   \n",
       "240       Vendita tardiva     (soggetto, lombardo)                  6   \n",
       "241       Vendita tardiva          (venduta, mese)                  6   \n",
       "236       Vendita tardiva   (decorrenza, scadenza)                 10   \n",
       "238       Vendita tardiva          (venduta, dopo)                  7   \n",
       "237       Vendita tardiva           (mese, dovuto)                  9   \n",
       "235       Vendita tardiva          (venduto, mese)                  8   \n",
       "228       Veicolo diverso      (versamenti, altra)                  2   \n",
       "225       Veicolo diverso           (targa, altra)                  5   \n",
       "268       Veicolo diverso               (diversa,)                  3   \n",
       "269       Veicolo diverso              (rimborso,)                  2   \n",
       "270       Veicolo diverso                    (tg,)                  1   \n",
       "272       Veicolo diverso           (motoveicolo,)                  1   \n",
       "273       Veicolo diverso            (effettuati,)                  1   \n",
       "274       Veicolo diverso                  (moto,)                  1   \n",
       "275       Veicolo diverso               (vettura,)                  1   \n",
       "226       Veicolo diverso         (targa, diversa)                  3   \n",
       "229       Veicolo diverso            (targa, file)                  2   \n",
       "227       Veicolo diverso        (targa, riferito)                  2   \n",
       "266       Veicolo diverso                   (bic,)                  2   \n",
       "234       Veicolo diverso        (può, richiedere)                  1   \n",
       "233       Veicolo diverso  (controlli, effettuati)                  1   \n",
       "232       Veicolo diverso        (riferito, altra)                  3   \n",
       "231       Veicolo diverso           (altra, targa)                 18   \n",
       "230       Veicolo diverso        (riferita, altra)                  2   \n",
       "267       Veicolo diverso             (proprieta,)                  2   \n",
       "271       Veicolo diverso            (correzione,)                  1   \n",
       "264   Variazioni tecniche                   (uso,)                  1   \n",
       "265   Variazioni tecniche                (cambio,)                  1   \n",
       "262   Variazioni tecniche                    (pv,)                  1   \n",
       "\n",
       "     sum_selfjoin     metric  \n",
       "248             2   2.335375  \n",
       "292           237  -5.927289  \n",
       "286             2   2.397895  \n",
       "287             5   1.704748  \n",
       "288             9   1.193922  \n",
       "289            27   0.985818  \n",
       "290            55  -0.528844  \n",
       "291            70  -1.532345  \n",
       "293           214  -7.496522  \n",
       "245             1   8.222520  \n",
       "246             1   2.740840  \n",
       "247             8   2.473525  \n",
       "283            12   6.520907  \n",
       "244             1  10.963360  \n",
       "239             7  14.900002  \n",
       "285             6   6.202390  \n",
       "284            12   6.520907  \n",
       "282            12   6.520907  \n",
       "242             1  10.963360  \n",
       "281             8   7.795698  \n",
       "280            17   7.879765  \n",
       "279             3   8.440853  \n",
       "278            20  10.847643  \n",
       "277             1  11.213442  \n",
       "276            16  11.276002  \n",
       "243             1  10.963360  \n",
       "240             2  14.012249  \n",
       "241             2  14.012249  \n",
       "236             4  18.245493  \n",
       "238             2  16.347624  \n",
       "237             4  16.420944  \n",
       "235             2  18.682999  \n",
       "228             2   4.670750  \n",
       "225             1  13.704200  \n",
       "268            10   3.295837  \n",
       "269             9   2.387845  \n",
       "270             3   2.110213  \n",
       "272             5   1.704748  \n",
       "273             6   1.550597  \n",
       "274             6   1.550597  \n",
       "275             7   1.417066  \n",
       "226             3   6.143079  \n",
       "229             2   4.670750  \n",
       "227             1   5.481680  \n",
       "266             1   5.606721  \n",
       "234             1   2.740840  \n",
       "233             1   2.740840  \n",
       "232            10   3.108276  \n",
       "231            24   3.872005  \n",
       "230             3   4.095386  \n",
       "267             2   4.795791  \n",
       "271             4   1.887070  \n",
       "264            18   0.552069  \n",
       "265            30   0.062520  \n",
       "262             1   2.803360  "
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_most_relevant_ngrams(df, n):\n",
    "    splitted_col_name = \"NOTE_OPERATORE_split_{}gram\".format(str(n))\n",
    "    df_cp=df.copy()\n",
    "\n",
    "    #concatenate every string within the same class\n",
    "    df_cp = df_cp.sort_values(by=\"CASISTICA_MOTIVAZIONE\")\n",
    "    df_cp['NOTE_OPERATORE'] = df_cp['NOTE_OPERATORE'].apply(str)\n",
    "    df_cp[\"cat\"] = df_cp.groupby(['CASISTICA_MOTIVAZIONE'])['NOTE_OPERATORE'].transform(lambda x : ' '.join(x))\n",
    "    df_cp[\"cat\"] = df_cp[\"cat\"].str.lower()\n",
    "\n",
    "    df_cp = remove_common_words(df_cp, top_n=15)\n",
    "\n",
    "    df_cp = df_cp.groupby([\"CASISTICA_MOTIVAZIONE\", \"cat\"]).count().reset_index()\n",
    "\n",
    "    #make column as a list of words (1-gram), then create a column of n-grams\n",
    "    df_cp[splitted_col_name] = df_cp[\"cat\"].str.split(\" \").apply(lambda x: apply_ngram(x, n))\n",
    "\n",
    "    #count the occurrences for every n-gram for every class\n",
    "    df_cp[\"count_placeholder\"] = 1\n",
    "    df_cp = df_cp[[\"CASISTICA_MOTIVAZIONE\", splitted_col_name, \"count_placeholder\"]].explode(splitted_col_name)\\\n",
    "        .groupby([\"CASISTICA_MOTIVAZIONE\", splitted_col_name]).count() \\\n",
    "        .reset_index() \\\n",
    "        .sort_values(by=\"count_placeholder\")\n",
    "\n",
    "    #self-join to find which classes (except the currently considerated one) have a given n-gram, for every n-gram and every class\n",
    "    df_cp_selfjoin = df_cp[[\"CASISTICA_MOTIVAZIONE\", splitted_col_name, \"count_placeholder\"]].rename(columns={\"CASISTICA_MOTIVAZIONE\": \"CASISTICA_MOTIVAZIONE_selfjoin\", \"count_placeholder\":\"count_selfjoin\"})\n",
    "    df_cp = df_cp.merge(df_cp_selfjoin, on=splitted_col_name)\n",
    "    df_cp = df_cp[df_cp[\"CASISTICA_MOTIVAZIONE\"] != df_cp[\"CASISTICA_MOTIVAZIONE_selfjoin\"]]\n",
    "\n",
    "    # find how many occurrences of a given n-gram exist in every other class (except the currently considerated one), for every n-gram and every class\n",
    "    df_cp_tfidf = df_cp[[\"CASISTICA_MOTIVAZIONE\", splitted_col_name, \"count_selfjoin\"]].groupby([\"CASISTICA_MOTIVAZIONE\", splitted_col_name]).sum(\"count_selfjoin\") \\\n",
    "        .rename(columns={\"count_selfjoin\": \"sum_selfjoin\"})\n",
    "    df_cp = df_cp.merge(df_cp_tfidf, on=[\"CASISTICA_MOTIVAZIONE\", splitted_col_name])\n",
    "\n",
    "    #for every n-gram and every class, divide the occurrences of that n-gram in that class by the sum of the occurrences of that n-gram in every other class\n",
    "    df_cp[\"tf_idf\"] = df_cp[\"count_placeholder\"] / df_cp[\"sum_selfjoin\"]\n",
    "\n",
    "    df_cp = df_cp.sort_values(by=\"tf_idf\")\n",
    "\n",
    "    num_classes = df_cp[\"CASISTICA_MOTIVAZIONE\"].drop_duplicates().count().item()\n",
    "    df_cp[\"metric\"] =  df_cp[\"count_placeholder\"] * np.log( num_classes / (1 + df_cp[\"sum_selfjoin\"] ))\n",
    "    return df_cp\n",
    "\n",
    "\n",
    "\n",
    "n=1\n",
    "df_cp = df.copy()\n",
    "unigrams = get_most_relevant_ngrams(df_cp, 1)\n",
    "\n",
    "a = unigrams.drop_duplicates([\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_1gram\"]).groupby(['CASISTICA_MOTIVAZIONE']).apply(lambda x: x.nlargest(10,['metric'])).reset_index(drop=True)\n",
    "a = a[[\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_1gram\", \"count_placeholder\", \"sum_selfjoin\", \"metric\"]]\n",
    "a.rename(columns={\"NOTE_OPERATORE_split_1gram\": \"n-gram\"}, inplace=True)\n",
    "bigrams = get_most_relevant_ngrams(df_cp, 2)\n",
    "\n",
    "b = bigrams.drop_duplicates([\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_2gram\"]).groupby(['CASISTICA_MOTIVAZIONE']).apply(lambda x: x.nlargest(10,['metric'])).reset_index(drop=True)\n",
    "b = b[[\"CASISTICA_MOTIVAZIONE\",\"NOTE_OPERATORE_split_2gram\", \"count_placeholder\", \"sum_selfjoin\", \"metric\"]]\n",
    "b.rename(columns={\"NOTE_OPERATORE_split_2gram\": \"n-gram\"}, inplace=True)\n",
    "\n",
    "res = pd.concat([a,b])\n",
    "res.sort_values(by=\"CASISTICA_MOTIVAZIONE\", ascending=False).iloc[:55]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "2e89a89e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\a.torchi\\AppData\\Local\\Temp\\ipykernel_21580\\173629697.py:3: FutureWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  res = res.groupby([\"CASISTICA_MOTIVAZIONE\"]).apply(lambda x: x.sort_values(by=\"metric\")).drop(\"CASISTICA_MOTIVAZIONE\", axis=1)\n"
     ]
    }
   ],
   "source": [
    "res = pd.concat([a,b])\n",
    "res.sort_values(by=\"CASISTICA_MOTIVAZIONE\", ascending=False).iloc[:55]\n",
    "res = res.groupby([\"CASISTICA_MOTIVAZIONE\"]).apply(lambda x: x.sort_values(by=\"metric\")).drop(\"CASISTICA_MOTIVAZIONE\", axis=1)\n",
    "res.to_excel(\"most_relevane_ngrams.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "f526078a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASISTICA_MOTIVAZIONE</th>\n",
       "      <th>n-gram</th>\n",
       "      <th>count_placeholder</th>\n",
       "      <th>sum_selfjoin</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(carta, circolazione)</td>\n",
       "      <td>85</td>\n",
       "      <td>7</td>\n",
       "      <td>115.136381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(iscrizione, asi)</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>52.075960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(codice, fiscale)</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>49.335120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(presa, carico)</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>47.096935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(x000d, x000d)</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>41.112600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(essere, mai)</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>41.112600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(n, licenza)</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>35.030624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(stato, proprietario)</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>32.890080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(documento, identità)</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>32.890080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>Generica</td>\n",
       "      <td>(cliente, aver)</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>32.763085</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    CASISTICA_MOTIVAZIONE                 n-gram  count_placeholder  \\\n",
       "119              Generica  (carta, circolazione)                 85   \n",
       "120              Generica      (iscrizione, asi)                 19   \n",
       "121              Generica      (codice, fiscale)                 18   \n",
       "122              Generica        (presa, carico)                 23   \n",
       "123              Generica         (x000d, x000d)                 15   \n",
       "124              Generica          (essere, mai)                 15   \n",
       "125              Generica           (n, licenza)                 15   \n",
       "126              Generica  (stato, proprietario)                 12   \n",
       "127              Generica  (documento, identità)                 12   \n",
       "128              Generica        (cliente, aver)                 16   \n",
       "\n",
       "     sum_selfjoin      metric  \n",
       "119             7  115.136381  \n",
       "120             1   52.075960  \n",
       "121             1   49.335120  \n",
       "122             3   47.096935  \n",
       "123             1   41.112600  \n",
       "124             1   41.112600  \n",
       "125             2   35.030624  \n",
       "126             1   32.890080  \n",
       "127             1   32.890080  \n",
       "128             3   32.763085  "
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[b[\"CASISTICA_MOTIVAZIONE\"]==\"Generica\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a788c9fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cp=df.copy()\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].fillna(\"NULL\")\n",
    "\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"]\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"x000d\", \" \")\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"000d\", \" \")\n",
    "\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].apply(lambda x: remove_stopwords(x))\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].apply(lambda x: replace_punct(x))\n",
    "\n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(\"/\", \" \", regex=True) \n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace(r\"\\s+\", \" \", regex=True) \n",
    "#df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.replace({\"à\":\"a\", \"è\":\"e\", \"ì\":\"i\", \"ò\":\"o\", \"ù\":\"u\"}) \n",
    "df_cp[\"NOTE_OPERATORE\"] = df_cp[\"NOTE_OPERATORE\"].str.lower()\n",
    "\n",
    "\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 1))\n",
    "X = vectorizer.fit_transform(df_cp[\"NOTE_OPERATORE\"])\n",
    "word_count_df = pd.DataFrame({\"word\":vectorizer.get_feature_names_out(), \"word_count_per_doc\" : X.toarray().transpose().tolist() })\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "af067ccd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>word_count</th>\n",
       "      <th>word_document_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>data</td>\n",
       "      <td>2421</td>\n",
       "      <td>1638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>allega</td>\n",
       "      <td>792</td>\n",
       "      <td>781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>bollo</td>\n",
       "      <td>657</td>\n",
       "      <td>621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2037</th>\n",
       "      <td>versamento</td>\n",
       "      <td>515</td>\n",
       "      <td>507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>esenzione</td>\n",
       "      <td>503</td>\n",
       "      <td>478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013</th>\n",
       "      <td>veicolo</td>\n",
       "      <td>500</td>\n",
       "      <td>449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>atto</td>\n",
       "      <td>394</td>\n",
       "      <td>377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1206</th>\n",
       "      <td>pagamento</td>\n",
       "      <td>365</td>\n",
       "      <td>346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>pra</td>\n",
       "      <td>346</td>\n",
       "      <td>339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1038</th>\n",
       "      <td>luogo</td>\n",
       "      <td>343</td>\n",
       "      <td>297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017</th>\n",
       "      <td>vendita</td>\n",
       "      <td>325</td>\n",
       "      <td>317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1256</th>\n",
       "      <td>periodo</td>\n",
       "      <td>306</td>\n",
       "      <td>303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>anno</td>\n",
       "      <td>299</td>\n",
       "      <td>288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>documentazione</td>\n",
       "      <td>261</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>dichiara</td>\n",
       "      <td>261</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050</th>\n",
       "      <td>manca</td>\n",
       "      <td>260</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1540</th>\n",
       "      <td>ricevuta</td>\n",
       "      <td>258</td>\n",
       "      <td>252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1855</th>\n",
       "      <td>targa</td>\n",
       "      <td>257</td>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>allegato</td>\n",
       "      <td>240</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1645</th>\n",
       "      <td>risulta</td>\n",
       "      <td>218</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>altra</td>\n",
       "      <td>212</td>\n",
       "      <td>212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>cliente</td>\n",
       "      <td>211</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>632</th>\n",
       "      <td>dovuto</td>\n",
       "      <td>204</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474</th>\n",
       "      <td>regione</td>\n",
       "      <td>197</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1212</th>\n",
       "      <td>pagato</td>\n",
       "      <td>186</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                word  word_count  word_document_count\n",
       "507             data        2421                 1638\n",
       "64            allega         792                  781\n",
       "256            bollo         657                  621\n",
       "2037      versamento         515                  507\n",
       "697        esenzione         503                  478\n",
       "2013         veicolo         500                  449\n",
       "187             atto         394                  377\n",
       "1206       pagamento         365                  346\n",
       "1315             pra         346                  339\n",
       "1038           luogo         343                  297\n",
       "2017         vendita         325                  317\n",
       "1256         periodo         306                  303\n",
       "98              anno         299                  288\n",
       "611   documentazione         261                  256\n",
       "557         dichiara         261                  260\n",
       "1050           manca         260                  260\n",
       "1540        ricevuta         258                  252\n",
       "1855           targa         257                  191\n",
       "72          allegato         240                  239\n",
       "1645         risulta         218                  213\n",
       "79             altra         212                  212\n",
       "350          cliente         211                  189\n",
       "632           dovuto         204                  203\n",
       "1474         regione         197                  196\n",
       "1212          pagato         186                  183"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_count_df[\"word_count\"] = word_count_df[\"word_count_per_doc\"].apply(lambda x: sum(x))\n",
    "word_count_df[\"word_document_count\"] = word_count_df[\"word_count_per_doc\"].apply(lambda x: sum([min(1,i) for i in x]))\n",
    "\n",
    "word_count_df[[\"word\",\"word_count\", \"word_document_count\"]].sort_values(\"word_count\", ascending=False).iloc[:25]\n",
    "#word_count_df[word_count_df[\"word_count\"] / word_count_df[\"word_document_count\"] > 1.2][[\"word\",\"word_count\", \"word_document_count\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f9a71c24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_per_class</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CASISTICA_MOTIVAZIONE</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Periodo diverso</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Generica</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Domiciliazione tardiva</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Demolizione</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Domiciliazione assente</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vendita tardiva</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Acquisto nel mese di rinnovo; concessionario</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Doc. sanitaria scaduta</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Acquisto nel mese di rinnovo; privato</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esenzione PH eredi</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esenzione PH senza istanza</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esenzione PH successiva</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esenzione per ultratrentennale</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esenzione successiva</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CRS intestata a soggetto diverso</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ecoincentivo</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eredi</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Furto non annotato</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fermo</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Esportazione non annotata</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No esenzione PH</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No minivoltura</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Perdita possesso con DS successiva</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rateizzazione</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Generica 2_non allegata documentazione</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Regione diversa</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ricevuta non valida</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rientro in possesso</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rientro ecoincentivo</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sequestro non annotato</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Variazioni tecniche</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Veicolo diverso</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Versamento annullato</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              data_per_class\n",
       "CASISTICA_MOTIVAZIONE                                       \n",
       "Periodo diverso                                           46\n",
       "Generica                                                  12\n",
       "Domiciliazione tardiva                                     3\n",
       "Demolizione                                                2\n",
       "Domiciliazione assente                                     1\n",
       "Vendita tardiva                                            1\n",
       "Acquisto nel mese di rinnovo; concessionario               0\n",
       "Doc. sanitaria scaduta                                     0\n",
       "Acquisto nel mese di rinnovo; privato                      0\n",
       "Esenzione PH eredi                                         0\n",
       "Esenzione PH senza istanza                                 0\n",
       "Esenzione PH successiva                                    0\n",
       "Esenzione per ultratrentennale                             0\n",
       "Esenzione successiva                                       0\n",
       "CRS intestata a soggetto diverso                           0\n",
       "Ecoincentivo                                               0\n",
       "Eredi                                                      0\n",
       "Furto non annotato                                         0\n",
       "Fermo                                                      0\n",
       "Esportazione non annotata                                  0\n",
       "No esenzione PH                                            0\n",
       "No minivoltura                                             0\n",
       "Perdita possesso con DS successiva                         0\n",
       "Rateizzazione                                              0\n",
       "Generica 2_non allegata documentazione                     0\n",
       "Regione diversa                                            0\n",
       "Ricevuta non valida                                        0\n",
       "Rientro in possesso                                        0\n",
       "Rientro ecoincentivo                                       0\n",
       "Sequestro non annotato                                     0\n",
       "Variazioni tecniche                                        0\n",
       "Veicolo diverso                                            0\n",
       "Versamento annullato                                       0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cp[\"data_per_class\"] = df_cp.groupby(['CASISTICA_MOTIVAZIONE'])['NOTE_OPERATORE'].transform(lambda x : x.str.count(\"vers \"))\n",
    "df_cp.groupby([\"CASISTICA_MOTIVAZIONE\"]).sum(\"data_per_class\")[[\"data_per_class\"]].sort_values(by=\"data_per_class\",ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "9f1e9db0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Generica',\n",
       " 'Furto non annotato',\n",
       " 'Periodo diverso',\n",
       " 'Veicolo diverso',\n",
       " 'Fermo',\n",
       " 'Ricevuta non valida',\n",
       " 'Vendita tardiva',\n",
       " 'Esportazione non annotata',\n",
       " 'Acquisto nel mese di rinnovo; privato',\n",
       " 'Sequestro non annotato',\n",
       " 'Esenzione PH successiva',\n",
       " 'Ecoincentivo',\n",
       " 'Regione diversa',\n",
       " 'Esenzione PH senza istanza',\n",
       " 'Versamento annullato',\n",
       " 'No esenzione PH',\n",
       " 'Generica 2_non allegata documentazione',\n",
       " 'Esenzione per ultratrentennale',\n",
       " 'Acquisto nel mese di rinnovo; concessionario',\n",
       " 'Domiciliazione tardiva',\n",
       " 'Demolizione',\n",
       " 'Domiciliazione assente',\n",
       " 'Esenzione PH eredi',\n",
       " 'No minivoltura',\n",
       " 'Perdita possesso con DS successiva',\n",
       " 'Rientro in possesso',\n",
       " 'Rateizzazione',\n",
       " 'CRS intestata a soggetto diverso',\n",
       " 'Rientro ecoincentivo',\n",
       " 'Esenzione successiva',\n",
       " 'Eredi',\n",
       " 'Doc. sanitaria scaduta',\n",
       " 'Variazioni tecniche']"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['CASISTICA_MOTIVAZIONE'].drop_duplicates().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3c42240",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data', 'allega', 'bollo', 'versamento', 'esenzione', 'veicolo', 'atto', 'pagamento', 'pra', 'luogo', 'vendita', 'periodo', 'anno', 'documentazione', 'dichiara']\n",
      "Count: 2889\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 330160 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_cp = df.copy()\n",
    "\n",
    "\n",
    "#concatenate every string within the same class\n",
    "df_cp = df_cp.sort_values(by=\"CASISTICA_MOTIVAZIONE\")\n",
    "df_cp['NOTE_OPERATORE'] = df_cp['NOTE_OPERATORE'].apply(str)\n",
    "df_cp[\"cat\"] = df_cp.groupby(['CASISTICA_MOTIVAZIONE'])['NOTE_OPERATORE'].transform(lambda x : ' '.join(x))\n",
    "df_cp[\"cat\"] = df_cp[\"cat\"].str.lower()\n",
    "\n",
    "\n",
    "df_cp = remove_common_words(df_cp, top_n=15)\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "class_name = \"Generica\"\n",
    "df_cp = df_cp[df_cp[\"CASISTICA_MOTIVAZIONE\"] == class_name]\n",
    "print(\"Count: {}\".format(df_cp.shape[0]))\n",
    "wordcloud2 = WordCloud().generate(' '.join(df_cp['cat'].drop_duplicates().str.lower()))\n",
    "\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud2, interpolation='bilinear')\n",
    "plt.axis('off')  \n",
    "plt.title(class_name)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "2f76df87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['data', 'allega', 'bollo', 'versamento', 'esenzione', 'veicolo', 'atto', 'pagamento', 'pra', 'luogo', 'vendita', 'periodo', 'anno', 'documentazione', 'dichiara']\n",
      "Count: 7\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 178720 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_cp = df.copy()\n",
    "\n",
    "\n",
    "#concatenate every string within the same class\n",
    "df_cp = df_cp.sort_values(by=\"CASISTICA_MOTIVAZIONE\")\n",
    "df_cp['NOTE_OPERATORE'] = df_cp['NOTE_OPERATORE'].apply(str)\n",
    "df_cp[\"cat\"] = df_cp.groupby(['CASISTICA_MOTIVAZIONE'])['NOTE_OPERATORE'].transform(lambda x : ' '.join(x))\n",
    "df_cp[\"cat\"] = df_cp[\"cat\"].str.lower()\n",
    "\n",
    "\n",
    "df_cp = remove_common_words(df_cp, top_n=15)\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "class_name = \"Ecoincentivo\"\n",
    "df_cp = df_cp[df_cp[\"CASISTICA_MOTIVAZIONE\"] == class_name]\n",
    "print(\"Count: {}\".format(df_cp.shape[0]))\n",
    "wordcloud2 = WordCloud().generate(' '.join(df_cp['cat'].drop_duplicates().str.lower()))\n",
    "\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(wordcloud2, interpolation='bilinear')\n",
    "plt.axis('off')  \n",
    "plt.title(class_name)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
