{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-08-23T03:39:42.424686Z","iopub.execute_input":"2022-08-23T03:39:42.425293Z","iopub.status.idle":"2022-08-23T03:39:43.865655Z","shell.execute_reply.started":"2022-08-23T03:39:42.425257Z","shell.execute_reply":"2022-08-23T03:39:43.864352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --quiet deepspeed","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom datasets import load_dataset, Dataset, load_metric\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSequenceClassification,\n    DataCollatorWithPadding,\n    TrainingArguments,\n    Trainer,\n)\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"","metadata":{"execution":{"iopub.status.busy":"2022-08-23T03:48:29.249391Z","iopub.execute_input":"2022-08-23T03:48:29.249742Z","iopub.status.idle":"2022-08-23T03:48:29.257893Z","shell.execute_reply.started":"2022-08-23T03:48:29.249710Z","shell.execute_reply":"2022-08-23T03:48:29.256844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import subprocess\nfrom kaggle_secrets import UserSecretsClient\n\nsecret_value = UserSecretsClient().get_secret(\"wandb-api-key\")\nassert subprocess.check_call([\"wandb\", \"login\", str(secret_value)]) == 0","metadata":{"execution":{"iopub.status.busy":"2022-08-23T03:39:43.869694Z","iopub.execute_input":"2022-08-23T03:39:43.870496Z","iopub.status.idle":"2022-08-23T03:39:46.144449Z","shell.execute_reply.started":"2022-08-23T03:39:43.870446Z","shell.execute_reply":"2022-08-23T03:39:46.143182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom datasets import load_dataset, Dataset, load_metric\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSequenceClassification,\n    DataCollatorWithPadding,\n    TrainingArguments,\n    Trainer,\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T17:08:07.289671Z","iopub.execute_input":"2022-08-23T17:08:07.290427Z","iopub.status.idle":"2022-08-23T17:08:18.655267Z","shell.execute_reply.started":"2022-08-23T17:08:07.290322Z","shell.execute_reply":"2022-08-23T17:08:18.653517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL = \"roberta-base\"\nconfig = {\n    \"weight_decay\": 0.01,\n    \"num_train_epochs\": 1.0,\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)\nmodel = AutoModelForSequenceClassification.from_pretrained(MODEL, num_labels=6) # 0-5, inclusive","metadata":{"execution":{"iopub.status.busy":"2022-08-23T03:40:01.273044Z","iopub.execute_input":"2022-08-23T03:40:01.273513Z","iopub.status.idle":"2022-08-23T03:40:31.351868Z","shell.execute_reply.started":"2022-08-23T03:40:01.273472Z","shell.execute_reply":"2022-08-23T03:40:31.350752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_function(examples):\n    return tokenizer(examples[\"text\"], truncation=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T03:40:31.353913Z","iopub.execute_input":"2022-08-23T03:40:31.354351Z","iopub.status.idle":"2022-08-23T03:40:31.359848Z","shell.execute_reply.started":"2022-08-23T03:40:31.354312Z","shell.execute_reply":"2022-08-23T03:40:31.358554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"/kaggle/input/goodreads-books-reviews-290312\"\n\ndf = pd.read_csv(\n    f\"{data_dir}/goodreads_train.csv\",\n    usecols=[\"rating\", \"review_text\"],\n    # chunksize=1000,\n)\n# df = next(iter(df))\n\nm = np.random.random(len(df)) < 0.95\ndf_trn = df.iloc[m, :]\ndf_tst = df.iloc[~m, :]\n\noversample, undersample = False, False\n\n##############\n## RESAMPLE ##\n\nif oversample and undersample:\n    raise Exception\n    \ndfs = []\nfor _, df_ in df_trn.groupby(\"rating\"):\n    dfs.append(df_)\n    \nif oversample:\n    n_post_populated_class = max(len(df) for df in dfs)\n    df_trn = pd.concat([df_.sample(n=n_post_populated_class, replace=True)\n              for df_ in dfs])\n    # If we don't resample, the dataset is too big and doesn't finish training\n    # before Kaggle boots us off -- restore original size\n    df_trn = df_trn.sample(n=len(df))\n    \nif undersample:\n    n_least_populated_class = min(len(df) for df in dfs)\n    df_trn = pd.concat([df_.sample(n=n_least_populated_class)\n              for df_ in dfs])\n    \n## RESAMPLE ##\n##############\n\ndataset = {}\nfor name, ddf in [\n    (\"train\", df_trn),\n    (\"test\", df_tst),\n]:\n    dataset[name] = (\n        Dataset.from_pandas(ddf)\n        .rename_columns({\"rating\": \"label\", \"review_text\": \"text\"})\n        .shuffle()\n        .map(preprocess_function, batched=True)\n    )","metadata":{"execution":{"iopub.status.busy":"2022-08-23T17:39:27.473154Z","iopub.execute_input":"2022-08-23T17:39:27.473997Z","iopub.status.idle":"2022-08-23T17:39:43.338212Z","shell.execute_reply.started":"2022-08-23T17:39:27.473943Z","shell.execute_reply":"2022-08-23T17:39:43.336738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = list(range(6))\ny = df_trn.groupby(\"rating\").size().to_list()\navg_rating = df.rating.mean()\n\nfig, ax = plt.subplots(figsize=(5,5))\nax.bar(x, y)\nax.plot([avg_rating, avg_rating], [0, max(y)], color=\"r\", linewidth=4, linestyle='dashed')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T17:39:58.581331Z","iopub.execute_input":"2022-08-23T17:39:58.582177Z","iopub.status.idle":"2022-08-23T17:39:58.809489Z","shell.execute_reply.started":"2022-08-23T17:39:58.582132Z","shell.execute_reply":"2022-08-23T17:39:58.808604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DataCollatorWithPadding(tokenizer=tokenizer)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:05:06.695186Z","iopub.execute_input":"2022-08-22T18:05:06.695548Z","iopub.status.idle":"2022-08-22T18:05:06.703288Z","shell.execute_reply.started":"2022-08-22T18:05:06.695514Z","shell.execute_reply":"2022-08-22T18:05:06.702188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = load_metric(\"accuracy\")\nf1 = load_metric(\"f1\")","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:05:06.704808Z","iopub.execute_input":"2022-08-22T18:05:06.705156Z","iopub.status.idle":"2022-08-22T18:05:10.430212Z","shell.execute_reply.started":"2022-08-22T18:05:06.705124Z","shell.execute_reply":"2022-08-22T18:05:10.428984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\n\ndef compute_metrics(eval_pred):\n    logits, labels = eval_pred\n    predictions = np.argmax(logits, axis=-1)\n    return {\n        **accuracy.compute(predictions=predictions, references=labels),\n        **f1.compute(predictions=predictions, references=labels, average=\"macro\"),\n    }\n\nrun_name = f\"hg-{int(time.time())}\"\n\nprint(\"=\" * 50)\nprint(f\"Starting run: {run_name}\")\nprint(\"=\" * 50)\n\ntraining_args = TrainingArguments(\n    run_name=run_name,\n    output_dir=\"./results\",\n    save_total_limit=1,\n    evaluation_strategy=\"steps\",\n    eval_steps=10_000,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=16,\n    **config,\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=dataset[\"train\"],\n    eval_dataset=dataset[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics,\n)\n\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:11:55.097470Z","iopub.execute_input":"2022-08-22T18:11:55.098160Z","iopub.status.idle":"2022-08-22T18:11:55.366354Z","shell.execute_reply.started":"2022-08-22T18:11:55.098123Z","shell.execute_reply":"2022-08-22T18:11:55.365468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_eval = (\n    load_dataset(\"csv\", data_files={\"test\": f\"{data_dir}/goodreads_test.csv\"})\n    [\"test\"]\n    .rename_columns({\"review_text\": \"text\"})\n    .map(preprocess_function, batched=True)\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:05:21.435946Z","iopub.execute_input":"2022-08-22T18:05:21.436226Z","iopub.status.idle":"2022-08-22T18:10:52.414691Z","shell.execute_reply.started":"2022-08-22T18:05:21.436198Z","shell.execute_reply":"2022-08-22T18:10:52.413736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = dataset_eval.to_pandas()\ndf_test[\"rating\"] = trainer.predict(dataset_eval).predictions.argmax(axis=1)\ndf_test[[\"review_id\", \"rating\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:10:52.416334Z","iopub.execute_input":"2022-08-22T18:10:52.416692Z","iopub.status.idle":"2022-08-22T18:11:51.546306Z","shell.execute_reply.started":"2022-08-22T18:10:52.416658Z","shell.execute_reply":"2022-08-22T18:11:51.544259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-22T18:11:51.547704Z","iopub.status.idle":"2022-08-22T18:11:51.548173Z","shell.execute_reply.started":"2022-08-22T18:11:51.547930Z","shell.execute_reply":"2022-08-22T18:11:51.547952Z"},"trusted":true},"execution_count":null,"outputs":[]}]}