{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a notebook to explore the competition metric by hand.","metadata":{}},{"cell_type":"code","source":"!pip install -q sacrebleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:00:18.412910Z","iopub.execute_input":"2026-03-10T15:00:18.413184Z","iopub.status.idle":"2026-03-10T15:00:24.358435Z","shell.execute_reply.started":"2026-03-10T15:00:18.413155Z","shell.execute_reply":"2026-03-10T15:00:24.357307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sacrebleu\nimport math\n\ndef metrics(hypotheses, references):\n    bleu = sacrebleu.corpus_bleu(hypotheses, references)\n    chrf = sacrebleu.corpus_chrf(hypotheses, references, word_order=2)\n    return {\n        \"bleu\": bleu.score,\n        'chrfpp': chrf.score,\n        \"geo_mean\": math.sqrt(bleu.score * chrf.score)\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:00:36.875551Z","iopub.execute_input":"2026-03-10T15:00:36.875898Z","iopub.status.idle":"2026-03-10T15:00:36.997810Z","shell.execute_reply.started":"2026-03-10T15:00:36.875869Z","shell.execute_reply":"2026-03-10T15:00:36.996736Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Note that we need to have at least 4 words in the sentences.","metadata":{}},{"cell_type":"code","source":"prediction = \"pašallum gold\"\nground_truths = [ \"pašallum gold\", \"pašallu-gold\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:00:39.086387Z","iopub.execute_input":"2026-03-10T15:00:39.086797Z","iopub.status.idle":"2026-03-10T15:00:39.096646Z","shell.execute_reply.started":"2026-03-10T15:00:39.086761Z","shell.execute_reply":"2026-03-10T15:00:39.095786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction = \"pašallum gold word1 word2\"\nground_truths = [ \"pašallum gold\", \"pašallu-gold\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:00:40.575177Z","iopub.execute_input":"2026-03-10T15:00:40.575507Z","iopub.status.idle":"2026-03-10T15:00:40.582198Z","shell.execute_reply.started":"2026-03-10T15:00:40.575480Z","shell.execute_reply":"2026-03-10T15:00:40.580850Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## -words","metadata":{}},{"cell_type":"code","source":"prediction = \"pašallum gold word1 word2 word3\"\nground_truths = [ \"pašallu-gold word1 word2 word3\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:04:49.192117Z","iopub.execute_input":"2026-03-10T15:04:49.192411Z","iopub.status.idle":"2026-03-10T15:04:49.198557Z","shell.execute_reply.started":"2026-03-10T15:04:49.192383Z","shell.execute_reply":"2026-03-10T15:04:49.197693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction = \"pašallu-gold word1 word2 word3\"\nground_truths = [ \"pašallu-gold word1 word2 word3\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:04:52.286167Z","iopub.execute_input":"2026-03-10T15:04:52.286513Z","iopub.status.idle":"2026-03-10T15:04:52.292521Z","shell.execute_reply.started":"2026-03-10T15:04:52.286468Z","shell.execute_reply":"2026-03-10T15:04:52.291606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction = \"pašallum gold word1 word2 word3\"\nground_truths = [ \"pašallum gold word1 word2 word3\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:04:54.481501Z","iopub.execute_input":"2026-03-10T15:04:54.481797Z","iopub.status.idle":"2026-03-10T15:04:54.488158Z","shell.execute_reply.started":"2026-03-10T15:04:54.481776Z","shell.execute_reply":"2026-03-10T15:04:54.487407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction = \"pašallu-gold word1 word2 word3\"\nground_truths = [ \"pašallum gold word1 word2 word3\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:04:55.797632Z","iopub.execute_input":"2026-03-10T15:04:55.797957Z","iopub.status.idle":"2026-03-10T15:04:55.804543Z","shell.execute_reply.started":"2026-03-10T15:04:55.797931Z","shell.execute_reply":"2026-03-10T15:04:55.803749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction = \"pašallu-gold word1 word2 word3\"\nground_truths = [ \"pašallu gold word1 word2 word3\"]\nmetrics(hypotheses=[prediction], references=[ground_truths])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-10T15:14:54.222142Z","iopub.execute_input":"2026-03-10T15:14:54.222492Z","iopub.status.idle":"2026-03-10T15:14:54.228857Z","shell.execute_reply.started":"2026-03-10T15:14:54.222469Z","shell.execute_reply":"2026-03-10T15:14:54.227670Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}