Skip to content

Instantly share code, notes, and snippets.

@hans

hans/.gitignore

Last active August 15, 2022 15:34
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save hans/73553e7eea002b261f639506ed04b8b9 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save hans/73553e7eea002b261f639506ed04b8b9 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
.gitignore
*.csv
__pycache__
.ipynb_checkpoints
Raw
analyze.Rmd
---
title: "R Notebook"
output: html_notebook
---
```{r}
library(tidyverse)
library(lme4)
library(lmerTest)
```
```{r}
SUITE = "number_prep"
```
```{r}
predictions_df <- read_csv(paste0(SUITE, ".predictions.csv"))
glimpse(predictions_df)
```
```{r}
predictions_df = predictions_df %>%
mutate(first_item_number=str_extract(used_item_numbers, "^(\\d+)"))
```
```{r}
lm = glmer(prediction_0 ~ log(prefix_length + 1) + (log(prefix_length + 1) | first_item_number),
family=binomial, data=predictions_df)
summary(lm)
```
```{r}
ranef(lm)
```
```{r}
#region_df = read_csv(paste0(SUITE, ".regions.csv"))
#glimpse(region_df)
region_df = rbind(read_csv("number_prep.regions.csv"), read_csv("number_prep2.regions.csv"))
glimpse(region_df)
```
```{r}
crit_region_data = region_df %>%
left_join(predictions_df %>% select(c(item_number, prefix_length, first_item_number)), by="item_number") %>%
filter(region_number == 7) %>%
mutate(grammatical=!grepl("mismatch", condition),
feature=str_extract(condition, "_(.+)$"))
```
```{r}
contrasts(crit_region_data$grammatical) = c(-1, 1)
lm = lmer(value ~ grammatical * log(prefix_length + 1) + (1 | first_item_number) + (1 | feature), data=crit_region_data,
control=lmerControl(optimizer="bobyqa", optCtrl=list(maxfun=100000)))
summary(lm)
```
```{r}
ranef(lm)
```
Display the source blob
Display the rendered blob
Raw
analyze.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "7bee86f2-615a-471f-b5f1-88a59bfb0e7b",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jgauthie/om2/miniconda/envs/huggingface/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.1\n",
" warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
]
}
],
"source": [
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import seaborn as sns"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "2e1d0fe3-f941-40bd-8516-3993d8365b49",
"metadata": {},
"outputs": [],
"source": [
"suites = [\"number_prep\", \"number_src\", \"number_orc\",\n",
" \"reflexive_src_fem\", \"reflexive_src_masc\",\n",
" \"reflexive_orc_fem\", \"reflexive_orc_masc\",\n",
" \"reflexive_prep_fem\", \"reflexive_prep_masc\",\n",
" \"subordination_src-src\", \"subordination_orc-orc\", \"subordination_pp-pp\",\n",
" \"mvrr\",\n",
" \"fgd_pp\", \"fgd_subject\", \"fgd_object\"]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "c43b04d0-3b0f-41a3-aeb3-ef3c985fdfce",
"metadata": {},
"outputs": [],
"source": [
"# Format: suite_first_token -> [((expected_grammatical_region, expected_grammatical_condition),\n",
"# (expected_ungrammatical_region, expected_ungrammatical_condition), ...]\n",
"# \n",
"# Will be used to compute margins between region-condition values.\n",
"\n",
"margin_measures = {\n",
" \"fgd_pp\": [((8, \"that_nogap\"), (8, \"what_nogap\")),\n",
" ((9, \"what_gap\"), (9, \"that_gap\"))],\n",
" \"fgd_subject\": [((4, \"that_nogap\"), (4, \"what_nogap\")),\n",
" ((5, \"what_gap\"), (5, \"that_gap\"))],\n",
" \"fgd_object\": [((6, \"that_nogap\"), (6, \"what_nogap\")),\n",
" ((7, \"what_gap\"), (7, \"that_gap\"))],\n",
" \n",
" \"number_prep\": [((7, \"match_sing\"), (7, \"mismatch_sing\")),\n",
" ((7, \"match_plural\"), (7, \"mismatch_plural\"))],\n",
" \"number_src\": [((8, \"match_sing\"), (8, \"mismatch_sing\")),\n",
" ((8, \"match_plural\"), (8, \"mismatch_plural\"))],\n",
" \"number_orc\": [((8, \"match_sing\"), (8, \"mismatch_sing\")),\n",
" ((8, \"match_plural\"), (8, \"mismatch_plural\"))],\n",
"\n",
" \"reflexive_prep_fem\": [((8, \"match_sing\"), (8, \"mismatch_sing\")),\n",
" ((8, \"match_plural\"), (8, \"mismatch_plural\"))],\n",
" \"reflexive_src_fem\": [((9, \"match_sing\"), (9, \"mismatch_sing\")),\n",
" ((9, \"match_plural\"), (9, \"mismatch_plural\"))],\n",
" \"reflexive_orc_fem\": [((9, \"match_sing\"), (9, \"mismatch_sing\")),\n",
" ((9, \"match_plural\"), (9, \"mismatch_plural\"))],\n",
" \"reflexive_prep_masc\": [((8, \"match_sing\"), (8, \"mismatch_sing\")),\n",
" ((8, \"match_plural\"), (8, \"mismatch_plural\"))],\n",
" \"reflexive_src_masc\": [((9, \"match_sing\"), (9, \"mismatch_sing\")),\n",
" ((9, \"match_plural\"), (9, \"mismatch_plural\"))],\n",
" \"reflexive_orc_masc\": [((9, \"match_sing\"), (9, \"mismatch_sing\")),\n",
" ((9, \"match_plural\"), (9, \"mismatch_plural\"))],\n",
" \n",
" \"subordination_src-src\": [((6, \"no-sub_no-matrix\"), (6, \"sub_no-matrix\")),\n",
" ((6, \"sub_matrix\"), (6, \"no-sub_matrix\"))],\n",
" \"subordination_orc-orc\": [((6, \"no-sub_no-matrix\"), (6, \"sub_no-matrix\")),\n",
" ((6, \"sub_matrix\"), (6, \"no-sub_matrix\"))],\n",
" \"subordination_pp-pp\": [((6, \"no-sub_no-matrix\"), (6, \"sub_no-matrix\")),\n",
" ((6, \"sub_matrix\"), (6, \"no-sub_matrix\"))],\n",
" \n",
" \"mvrr\": [((6, \"unreduced_ambig\"), (6, \"reduced_ambig\")),\n",
" ((6, \"reduced_unambig\"), (6, \"reduced_ambig\"))],\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "de68940b-c6d5-44cd-8794-1b723afcce8f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'fgd_pp': {'8 that_nogap', '8 what_nogap', '9 that_gap', '9 what_gap'},\n",
" 'fgd_subject': {'4 that_nogap', '4 what_nogap', '5 that_gap', '5 what_gap'},\n",
" 'fgd_object': {'6 that_nogap', '6 what_nogap', '7 that_gap', '7 what_gap'},\n",
" 'number_prep': {'7 match_plural',\n",
" '7 match_sing',\n",
" '7 mismatch_plural',\n",
" '7 mismatch_sing'},\n",
" 'number_src': {'8 match_plural',\n",
" '8 match_sing',\n",
" '8 mismatch_plural',\n",
" '8 mismatch_sing'},\n",
" 'number_orc': {'8 match_plural',\n",
" '8 match_sing',\n",
" '8 mismatch_plural',\n",
" '8 mismatch_sing'},\n",
" 'reflexive_prep_fem': {'8 match_plural',\n",
" '8 match_sing',\n",
" '8 mismatch_plural',\n",
" '8 mismatch_sing'},\n",
" 'reflexive_src_fem': {'9 match_plural',\n",
" '9 match_sing',\n",
" '9 mismatch_plural',\n",
" '9 mismatch_sing'},\n",
" 'reflexive_orc_fem': {'9 match_plural',\n",
" '9 match_sing',\n",
" '9 mismatch_plural',\n",
" '9 mismatch_sing'},\n",
" 'reflexive_prep_masc': {'8 match_plural',\n",
" '8 match_sing',\n",
" '8 mismatch_plural',\n",
" '8 mismatch_sing'},\n",
" 'reflexive_src_masc': {'9 match_plural',\n",
" '9 match_sing',\n",
" '9 mismatch_plural',\n",
" '9 mismatch_sing'},\n",
" 'reflexive_orc_masc': {'9 match_plural',\n",
" '9 match_sing',\n",
" '9 mismatch_plural',\n",
" '9 mismatch_sing'},\n",
" 'subordination_src-src': {'6 no-sub_matrix',\n",
" '6 no-sub_no-matrix',\n",
" '6 sub_matrix',\n",
" '6 sub_no-matrix'},\n",
" 'subordination_orc-orc': {'6 no-sub_matrix',\n",
" '6 no-sub_no-matrix',\n",
" '6 sub_matrix',\n",
" '6 sub_no-matrix'},\n",
" 'subordination_pp-pp': {'6 no-sub_matrix',\n",
" '6 no-sub_no-matrix',\n",
" '6 sub_matrix',\n",
" '6 sub_no-matrix'},\n",
" 'mvrr': {'6 reduced_ambig', '6 reduced_unambig', '6 unreduced_ambig'}}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"critical_regions = {suite: set(region for region_comparison in region_comparisons for region in region_comparison)\n",
" for suite, region_comparisons in margin_measures.items()}\n",
"critical_region_strs = {suite: {\" \".join(map(str, region)) for region in regions} for suite, regions in critical_regions.items()}\n",
"critical_region_strs"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "f26af2bf-c4aa-47e2-ac30-6b20ca7424d1",
"metadata": {},
"outputs": [],
"source": [
"sns.set_context(\"talk\")"
]
},
{
"cell_type": "markdown",
"id": "799003d6-1fb6-49a7-a5bb-313d5c630a1c",
"metadata": {},
"source": [
"## Prediction results"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "0444e056-8b0d-4dac-9a04-1ac6c275cff0",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>item_number</th>\n",
" <th>prediction_0</th>\n",
" <th>prediction_1</th>\n",
" <th>used_item_numbers</th>\n",
" <th>used_conditions</th>\n",
" <th>prefix_length</th>\n",
" <th>prediction_2</th>\n",
" <th>first_item</th>\n",
" <th>canonical_suite</th>\n",
" <th>suite_category</th>\n",
" <th>critical_region</th>\n",
" <th>num_prefix_sentences</th>\n",
" <th>prefix_length_bin</th>\n",
" </tr>\n",
" <tr>\n",
" <th>suite</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"5\" valign=\"top\">number_prep</th>\n",
" <th>0</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1.0</td>\n",
" <td>2</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>2</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>2</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2.0</td>\n",
" <td>3</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>3</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>3</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>4</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>4</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>4</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4.0</td>\n",
" <td>5</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>5</td>\n",
" <td>NaN</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>5</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"5\" valign=\"top\">fgd_object</th>\n",
" <th>2451</th>\n",
" <td>NaN</td>\n",
" <td>2452</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>5 10 13 5 4 13 21 6 15 1 7 10 9 17 12 23 20 4 ...</td>\n",
" <td>what_gap what_gap what_gap that_nogap what_gap...</td>\n",
" <td>494</td>\n",
" <td>NaN</td>\n",
" <td>5</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>31.0</td>\n",
" <td>(474.3, 500.65]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2452</th>\n",
" <td>NaN</td>\n",
" <td>2453</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>7 8 22 6 1 21 18 23 1 12 15 19 5 17 6 20 4 10 ...</td>\n",
" <td>what_gap what_gap that_nogap that_nogap what_g...</td>\n",
" <td>502</td>\n",
" <td>NaN</td>\n",
" <td>7</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>31.0</td>\n",
" <td>(500.65, 527.0]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2453</th>\n",
" <td>NaN</td>\n",
" <td>2454</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>2 16 15 2 22 10 4 24 4 8 19 7 13 20 18 15 23 7...</td>\n",
" <td>that_nogap what_gap that_nogap what_gap that_n...</td>\n",
" <td>497</td>\n",
" <td>NaN</td>\n",
" <td>2</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>31.0</td>\n",
" <td>(474.3, 500.65]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2454</th>\n",
" <td>NaN</td>\n",
" <td>2455</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>20 15 21 8 4 18 17 20 11 23 19 15 12 13 10 13 ...</td>\n",
" <td>what_gap that_nogap that_nogap what_gap that_n...</td>\n",
" <td>501</td>\n",
" <td>NaN</td>\n",
" <td>20</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>31.0</td>\n",
" <td>(500.65, 527.0]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2455</th>\n",
" <td>NaN</td>\n",
" <td>2456</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>10 18 13 13 17 1 6 11 16 21 15 7 22 24 18 10 1...</td>\n",
" <td>that_nogap what_gap that_nogap what_gap that_n...</td>\n",
" <td>494</td>\n",
" <td>NaN</td>\n",
" <td>10</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>31.0</td>\n",
" <td>(474.3, 500.65]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>41868 rows × 14 columns</p>\n",
"</div>"
],
"text/plain": [
" Unnamed: 0 item_number prediction_0 prediction_1 \\\n",
"suite \n",
"number_prep 0 0.0 1 True True \n",
" 1 1.0 2 True False \n",
" 2 2.0 3 False False \n",
" 3 3.0 4 True False \n",
" 4 4.0 5 False True \n",
"... ... ... ... ... \n",
"fgd_object 2451 NaN 2452 True True \n",
" 2452 NaN 2453 True True \n",
" 2453 NaN 2454 True True \n",
" 2454 NaN 2455 True True \n",
" 2455 NaN 2456 True True \n",
"\n",
" used_item_numbers \\\n",
"suite \n",
"number_prep 0 1 \n",
" 1 2 \n",
" 2 3 \n",
" 3 4 \n",
" 4 5 \n",
"... ... \n",
"fgd_object 2451 5 10 13 5 4 13 21 6 15 1 7 10 9 17 12 23 20 4 ... \n",
" 2452 7 8 22 6 1 21 18 23 1 12 15 19 5 17 6 20 4 10 ... \n",
" 2453 2 16 15 2 22 10 4 24 4 8 19 7 13 20 18 15 23 7... \n",
" 2454 20 15 21 8 4 18 17 20 11 23 19 15 12 13 10 13 ... \n",
" 2455 10 18 13 13 17 1 6 11 16 21 15 7 22 24 18 10 1... \n",
"\n",
" used_conditions \\\n",
"suite \n",
"number_prep 0 NaN \n",
" 1 NaN \n",
" 2 NaN \n",
" 3 NaN \n",
" 4 NaN \n",
"... ... \n",
"fgd_object 2451 what_gap what_gap what_gap that_nogap what_gap... \n",
" 2452 what_gap what_gap that_nogap that_nogap what_g... \n",
" 2453 that_nogap what_gap that_nogap what_gap that_n... \n",
" 2454 what_gap that_nogap that_nogap what_gap that_n... \n",
" 2455 that_nogap what_gap that_nogap what_gap that_n... \n",
"\n",
" prefix_length prediction_2 first_item canonical_suite \\\n",
"suite \n",
"number_prep 0 0 NaN 1 number_prep \n",
" 1 0 NaN 2 number_prep \n",
" 2 0 NaN 3 number_prep \n",
" 3 0 NaN 4 number_prep \n",
" 4 0 NaN 5 number_prep \n",
"... ... ... ... ... \n",
"fgd_object 2451 494 NaN 5 fgd_object \n",
" 2452 502 NaN 7 fgd_object \n",
" 2453 497 NaN 2 fgd_object \n",
" 2454 501 NaN 20 fgd_object \n",
" 2455 494 NaN 10 fgd_object \n",
"\n",
" suite_category \\\n",
"suite \n",
"number_prep 0 number \n",
" 1 number \n",
" 2 number \n",
" 3 number \n",
" 4 number \n",
"... ... \n",
"fgd_object 2451 fgd \n",
" 2452 fgd \n",
" 2453 fgd \n",
" 2454 fgd \n",
" 2455 fgd \n",
"\n",
" critical_region \\\n",
"suite \n",
"number_prep 0 {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
" 1 {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
" 2 {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
" 3 {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
" 4 {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
"... ... \n",
"fgd_object 2451 {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
" 2452 {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
" 2453 {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
" 2454 {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
" 2455 {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
"\n",
" num_prefix_sentences prefix_length_bin \n",
"suite \n",
"number_prep 0 0.0 (-0.527, 26.35] \n",
" 1 0.0 (-0.527, 26.35] \n",
" 2 0.0 (-0.527, 26.35] \n",
" 3 0.0 (-0.527, 26.35] \n",
" 4 0.0 (-0.527, 26.35] \n",
"... ... ... \n",
"fgd_object 2451 31.0 (474.3, 500.65] \n",
" 2452 31.0 (500.65, 527.0] \n",
" 2453 31.0 (474.3, 500.65] \n",
" 2454 31.0 (500.65, 527.0] \n",
" 2455 31.0 (474.3, 500.65] \n",
"\n",
"[41868 rows x 14 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prediction_dfs = [pd.read_csv(f\"{suite}.predictions.csv\") for suite in suites]\n",
"prediction_df = pd.concat(prediction_dfs, names=[\"suite\"], keys=suites)\n",
"prediction_df[\"first_item\"] = prediction_df.used_item_numbers.str.split().str.get(0)\n",
"\n",
"canonical_suite = {suite_name: next(can_suite for can_suite in critical_regions.keys()\n",
" if can_suite in suite_name)\n",
" for suite_name in set(prediction_df.index.get_level_values(\"suite\"))}\n",
"\n",
"prediction_df[\"canonical_suite\"] = prediction_df.index.get_level_values(\"suite\").map(canonical_suite)\n",
"prediction_df[\"suite_category\"] = prediction_df.canonical_suite.str.split(\"_\").apply(lambda parts: \"_\".join(parts[:-1]) if len(parts) > 1 else parts[0])\n",
"prediction_df[\"critical_region\"] = prediction_df.canonical_suite.map(critical_regions)\n",
"\n",
"prediction_df[\"num_prefix_sentences\"] = prediction_df.used_conditions.str.count(\" \").fillna(0)\n",
"prediction_df[\"prefix_length_bin\"] = pd.cut(prediction_df.prefix_length, 20).astype(str)\n",
"\n",
"prediction_df"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "34385397-ba33-4a91-9dfe-c64567737053",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Unnamed: 0</th>\n",
" <th>item_number</th>\n",
" <th>prediction_0</th>\n",
" <th>prediction_1</th>\n",
" <th>used_item_numbers</th>\n",
" <th>used_conditions</th>\n",
" <th>prefix_length</th>\n",
" <th>prediction_2</th>\n",
" <th>first_item</th>\n",
" <th>suite_category</th>\n",
" <th>critical_region</th>\n",
" <th>num_prefix_sentences</th>\n",
" <th>prefix_length_bin</th>\n",
" </tr>\n",
" <tr>\n",
" <th>canonical_suite</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>fgd_object</th>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>fgd</td>\n",
" <td>{(7, that_gap), (6, what_nogap), (7, what_gap)...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fgd_pp</th>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>fgd</td>\n",
" <td>{(8, what_nogap), (9, that_gap), (8, that_noga...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fgd_subject</th>\n",
" <td>NaN</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>fgd</td>\n",
" <td>{(4, that_nogap), (5, that_gap), (4, what_noga...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mvrr</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>mvrr</td>\n",
" <td>{(6, reduced_ambig), (6, unreduced_ambig), (6,...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>number_orc</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>number</td>\n",
" <td>{(8, match_plural), (8, mismatch_sing), (8, ma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>number_prep</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>number</td>\n",
" <td>{(7, mismatch_plural), (7, match_sing), (7, mi...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>number_src</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>number</td>\n",
" <td>{(8, match_plural), (8, mismatch_sing), (8, ma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_orc_fem</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_orc</td>\n",
" <td>{(9, match_sing), (9, match_plural), (9, misma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_orc_masc</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_orc</td>\n",
" <td>{(9, match_sing), (9, match_plural), (9, misma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_prep_fem</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_prep</td>\n",
" <td>{(8, match_plural), (8, mismatch_sing), (8, ma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_prep_masc</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_prep</td>\n",
" <td>{(8, match_plural), (8, mismatch_sing), (8, ma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_src_fem</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_src</td>\n",
" <td>{(9, match_sing), (9, match_plural), (9, misma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>reflexive_src_masc</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>reflexive_src</td>\n",
" <td>{(9, match_sing), (9, match_plural), (9, misma...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>subordination_orc-orc</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>subordination</td>\n",
" <td>{(6, sub_no-matrix), (6, sub_matrix), (6, no-s...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>subordination_pp-pp</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>subordination</td>\n",
" <td>{(6, sub_no-matrix), (6, sub_matrix), (6, no-s...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" <tr>\n",
" <th>subordination_src-src</th>\n",
" <td>0.0</td>\n",
" <td>1</td>\n",
" <td>True</td>\n",
" <td>True</td>\n",
" <td>1</td>\n",
" <td>None</td>\n",
" <td>0</td>\n",
" <td>None</td>\n",
" <td>1</td>\n",
" <td>subordination</td>\n",
" <td>{(6, sub_no-matrix), (6, sub_matrix), (6, no-s...</td>\n",
" <td>0.0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" Unnamed: 0 item_number prediction_0 prediction_1 \\\n",
"canonical_suite \n",
"fgd_object NaN 1 True True \n",
"fgd_pp NaN 1 True True \n",
"fgd_subject NaN 1 True True \n",
"mvrr 0.0 1 True True \n",
"number_orc 0.0 1 True True \n",
"number_prep 0.0 1 True True \n",
"number_src 0.0 1 True True \n",
"reflexive_orc_fem 0.0 1 False True \n",
"reflexive_orc_masc 0.0 1 True True \n",
"reflexive_prep_fem 0.0 1 False True \n",
"reflexive_prep_masc 0.0 1 True True \n",
"reflexive_src_fem 0.0 1 False True \n",
"reflexive_src_masc 0.0 1 True True \n",
"subordination_orc-orc 0.0 1 True True \n",
"subordination_pp-pp 0.0 1 True True \n",
"subordination_src-src 0.0 1 True True \n",
"\n",
" used_item_numbers used_conditions prefix_length \\\n",
"canonical_suite \n",
"fgd_object 1 None 0 \n",
"fgd_pp 1 None 0 \n",
"fgd_subject 1 None 0 \n",
"mvrr 1 None 0 \n",
"number_orc 1 None 0 \n",
"number_prep 1 None 0 \n",
"number_src 1 None 0 \n",
"reflexive_orc_fem 1 None 0 \n",
"reflexive_orc_masc 1 None 0 \n",
"reflexive_prep_fem 1 None 0 \n",
"reflexive_prep_masc 1 None 0 \n",
"reflexive_src_fem 1 None 0 \n",
"reflexive_src_masc 1 None 0 \n",
"subordination_orc-orc 1 None 0 \n",
"subordination_pp-pp 1 None 0 \n",
"subordination_src-src 1 None 0 \n",
"\n",
" prediction_2 first_item suite_category \\\n",
"canonical_suite \n",
"fgd_object None 1 fgd \n",
"fgd_pp None 1 fgd \n",
"fgd_subject None 1 fgd \n",
"mvrr True 1 mvrr \n",
"number_orc None 1 number \n",
"number_prep None 1 number \n",
"number_src None 1 number \n",
"reflexive_orc_fem None 1 reflexive_orc \n",
"reflexive_orc_masc None 1 reflexive_orc \n",
"reflexive_prep_fem None 1 reflexive_prep \n",
"reflexive_prep_masc None 1 reflexive_prep \n",
"reflexive_src_fem None 1 reflexive_src \n",
"reflexive_src_masc None 1 reflexive_src \n",
"subordination_orc-orc None 1 subordination \n",
"subordination_pp-pp None 1 subordination \n",
"subordination_src-src None 1 subordination \n",
"\n",
" critical_region \\\n",
"canonical_suite \n",
"fgd_object {(7, that_gap), (6, what_nogap), (7, what_gap)... \n",
"fgd_pp {(8, what_nogap), (9, that_gap), (8, that_noga... \n",
"fgd_subject {(4, that_nogap), (5, that_gap), (4, what_noga... \n",
"mvrr {(6, reduced_ambig), (6, unreduced_ambig), (6,... \n",
"number_orc {(8, match_plural), (8, mismatch_sing), (8, ma... \n",
"number_prep {(7, mismatch_plural), (7, match_sing), (7, mi... \n",
"number_src {(8, match_plural), (8, mismatch_sing), (8, ma... \n",
"reflexive_orc_fem {(9, match_sing), (9, match_plural), (9, misma... \n",
"reflexive_orc_masc {(9, match_sing), (9, match_plural), (9, misma... \n",
"reflexive_prep_fem {(8, match_plural), (8, mismatch_sing), (8, ma... \n",
"reflexive_prep_masc {(8, match_plural), (8, mismatch_sing), (8, ma... \n",
"reflexive_src_fem {(9, match_sing), (9, match_plural), (9, misma... \n",
"reflexive_src_masc {(9, match_sing), (9, match_plural), (9, misma... \n",
"subordination_orc-orc {(6, sub_no-matrix), (6, sub_matrix), (6, no-s... \n",
"subordination_pp-pp {(6, sub_no-matrix), (6, sub_matrix), (6, no-s... \n",
"subordination_src-src {(6, sub_no-matrix), (6, sub_matrix), (6, no-s... \n",
"\n",
" num_prefix_sentences prefix_length_bin \n",
"canonical_suite \n",
"fgd_object 0.0 (-0.527, 26.35] \n",
"fgd_pp 0.0 (-0.527, 26.35] \n",
"fgd_subject 0.0 (-0.527, 26.35] \n",
"mvrr 0.0 (-0.527, 26.35] \n",
"number_orc 0.0 (-0.527, 26.35] \n",
"number_prep 0.0 (-0.527, 26.35] \n",
"number_src 0.0 (-0.527, 26.35] \n",
"reflexive_orc_fem 0.0 (-0.527, 26.35] \n",
"reflexive_orc_masc 0.0 (-0.527, 26.35] \n",
"reflexive_prep_fem 0.0 (-0.527, 26.35] \n",
"reflexive_prep_masc 0.0 (-0.527, 26.35] \n",
"reflexive_src_fem 0.0 (-0.527, 26.35] \n",
"reflexive_src_masc 0.0 (-0.527, 26.35] \n",
"subordination_orc-orc 0.0 (-0.527, 26.35] \n",
"subordination_pp-pp 0.0 (-0.527, 26.35] \n",
"subordination_src-src 0.0 (-0.527, 26.35] "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Store basic representation of first item of each suite\n",
"basic_suite_df = prediction_df[prediction_df.prefix_length == 0].groupby(\"canonical_suite\").first()\n",
"basic_suite_df"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "15424ba5-746e-4162-bc85-7f397bbffa57",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"canonical_suite variable \n",
"fgd_object prediction_0 1.000000\n",
" prediction_1 0.958333\n",
"fgd_pp prediction_0 0.916667\n",
" prediction_1 0.958333\n",
"fgd_subject prediction_0 0.625000\n",
" prediction_1 0.875000\n",
"mvrr prediction_0 1.000000\n",
" prediction_1 0.892857\n",
" prediction_2 0.857143\n",
"number_orc prediction_0 0.631579\n",
" prediction_1 0.473684\n",
"number_prep prediction_0 0.789474\n",
" prediction_1 0.526316\n",
"number_src prediction_0 0.842105\n",
" prediction_1 0.473684\n",
"reflexive_orc_fem prediction_0 0.473684\n",
" prediction_1 1.000000\n",
"reflexive_orc_masc prediction_0 0.842105\n",
" prediction_1 0.947368\n",
"reflexive_prep_fem prediction_0 0.210526\n",
" prediction_1 1.000000\n",
"reflexive_prep_masc prediction_0 0.894737\n",
" prediction_1 0.894737\n",
"reflexive_src_fem prediction_0 0.157895\n",
" prediction_1 1.000000\n",
"reflexive_src_masc prediction_0 0.684211\n",
" prediction_1 0.789474\n",
"subordination_orc-orc prediction_0 1.000000\n",
" prediction_1 1.000000\n",
"subordination_pp-pp prediction_0 1.000000\n",
" prediction_1 1.000000\n",
"subordination_src-src prediction_0 1.000000\n",
" prediction_1 1.000000\n",
"Name: value, dtype: float64"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Get baseline accuracy results.\n",
"prediction_df[prediction_df.used_conditions.isna()] \\\n",
" [[col for col in prediction_df if col.startswith(\"prediction_\") or col == \"canonical_suite\"]] \\\n",
" .melt(id_vars=[\"canonical_suite\"]).groupby([\"canonical_suite\", \"variable\"]).value.mean().dropna()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "39badb4d-0271-4bd2-b6b8-cefcf1efd447",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<AxesSubplot:xlabel='prefix_length', ylabel='Count'>"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAn4AAAHzCAYAAAC+FhD7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABYIElEQVR4nO3deXhU5dnH8e8dVhFIgIAEREBRQEHiCipCkMWibKIoCEpwfytVLCoiKKDivoC11qUgi4pSFalKKSqLShXckAKKGyBSQAklQBU05Hn/mEmcJBMySWZyJnN+n+s612Sec885zzkTzc2zHXPOISIiIiKJL8nrCoiIiIhIxVDiJyIiIuITSvxEREREfEKJn4iIiIhPKPETERER8YmqXlcg3plZDoEEebfXdREREREpQV0g1zkXNsczLedycGaWC1hycrLXVRERERE5qOzsbADnnAvbq6sWv5LtTk5OTt61a5fX9RARERE5qJSUFLKzs4vtpdQYPxERERGfUOInIiIi4hNK/ERERER8QomfiIiIiE8o8RMRERHxCc3qFRERX8vNzSU7O5u9e/eSm5vrdXVEwkpKSqJ27dokJyeTlFT2dju1+ImIiK/95z//Ydu2bfzyyy9eV0WkWL/88gvbtm1j69at5TqOWvxERMS3/ve//7Fnzx5SU1NJTU3FzLyukkhYzjl+/PFHsrKyqFevHrVq1SrTcdTiJyIivrV7926SkpJo0KCBkj6Ja2ZGamoqSUlJeU/nKBMlfiIi4ls5OTlUq1atXGOmRCpKUlIS1apVIycnp+zHiGJ9REREKpXc3FyqVKnidTVEIlalSpVyTUJS4iciIiLiE0r8RERERHxCiZ+IiEglMnHiRMyMjRs3el2VuLFx40bMjIkTJ0YUn5mZWa7JPEuXLsXMmDFjRpmP4RUlfiIiIpXc0qVLmThxIrt27fK6KmWyceNGJk6cyKpVq7yuSsJT4iciIlKJjB8/np9//pnmzZvnly1dupRJkyZV6sRv0qRJFZb4Pf300/z8888Vcq54owWcRUREKpGqVatStar+fJdHtWrVqFatmtfV8IRa/ERERGJo3759TJw4kdatW1OrVi1SUlJo3749N910E3Dw8WnhxvMVLsvMzGTSpEkAtGzZEjMrcrzs7GzGjBlDq1atqFGjBg0bNmTIkCF8++23Zbom5xxPP/00HTt2pHbt2tSuXZv27dtz++2358fs2bOH8ePH07FjR1JTU6lRowatWrXilltu4aeffsqPmzFjBt26dQNgxIgR+fXPyMgoU91ef/11TjnlFGrWrElaWho33XRTkXXvihvjt2zZMk477TQOOeQQGjduzPXXX8/atWsPOn7wmWee4bjjjqNGjRo0b96c+++/v0z1rij6J4OIiEgMXXvttUyfPp1LL72UG264gQMHDvDVV1+xePHiqBz/6quvZvfu3cybN49HHnmE1NRUAI4//nggkPSdfvrpfPfdd1x22WUcd9xxbN26lccff5yOHTvy0UcfFeg2jsQll1zCc889R8eOHRk3bhwpKSl88cUXvPTSS9xxxx0AbNmyhb/+9a+cf/75XHzxxVStWpVly5Zx//338+mnn/LPf/4TgC5dunDrrbdy9913c9VVV3HmmWcCcNhhh5X6XixYsIDHH3+ca665hssuu4z58+fz4IMPUq9ePW699daDfva9996jV69e1KtXj1tuuYWUlBTmzp3L8uXLi/3ME088wfbt27n88stJSUnh2WefZcyYMRx++OFcfPHFpa5/hXDOaTvIBuxKTk52iarFEYc7Mytxa3HE4V5XVUQk6jZu3Og2btwY03PUq1fP9e7du9j9GzZscICbMGFCkX0TJkxwgNuwYUOpy/Jcd911rmbNmm7VqlUFyjdu3Ojq1Knjhg8fXqrrefHFFx3ghg0b5g4cOFBgX+j7/fv3u19++aXI58ePH+8At2LFivyyJUuWOMA988wzpapLnrx7WKtWrQL3IDc31x133HGucePGBeKHDx/uAinQb0455RRXo0YN98033+SX/fLLL+70008v8v3k1TctLc3997//zS//3//+51JTU12nTp3KdB2RKOl3Njk52QG7XDF5jVr8fG7T5i3kvj25xLik7uMqoDYiIoknOTmZtWvXsmbNGtq1a1eh53bO8dxzz9GlSxeaNm3Kjh078vcdeuihdOrUiUWLFpXqmM899xwADz74YJFH3YW+r169ev7POTk57NmzhwMHDtCjRw/uuusuVqxYwamnnlqWyyrWgAEDaNGiRf57M6Nbt2489thj7N27l9q1a4f93Pbt2/nwww+58MILOfLII/PLq1WrxvXXX8+//vWvsJ8bMWIEKSkp+e9r1apFp06deP/996NyPbGgxE9ERCSGpkyZwiWXXEL79u058sgj6datG3379qVv374xf0bwjz/+SFZWFosWLaJhw4ZhY0pbh6+++oq0tLSIumIff/xxnnjiCdauXVvkMWP//e9/S3XeSIQmbXkaNGgAQFZWVrGJ34YNGwBo3bp1kX3hyko6X1ZWVkT19YISPxERkRjq378/GzduZMGCBSxbtoy33nqLadOmceaZZ/LWW28ddCHhwpMSSssFhizRo0cPxowZU65jhR4zksWPH374YUaPHk2vXr247rrraNKkCdWrV2fLli1kZmaW63mzxTnYc5fz7kVp95X1fPFKiZ+IiEiM1a9fn2HDhjFs2DCcc9xyyy3cf//9zJ8/n9/97ncA7Ny5s8jnIp11W1wi1rBhQ1JSUti9ezc9evQo+wWEaN26NfPnz2f79u0HbfWbPXs2LVq04B//+EeBVsWFCxcWiS3PUzSiIa/lbv369UX2hSurzLSci4iISIwcOHCgyKLKZsYJJ5wABJK9OnXq0LhxYxYvXlyg5enbb7/l1Vdfjeg8eV2YhZPHpKQkhg4dysqVK3nppZfCfvaHH36I8GoChg4dCsDNN99cpNUutP5VqlTBzAqU5eTkcO+990Zc/4py2GGHcfLJJzN//vwCyfavv/7K1KlTPalTrKjFT0REJEb27NlDWloa/fr144QTTqBRo0Zs2LCBv/zlL9SrV4++ffsCMHLkSMaPH0/v3r0ZMGAA//nPf3jiiSdo164dH374YYnn6dSpEwBjxoxh6NCh1KxZk3bt2tGuXTsmT57M8uXLufDCC7nwwgvp1KkT1atXZ9OmTSxYsICTTjqpVM+cHTRoEBdddBGzZs3iq6++ol+/ftSrV48vv/ySf/7zn6xZswaACy64gLFjx9K7d28GDhzI7t27ef7558MunHzsscdSp04dHn/88fy1Dhs1asRZZ50Vcb3K68EHH6Rnz56cfvrp/P73vyc5OZm5c+fyyy+/AN63SkaLEj8RkXJo2bwZmzZvKTGuebOmbNi0uQJqJPGkVq1ajBo1irfffpu33nqLvXv35ieCY8eOpUmTJkAgYcvOzmb27NksXbqUY489lmnTpvHxxx9HlPidccYZ3HfffTzxxBNceeWV5OTkMGHCBNq1a0dycjLLly/noYceYu7cucyfP5+qVaty+OGH07lzZ6644opSX9fzzz/PmWeeybRp07jjjjuoUqUKLVu2ZNCgQfkxN910E845pk2bxvXXX0/jxo256KKLGDFiBMcee2yB4x1yyCG88MILjB8/nlGjRrF//366du1aoYlf165dWbhwYf6agsnJyQwePJiLL76YTp06ccghh1RYXWLJyjqg0S/MbFdycnJyZX3+YUmSkpIiXs4lFgNxRSo7/TdUuW3atAmg1AsYi3+8/PLLXHDBBcyZM4fBgwd7XZ0Sf2dTUlLIzs7Ods6lhNuvMX4iIiLie8459u3bV6Ds119/5eGHH6Zq1aplfoRcvFFXr4iIiM8dOHCAH3/8scS4+vXrF1iYOda2bdtWYkxycnJUumH3799P8+bNGTp0KK1btyYrK4sXX3yR1atXM2bMGBo3blzuc8QDJX4iIiI+t3nzZlq2bFli3JIlSyq05SstLa3EmGeeeYbMzMxyn6tatWqce+65zJ8/n61bt+Kco3Xr1vz5z3/m97//fbmPHy+U+ImIiPhc48aNefPNN0uM69ChQwXU5jeR1Om4446LyrmqVKnC9OnTo3KseKbET0RExOdq1qwZtQWeoyke61TZeTq5w8xmmJk7yNY4JLanmX1gZj+b2Q9m9qSZpYQ5Zm0ze9TMtgZjPzKzfhV6YSIiIiJxyOsWvzuBJwqVVQP+Cax2zm0DMLMMYAHwKjAeaALcB7QzszOdc6FrJMwDTgRuBjYAmcA8M+vrnFsQqwsRERERiXeeJn7OuW+Ab0LLzGwgcAgwLaT4fmANcFFekmdmW4FFwCDgxWDZOUAPYKBzbl6wbAlwJPAQgeRRRERExJficR2/y4Cf+C2ZawqcAswObdlzzr0JbAHOD/nseUA2MD8kzgEzgTZmVnCpcBEREREfiavEz8zSgN8BLznndgeL2wVf14T5yL9D9ufFrivU9QuwutCxRERERHzH6zF+hQ0HqlCwm7dB8HVnmPidBMbzhcZ+WUxc6LHymdmuEuqUXMJ+ERERkUoh3hK/TOBr59w7YfYV91DhwuUHe/iwHkwskqBaNm/Gps1bSoxr3qwpGzZtroAaSWU3atQoVq1a5XU1SE9PZ8qUKV5XQxJE3CR+ZtYZaA2MK7QrK/hapLUOqE/BlsCsg8RBmFbD4h5iHFKvXajVTyTubdq8hdy3J5cYl9S98P9iRMJbtWoVKz76hKZHtfWsDlu++dyzc0fbjBkzGDFiBJ9++inp6eleV8e34ibxIzCp4wCBiRih1gZf2xGYxRuqPfCvQrHnm1lSoXF+7YOv4cYJioiIhNX0qLZc++Bsz87/5xsv8ezckpjiYnKHmR1KYFmWfzrnCvTVOOe+Bz4ChppZUshnugNNgVdCwucBKUDfQqe4FFjvnFsX/dqLiIhIvNi/f7/XVYhrcZH4ARcBtYHiHpI3BugAzDGz7mZ2CTAbWAH8LSRuAbAEmGZml5lZNzObAXQGbopV5UVEROLNxIkTMTM+//xzLrroIurWrcthhx3GZZddRnZ2NgAbN27EzJgxY0aRz5sZEydOLHK81atX079/f+rUqUPDhg0ZO3Ysubm5fPzxx3Tt2pVDDz2Uo48+mtmzw7eU7tixg4svvpjk5GTq1q3LkCFD+OGHH4rEzZ49m1NOOYVatWqRnJzMgAED+OqrrwrEZGRkkJ6ezltvvcUpp5xCzZo1ueeee8p+03wgXhK/EcAO4O/hdjrnFgN9gBbAG8DDwdfezrkDIXEOGAC8ANwN/AM4nsCCzq/FrvoiIiLxaeDAgbRt25ZXXnmFMWPG8Pzzz3PDDTeU+XgXXnghp512GvPmzWPIkCHce++93HjjjQwZMoRhw4Yxb9482rZty/Dhw8NOjrnssstISUlh7ty5TJ48mTfeeINevXrx66+/5sdMmDCBzMxMTj31VF555RWeeuopvv76a8444wy2bt1a4Hjff/89l19+OVdeeSX/+Mc/6NOnT5mvzQ/iYoyfc+7MCGIWAgsjiNsNjAxuIiIivnbVVVflJ3o9evTgm2++Yfr06UybNq2ET4Z37bXX8oc//AGA7t278/rrr/PII4/w7rvv0rlzZwBOPvlkGjVqxJw5c4pM5DjjjDN4/PHHATj77LNp1KgRgwcP5qWXXmLIkCF899133H333YwePZr7778//3NnnnkmRx99NA8//DAPPPBAfnlWVhbz58/njDPOKNP1+E28tPiJiIhIDPTr16/A++OPP559+/axffv2Mh3v3HPPzf/ZzGjTpg116tTJT/oA6tevT6NGjdi0aVORzw8ePLjA+/PPP5+qVauybNkyABYtWkROTg5Dhw4lJycnf2vUqBEnnnhiflyehg0bKukrhbho8RMREZHYaNCg4CpnNWrUAGDfvn1lOl79+vULvK9evXqRsrzycOdo3LhxgfdVq1alQYMGZGUFVm/LS0iLW/KlZcuWBd6npaVFXHdR4iciIuJbNWvWBIrOhM1LwmJh27ZtBd7n5OSQlZWVn6CmpqYC8Oqrr9K0adMin89LXPOYWYxqmpiU+ImIiPjUYYcdRs2aNVm9enWB8vnz58fsnC+88AL9+/fPf//yyy+Tk5ND165dAejVqxdVqlTh22+/LRAn0aHET0RExKfMjKFDhzJ9+nSOOuooOnTowMqVK3n++edjds7ly5dz7bXX0r9/f9avX8+4cePo0KEDF1xwARDoyr3tttsYM2YM3377LT179qRu3bps3bqV5cuX06ZNG0aO1PzNslLiJyIiUowt33zu6dMztnzzOYeffGJMz/Hwww9jZtx///3s3buXs846i9dff50WLVrE5HzPPPMM06ZNY9CgQeTm5nLuuecydepUqlWrlh8zYcIEjj32WP70pz8xY8YMcnJyaNKkCaeddhodO3aMSb38wgJL30lxzGxXcnJy8q5du7yuSkwkJSVF/HzT3NzcEuNEvOLV77L+G6rc8madNm/evMi+UaNGhV2HrqKlp6czZcoUr6shceJgv7MAKSkpZGdnZzvnUsLtV4ufiIhIGEq2JBFpHT8RERERn1DiJyIiIuITSvxEREREfEKJn4iIiIhPKPETERER8QklfiJhtGzejKSkpBK3ls2beV1VERGRiGk5F5EwNm3eEvHabCIiIpWFWvxEREREfEItfiIiImHoyR2SiJT4iYiIhLFq1SpWfbSC9FZp3tXh662enXv//v384Q9/4O9//zs//PADXbp0YenSpZgZEyZMYOLEiTE5b4sWLcjIyGDGjBkxOb7fKfETEREpRnqrNJY+cqVn58+44WnPzv2Xv/yFp59+mieeeILjjz+e5OTkCjnvvHnzqFu3boWcy4+U+ImIiPjE/v37qVGjRkSxa9eupU6dOlx99dUxrlVBJ5xwQoWez280uUNERCQBTZw4ETPjk08+oW/fvtStW5ff/e535Obm8sgjj9C+fXtq1qxJamoql156Kdu2bcv/rJnx17/+lT179mBmmNlBu14///xzBg0aRGpqKjVq1OD444/n+eefz9+/Y8cODj/8cDIyMjhw4EB++Ycffkj16tW54YYb8statGhBZmYmEOhuNzNmzpxZ5JzPPvssZsaKFSvyy95//3169+5NSkoKhxxyCJ06dWLRokVluX0JS4mfiIhIAhs4cCAnnXQS8+bNY+zYsYwYMYKxY8fSr18/XnvtNR588EGWLFlCRkYG//vf/4BAAnXOOedQu3Zt3n//fd5//33OPffcsMdfvXo1p556Khs3buRPf/oTr732Gh07dmTo0KFMmzYNgNTUVObMmcN7773HhAkTANi1axcXXXQR6enp3HfffWGPnZ6eTocOHcImnTNnzqRNmzZ07NgRgEWLFtG1a1cOHDjA9OnTefnll0lLS+Occ85R8hdCXb0iIiIJ7Morr2TcuMCao//617+YNWsWf/7zn/n973+fH5Oens6JJ57IjBkzuPbaa+nUqRMNGzakSpUqdOrU6aDHv/HGG0lNTWXp0qUceuihAPTq1YsdO3Ywbtw4RowYQVJSEmeeeSaTJk3i9ttvp0uXLjz55JPs3LmTt99+m+rVqxd7/OHDhzN69Gg2btxIixYtANi8eTOLFy/mnnvuyY8bOXIkJ554IgsXLiQpKdCu1bt3b04++WTGjRtHr169ynT/Eo1a/ERERBLYeeedl//zggULSEpKYvDgweTk5ORv7dq1o2nTpixbtqxUx963bx9Llixh4MCB1KhRo8AxzznnHLZv384XX3yRHz927Fi6d+9Ov379eOWVV5g2bRotW7Y86DmGDRtG1apVC3T3zp49GzPjkksuAeDrr7/mq6++4uKLLyY3Nze/DgcOHKB37958/PHH7N27t1TXlqiU+ImIiCSwtLTflqPZvn07ubm5NGjQgGrVqhXYvv/+e3bs2FGqY2dlZZGTk8PDDz9c5HhXXXUVQIFjJiUlMXz4cPbv30+rVq0YMGBAiedo2LAhvXv3ZtasWTjngEA379lnn51/bdu3bwfg+uuvL1KPyZMn45xj586dpbq2RKWuXhERkQRmZvk/p6amkpSUxHvvvUe1atWKxNapU6dUx65Xrx5JSUmMGDGCa665JmxM69at83/evHkz119/PSeddBKffvopd955Z0TrAWZmZjJw4EDeeecdqlevzpdffsldd91V4LoAbrvtNvr16xf2GI0bNy7FlSUuJX4ilVDL5s3YtHlLiXHNmzVlw6bNFVAjEakMevfuzb333su2bdsKdAGXVa1atejatSurVq0iPT2dqlWLTytycnIYPHgwhx56KG+++SYPP/wwd955J127dqVbt24HPU+fPn1ITU1lxowZ1KhRg/r16xdI8I455hiOPPJIVq9ezR133FHu60pkSvxEKqFNm7eQ+/bkEuOSuo+rgNqISGXRpUsXMjMzufTSS/nDH/5A586dqVmzJlu2bGHJkiWcc845XHDBBaU65pQpUzjzzDPJyMjg6quv5ogjjmDXrl188cUXrFy5kpdffhmAcePGsXLlSt555x3q1avHxIkTeffddxk6dCifffYZDRs2LPYc1apV4+KLL2b69OlUrVqVoUOHFliP0Mz4y1/+Qp8+fejTpw+XXHIJaWlp7Nixg9WrV7N161aefPLJst20BKPET0REpBirvt7q6dMzVn29lfSTj4jqMadPn06nTp14+umnmTJlCklJSTRt2pSuXbty/PHHl/p4xx9/PB9//DF33HEHY8aMYceOHdSvX5+2bdty4YUXAvCPf/yDBx54gHvuuYfTTjsNgCpVqjBnzhzS09O59NJLWbBgQYFu6cJGjBjBo48+CpC/zl+oXr168f777zN58mRGjhxJdnY2DRs2pEOHDgwfPrzU15WolPiJiIiEkZ6e7nUVSD/5iDLXY+LEiWHHz5kZV199dYlP5Chuwea8CRahWrVqxaxZs4o9Vu/evcnNzS1SnpaWlj8xI8/GjRvDHiM9PT3suUOddNJJvPLKKweN8TslfiIiImFMmTLF6yqIRJ2WcxERERHxCSV+IiIiIj6hxE9ERETEJ5T4iYiIiPiEEj/xRMvmzUhKSipxa9m8mddVFRERSRia1Sue0ALEIiIiFU8tfiIiIiI+ocRPRERExCfU1SsiIhLGqFGjWLVqldfVID09XYtJS9Qo8RMREQlj1apVrPx4JYe3PtyzOny//nvPzh1qxowZjBgxgk8//dSzR9ktXbqUbt26sWTJEjIyMgDyX5cuXRqz8y5cuJAPPvigyOPvNm7cSMuWLXnmmWfCPjs4XinxExERKcbhrQ9n1F9HeXb+KVdM8ezclcHjjz8e83MsXLiQqVOnFkn80tLSeP/99znqqKNiXodoUuInIiIicWH//v3UqFEj4vhjjz02hrU5uBo1atCpUyfPzl9Wnk/uMLMMM1tkZrvM7CczW2dmVxWK6WlmH5jZz2b2g5k9aWYpYY5V28weNbOtwdiPzKxfhV2MiIhIHPnxxx+58soradasGTVq1KBRo0Z07dqVFStWAGBmRVqyAFq0aBG2+3LHjh1cfPHFJCcnU7duXYYMGcIPP/xQICYnJ4e77rqLVq1aUb16ddLS0rjqqqvIysoqco4BAwYwZ84c2rVrR/Xq1ZkzZw4An3/+OWeffTa1atUiNTWVK664gt27dxepT0ZGRn53LwS6fM2MuXPncsstt5CWlkbdunXp0aMH69evL/DZN998k379+tG0aVMOOeQQjjnmGP74xz8WOE9mZiZTp07Nv1d5GwS6es2MGTNmFDjuSy+9xMknn8whhxxCcnIyffr0YfXq1QViMjMzSUlJYf369fzud7+jdu3aNGvWjNGjR7N///4i1xlNnrb4mdlwYBrwNPAw8CvQBqgeEpMBLABeBcYDTYD7gHZmdqZzLjfkkPOAE4GbgQ1AJjDPzPo65xbE9mpERETiy7Bhw/jmm2+YPHkyLVq0ICsrixUrVrBz584yHe+yyy6jT58+zJ07ly+//JJx48bx+eef8+GHH1KtWjUArrjiCmbNmsXo0aPp2bMnX3zxBePHj+f9999n5cqVHHLIIfnHW7lyJZ9//jm33XYbaWlpNGnShG3bttG1a1dq1qzJ448/zmGHHcbs2bMZOXJkxPUcM2YMZ555JtOmTSM7O5sxY8bQt29fPv/8c6pUqQLAN998Q+fOnbnqqquoU6cOX331Fffccw8ffvgh7777LgC33XYb+/bt48UXX+T9998v8bx5YyH79evHhAkT2LNnD3fccQdnnHEGK1eupG3btvmx+/fvp3///lx11VXcdNNNLFu2jLvuuovk5GRuv/32iK+1tDxL/MysGfAX4Fbn3P0hu94uFHo/sAa4KC/JM7OtwCJgEPBisOwcoAcw0Dk3L1i2BDgSeIhA8igiIuIby5cvZ/LkyVx66aX5Zeedd16Zj3fGGWfkj6s7++yzadSoEYMHD+all15iyJAhrFu3jpkzZzJ69GgeeOABAHr16kXz5s0ZMGAAs2bN4uqrr84/3o4dO/jXv/5FixYt8svGjBnDjh07WL16Ne3atQOgd+/e9OjRg82bN0dUz+OPP55Zs2blv69atSoXXnghK1eu5LTTTgPgmmuuyd/vnOOMM86gdevWdOnShc8++4wOHTpw1FFH0bhxY4ASu3Vzc3MZO3YsJ598Mq+++mp+y2C3bt046qijuOOOO/JbNAH27dvHPffck/99dO/enY8//pjnnnsupomfl129lwdf/1RcgJk1BU4BZoe27Dnn3gS2AOeHhJ8HZAPzQ+IcMBNoY2beDQQQERHxQMeOHbnvvvt46KGHWLVqFQcOHCjX8QYPHlzg/fnnn0/VqlVZtmwZ8Nvs2mHDhhWI69+/P8nJySxZsqRAeXp6eoGkL+8YHTp0yE/68gwdOjTievbrV3CU1/HHHw/Apk2b8su2bdvGddddR8uWLalZsybVqlWjS5cuAHzxxRcRnyvPF198wbZt2xg6dGh+0geBSSA9e/Yscu1VqlTh3HPPLVLP0DrGgpeJXxfgc2Cgma03swNm9r2Z3WtmeV29ed/6mjCf/3fI/rzYdYW6fgFWh+wXkXLQM5ZFKpcXX3yRQYMGMXXqVE444QQaNWrEtddey65du8p0vLzWrzxVq1alQYMG+eP38rqQC8fllRUe55eWllYkLisrK+znw8UWp0GDBgXe500Y2bdvHxBonevZsycvvvgiN954I4sWLeLDDz/klVdeAeDnn3+O+Fx5DnbtaWlpRa69du3aVK9evUBZjRo1EnqMX5Pg9ifgNmAtcBYwFmgGDAXyvrlwgxF2EhjPl6cB8GUxcXn7izCzXSXUM7mE/SK+oWcsi1QuqampTJ06lalTp7J582Zeeuklxo4dy549e5g1a1axiUbhJCXPtm3bCrzPyckhKysrP9HKe922bVuRBGjbtm1F1gAMbRnL06BBgyLnAdi6dWvxF1pK//73v1mzZg0zZ84s0A2enZ1d5mOGXnthW7duLZKMesXLFr8koA7we+fcn51zS51ztwNTgIvNrFVIrCvmGIXLi4sraZ+IiEhCa9asGTfccAOnnXYan332GRCYWVt4xunixYvZu3dv2GO88MILBd6//PLL5OTk0LVrVyAwng3g2WefLRD32muvkZ2dzVlnnVViPbt168Znn33G2rVrC5Q/99xzJX42UnkJZ+EWt6effrpIbF5rYUmtgK1bt6ZJkyY899xzBEaaBWzfvp233noromuvCF62+GUBRwP/LFT+D+BGAq15ef/kCJcm16dgS2DWQeIgfKshzrmUg1Uy2CKoVj8REalU8hKtiy++mDZt2nDooYfy3nvv8d577zF69GggMBbv9ttv5/bbb6dr166sW7eOxx57jOTk8H/2li9fzrXXXkv//v1Zv34948aNo0OHDlxwwQUAtGnThssvv5yHH34YM6NXr1588cUX3HbbbXTo0KFA61pxRo0axfTp0+nduzd33nknjRo1Yvbs2Xz5ZbhOvbJp27YtRx55JLfccgsAycnJzJkzh48//rhIbN5YwwceeICzzz6bKlWqcPLJJxeJS0pK4t577+XSSy9lwIABXHnllezdu5c77riDKlWqMGHChKjVvzy8TPz+DYSbIpPX7ptLoPsXAuPzFhWKaw/8K+T9WuB8M0sqNM6vffA13DhBERGRYn2//ntPn57x/frvaXJSkzJ9tmbNmpx66qnMmDGDTZs2ceDAAVq0aMGdd97JTTfdBMDNN9/Mnj17mDFjBg8++CCnnnoqc+fOpX///mGP+cwzzzBt2jQGDRpEbm4u5557LlOnTs1fygXgySef5Mgjj2T69OlMnTqV+vXrc9FFF3HPPfdQs2bNEuvduHFjli1bxvXXX88111xDrVq1OO+883jssceKrVdpVatWjb///e9cd911XHnlldSoUYO+ffvywgsvFEnqLrroIpYtW8ajjz7KxIkTcc4VaNELdckll3DooYdyzz33cMEFF1C9enW6dOnCiy++SOvWraNS9/Ky4iof8xOb/Y5A694w59xzIeUPATcALZ1zm8zsQwLdwqeELOfSHXgLGOKceyFYdi7wOjDAOTc/5HjvAI2cc23KWM9dycnJyWUdCBvvkpKSIh6zlZtbeN5M5TtvpFS/+DpvJHRPpCzyZlA2b968yL5Ro0axatWqCq5RUenp6UyZMsXrakicONjvLEBKSgrZ2dnZxfVoetbi55xbaGb/AP5sZqn8NrnjeuAJ51zefOYxBFr75pjZU/y2gPMK4G8hh1wALAGmmVkDAgs4Dwc6A9H5J4KIiPiGki1JRF4/q3cQMInAkzYaAt8ReDpH/oLOzrnFZtYnGPcGsIfAUzxuds4dCIlzZjYAuDu4pQDrCCzo/FoFXIuIiIhIXPM08XPO/Y/ARI4bS4hbCCyM4Hi7gZHBTURERERCeLmci4iIiIhUICV+IiIiIj6hxE9ERETEJ5T4iYiIiPiEEj8RERERn1DiJyIiIuITSvxEREREfMLrBZxFRETikh7ZJolIiZ+IiEgYq1at4tOVK2nXtKlndVizZYtn55bEpMRPRESkGO2aNuXV66737PwDHp3q2bnLav/+/dSoUaNIeW5uLgcOHKBatWoe1EryaIyfiIhIApo4cSJmxurVq+nfvz916tShYcOGjB07ltzcXD7++GO6du3KoYceytFHH83s2bOBQEunmTFz5swix3z22WcxM1asWAFAZmYmKSkpfPLJJ2RkZHDooYdy9dVXA2BmjBo1iqlTp9KqVSuqV6/O8uXLK+4GSFhK/ERERBLYhRdeyGmnnca8efMYMmQI9957LzfeeCNDhgxh2LBhzJs3j7Zt2zJ8+HBWrVpFeno6HTp0YMaMGUWONXPmTNq0aUPHjh3zy/bt28fAgQPp168fr732Gpdffnn+vr/97W8899xz3Hvvvbz++uscddRRFXHJchDq6hUREUlg1157LX/4wx8A6N69O6+//jqPPPII7777Lp07dwbg5JNPplGjRsyZM4f09HSGDx/O6NGj2bhxIy1atABg8+bNLF68mHvuuafA8ffv38/kyZMZOnRokXPv27ePt956i7p168b2IiViavETERFJYOeee27+z2ZGmzZtqFOnTn7SB1C/fn0aNWrEpk2bABg2bBhVq1Yt0N07e/ZszIxLLrmkyDnOO++8sOfu3r27kr44o8RPREQkgdWvX7/A++rVqxcpyyvft28fAA0bNqR3797MmjUL5xwQ6OY9++yzSUtLK/C5unXrUqtWrbDnLhwr3lPiJyIiIkVkZmby7bff8s477/D+++/z5ZdfkpmZWSTOzIo9xsH2iTc0xk9ERESK6NOnD6mpqcyYMYMaNWpQv359+vXr53W1pJyU+ImIiEgR1apV4+KLL2b69OlUrVqVoUOHhl2fTyoXJX4iIiLFWLNli6eLKK/ZsoUTPHxyyIgRI3j00UcBwnbzSuWjxE9ERCSM9PR0r6vACU2blrkeEydOZOLEiUXKX3311bDxGzduLFKWnp6eP7kjnHBr/eU52OfEO0r8RCTqqpgjKankuWPNmzVlw6bNFVAjkdKbMmWK11UQiTolfiISdTm54BZPLjEuqfu4CqiNiIjk0XIuIiIiIj6hxE9ERETEJ5T4iYiIiPiEEj8RERERn1DiJyIiIuITSvxEREREfEKJn4iIiIhPKPETERER8Qkt4CwiIhLGqFGjWLVqldfVID09XU8RkahR4iciIhLGqlWr+PijTzim5bGe1eHLDevK9fnnn3+eu+66i2+//Zb9+/ezYcMGWrRoUebjbdy4kZYtW/LMM8+QmZlZrrqJN5T4iYiIFOOYlsfyl7uf9+z8/3frxWX+7A8//EBmZiZ9+/bliSeeoHr16qSlpUWxdlIZKfETERFJQF999RW//vorQ4cOpUuXLl5XR+KEJneIiIgkmMzMTDp37gzA+eefj5mRkZEBwNNPP80xxxxDjRo1aNu2LbNmzSIzM7NIF/CWLVsYNGgQderUITk5mQsuuICtW7dW8JVItKnFT+JaFXMkJZX875PmzZqyYdPmCqiRiEj8u+222zj11FO59tprue++++jSpQt169blqaee4uqrr2bQoEE88sgjZGdnM2nSJPbt24eZ5X/+p59+onv37mzfvp17772XVq1a8frrrzN48GAPr0qiQYmfxLWcXHCLJ5cYl9R9XAXURkSkcjjqqKM49tjApJRjjjmGTp06ceDAAc466yzOOOMM5s6dmx/buXNnWrVqRZMmTfLLZs6cyfr163njjTc455xzADj77LP56aefmD59esVejESVunpFRER8YP369Wzfvp0LL7ywQPkRRxzBGWecUaBs6dKlpKSk5Cd9eYYNGxbzekpsKfETERHxgZ07dwJw2GGHFdlXuCwrK4vGjRsXidOs4MpPiZ+IiIgPNGjQAIDt27cX2Ve4rEGDBmzbtq1InCZ3VH5K/ERERHygdevWNG7cuMD4PoDvvvuO5cuXFyjr1q0bu3btYsGCBQXKn3322ZjXU2JLiZ+IiIgPJCUlMWnSJJYvX86FF17IggULeP755+nZsydpaWkFVlC49NJLOeaYYxg2bBiPP/44ixYt4rrrruOtt97y8AokGjSrV0REpBhfblhXrqdnROP8JzU4MWrHu+qqqzAz7r//fubPn0+LFi245ZZbmD9/Pps2bcqPq1WrFosXL+b6669nzJgxmBm9evXihRde4PTTT49afaTieZb4mVkGsKSY3W2dc1+ExPYE7gQ6AHuAecAY59yuQsesDdwNDAJSgLXAHc65v0e39rHRsnkzNm3eUmKc1qwTEYm99PR0r6vASQ1OLHM9MjIycM4VKb/yyiu58sor89/v2rWLMWPG0L9//wJxTZs25aWXXiry+XDHlMojHlr8xgDvFCrbmPdDMEFcALwKjAeaAPcB7czsTOdcbsjn5gEnAjcDG4BMYJ6Z9XXOFRyoEIc2bd5C7ttas05EJB5MmTLF6ypE3ZYtW3jggQfIyMigXr16bNq0iUceeYQ9e/YwatQor6snFSAeEr8vnXMfHGT//cAa4KK8JM/MtgKLCLTsvRgsOwfoAQx0zs0Lli0BjgQeIpA8ioiI+FbNmjX5+uuveeGFF9i5cye1atXitNNO48knn+S4447zunpSAeIh8SuWmTUFTgFGh7bsOefeNLMtwPkEEz/gPCAbmB8S58xsJvCUmR3rnFtXcbUXERGJLw0aNOD111/3uhrioXiY1fukmeWYWbaZvW5mJ4Xsaxd8XRPmc/8O2Z8Xu65Q1y/A6kLHEhEREfElL1v8soEpwFJgJ9AWuAVYbmZdnXMrgAbB2J1hPr+TwHi+PA2AL4uJy9tfhJntKqGeySXsFxEREakUPEv8nHOfAp+GFL1rZn8n0Lo3mcB4vfzw4g5TwvtI94mIiA8lJSXx66+/el0NkYgdOHCAatWqlfnzcTXGzzm3zcwWAf2CRVnB13CtdfUp2BKYdZA4CN9qiHMu5WB1CrYIqtVPRCQB1axZk71797Jz507q169f8gdEPLRz5072799PnTp1ynyMuEr8gpL4rXVubfC1HYFZvKHaA/8Keb8WON/MkgqN82sffA03TlAkrkS6lqPW0RKJjtTUVPbv38/27dvZtWsXVapU8bpKImEdOHAgP+lLTU0t83HiKvEzs8ZAT+ADAOfc92b2ETDUzKaELOfSHWgKvBLy8XnA5UBfQmb2ApcC6zWjVyqDSNdytLNurYDaiCQ+M6Np06bs2LGDffv2kZtbeH6gSHyoVq1aftJnZmU+jpdP7ngO+Bb4BPgv0IbAYs6HAGNDQscQaO2bY2ZP8dsCziuAv4XELSDwJJBpZtaAwALOw4HOQMHlyEVERILMjIYNG3pdDZEK4WWL37+BwcAfgEMJjNFbCtzlnMvvlnXOLTazPsAk4A0Cj2x7FbjZOXcgJM6Z2QACj2y7m8Aj29YRWND5tdhfjoiIiEh883JW773AvRHGLgQWRhC3GxgZ3EREREQkRDws4CwiIiIiFUCJn4iIiIhPKPETERER8QklfiIiIiI+ocRPRERExCeU+ImIVEItmzcjKSmpxK1l82ZeV1VE4khcPblDREQiE+lTXpK6j6uA2ohIZaEWPxERERGfUOInIiIi4hNK/ERERER8QomfSDlUMacB9jEW6SQG55zXVRURiXua3CFSDjm54BZrgH0sRTqJwc66tQJqIyJSuanFT0RERMQnlPiJiIiI+IQSP5EEpjGIIiISSmP8RCpAXgJWkmhPUNAYRBERCaXET6QCRJqAaYKCiIjEkrp6RURERHxCiZ+IiIiIT6irVxJCpGPoqldN4pec3BLjtBiwiIgkIiV+khBKM4bOLb47ojgREZFEo65eEREREZ9Q4iciIiLiE0r8RERERHxCiZ+ISBgtmzeL6KknmggkIpWJJneIiGdPFolnmzZvIfdtLbotIolFiZ9EJNLEoHmzpmzYtLkCaiTRpCeLiIj4gxI/iYie+SpSMVo2b8amzVtKjPNT66uIRI8SPxHxTCQtyX5LcNTFLCKxpMRPRDwTSUtyoiQ4GkcpIvFAiV+C8qq7SH/cRMLTOEoRiQdK/BKUV91F+uMmIiISv7SOn4iIiIhPlCrxM7NvzazfQfb3MbNvy18tEZHYyBuOoIWZRcSPStvV2wKofZD9hwLNy1wbEZEY03AEEfGzaHf1Hgb8FOVjioiIiEgUlNjiZ2ZdgIyQooFm1ipMaH1gMLAqKjUTERERkaiKpKu3GzAh+LMDBga3cL4GbohCvUREREQkyiJJ/KYAMwADvgVGAfMLxThgr3NuZxTrJiIiIiJRVGLi55zLBrIBzKwb8Llz7odYV0xEREREoqtUs3qdc8tiVRERERERia1SP7nDzI4ArgaOBhoQ6AIO5Zxz3aNQNymGHmwvIiIiZVGqxM/MegPzgOrAHkBj+jzgpwfbi0j5RPr87ObNmrJh0+YKqJGIeKm0LX73ADuAAc65j2JQHxERiaJIF6xO6j6uAmojIl4r7QLObYApsUr6zGyimTkzWxVmX08z+8DMfjazH8zsSTNLCRNX28weNbOtwdiPDvaYORERERG/KG3i9yPwSywqYmbHAWOA7WH2ZQALgM1AX+BGoB/whpkVvoZ5wFBgPHAusA6YZ2bnxKLeIiIiIpVFabt6ZwPnA49GsxLB5G0a8FegPZBSKOR+YA1wkXMuN/iZrcAiYBDwYrDsHKAHMNA5Ny9YtgQ4EniIQPIoIiIi4kulbfGbAVQ3s/lmdpaZtTSzIwpvZajHDcDhQJFBJmbWFDgFmJ2X9AE4594EthBIRPOcR2DNwfkhcQ6YCbQxs2PLUDcRERGRhFDaFr8vCDylw4A+B4mrEukBzexI4A5gqHNut1nh1WFoF3xdE+bj/w7Znxe7LjRBDFoduj/SuomIiIgkktImfncQSPyiwgJZ3tPAP51zrxYT1iD4Gm7pmJ3AiYVivywmLvRYoXXYVUI1k0vYLyIiIlIplPbJHROjfP4rgZOBSLpgi0s4C5cfLDHVqsYiIiLiW6V+cke0mFkqgUkb9wD/C1mapSpQJfh+H5AVLC/SWgfUp2BLYNZB4iBMq6FzLqVwWaF67kKtfiISbywpsrX3iix8ICJ+Vtond3SJJM45904EYYcTSKjuCW6F/Re4D3gs+L4dgVm8odoD/wp5vxY438ySCo3zax98DTdOUESk8nG5PPTUtBLD/njV5RVQGRGpLErb4reUyLpLI5nc8TXQLUz5FKA2cAXwnXPuezP7CBhqZlNClnPpDjQFXgn57DzgcgJr/c0PKb8UWO+c08QOERER8a3SJn4jijnGUUAmsBF4MpIDOef2EkgkC8ibbOGcC903hkBr3xwzewpoQqA1cAXwt5C4BcASYJqZNQA2AMOBzkD/SOolIiIikqhKO7ljZnH7zOwB4JNy1yj8eRebWR9gEvAGsAd4FbjZOXcgJM6Z2QDg7uCWQmD5loHOuddiUTcRERGRyiJqkzucc/81s78CNxNYMLmsx8kopnwhsDCCz+8GRgY3EREREQmK9nSv/xJ4PJqIiIiIxJmoJX5mVhO4BNgWrWOKiIiISPSUdjmX6cXsqg+cBjQEbipvpURiRmufiYiIj5V2jF9mMeU7CTwq7Qbn3PPlqpFILGntMxER8bHSzupVM4iIiIhIJaVETkRERMQnyrSci5nVBXrw2wzeb4E3nXN7olUxEREREYmuUid+ZnYF8BCBx6pZsNgBe83sj865kgdQiYiIiEiFK+2s3n7AUwRa+G4H1gR3HQf8AXjKzH7QUzJEfE6zp0VE4lJpW/xuBj4HOgaftZvnbTN7BviAwHN1lfiJ+JlmT4uIxKXS/nO7AzCjUNIHQHB838xgjIiIiIjEmbL0s9hB9rmyVkREREREYqu0id9nwHAzO7TwDjOrTWCB58+iUC8RERERibLSjvF7EHgF+MTMHgXWBcvzJne0AgZGr3oiIiIiEi2lfXLHq2Y2ErgP+BO/de0a8D9gpHNufnSrKCIiIiLRUOp1/Jxzj5vZ80BPoCWBpO8bAgs4Z0e5fiIiIiISJWV6codzbhfwt+hWRURERERiqcTJHWZWxczuNbNrSoj7PzO728wONutXRERERDwSSYvfMOAm4NQS4lYCjxF4msfz5ayXxBs9iUFERKTSiyTxuxB4yzn38cGCnHMfm9k/gSEo8as8Ik3onNOTGMpDibOIiMSBSBK/k4CHIjzeEuCPZa+OVDg9Wqti6D6LiEgciKR5oT7wQ4TH+zEYLyIiIiJxJpLEbw+QGuHxGgBFnuMrIiIiIt6LJPFbC/SK8Hg9g/EiIiIiEmciSfxeAXqYWf+DBZlZPwKJ38vRqJiIiIiIRFckid+TwNfAXDObbGYtQneaWQszuwuYC3wZjBcRERGROFPirF7n3M9mdi7wOjAWuMXM9gC7gTpAXQKPbVsP9HHO7YthfUVERESkjCJaNMw59zWQDlwPvAfkAI2BA8C7wfITnXPfxKaaIiIiIlJeET+rN9iS96fgJiISX6K9SLYW3RaRBBRx4iciEtciXST76iv1tBoR8S0lfiLiL9F+iopaBkWkElHiJyJSHnocn4hUIkr8ROKJWo9ERCSGlPhJYog0YYp38d4NmSj3WUTEp5T4SWJQd1t40b4vus8iIpWa+otEREREfEItfhJdEXctVtFYNhERkQqmxE+iqxRdgeoyFBERqVhK/EQqI02ykEhF+LvisAqojIh4TYmfSGWkSRYSKf2uiEgIJX5xxGEa9yYiIiIxo8Qvnuhf5uI3kXRD6h86IiJRo8RPJBw9QaNiRPCPHf1DR0QkejxL/MzsdGAC0A5oAOwB/g084Jz7R6HYnsCdQIdg3DxgjHNuV6G42sDdwCAgBVgL3OGc+3ssr0USkFpfRUQkAXnZ4lcPWA88A2wLvr8KWGBmQ5xzLwCYWQawAHgVGA80Ae4D2pnZmc653JBjzgNOBG4GNgCZwDwz6+ucWxD7SxIRKSfN2BaRGPIs8XPOvQG8EVpmZq8RSNiuAl4IFt8PrAEuykvyzGwrsIhAy96LwbJzgB7AQOfcvGDZEuBI4CECyaOISHxTa7OIxFBcjfFzzuWYWTbwK4CZNQVOAUaHtuw55940sy3A+QQTP+A8IBuYHxLnzGwm8JSZHeucW1dBlyJSuaiVSUTEFzxP/MwsicAzgxsBVwPHADcGd7cLvq4J89F/h+zPi11XqOsXYHXo/mjUWSThqJUp9pRci0gc8DzxA+YSaLkD2A1c6JxbGHzfIPi6M8zndhIYz0dI7JfFxIUeqwAz21VC/ZJL2C8iUjIl1yISB+JhLYqbgVOBfgTG4c01syGFYlwxny1cXlxcSftEREREEp7nLX7OuW+Bb4NvXwtO8Pizmb0IZAXLw7XW1adgS2DWQeIgfKshzrmUg9Uv2CKoVj8JT913sad7LCISNZ4nfmGsBPoADQmswweB8XmLCsW1B/4V8n4tcL6ZJRUa59c++BpunKDEu3j/o6/uu9jTPRYRiZq4SvzMzIAMYBeQFZzl+xEw1MymhCzn0h1oCrwS8vF5wOVAX0Jm9gKXAus1o7eS0h99ERGRqPHyyR3PAZuAj4EdQBowHDgL+INzLicYOoZAa98cM3uK3xZwXgH8LeSQC4AlwDQza0BgPcDhQGegf8wvSERERCTOedni9z4wlMASLskE1uD7COjnnHstL8g5t9jM+gCTCCz4vIfAUzxuds4dCIlzZjaAwCPb7ibwyLZ1BBZ0zj+eiIiIiF95+eSOx4DHIoxdCCyMIG43MDK4iYiIiEiIeFjORUREREQqgBI/EREREZ9Q4iciIiLiE0r8RERERHxCiZ+IiIiIT8TVAs4SoUieZmHK6UVERKQgJX6VUQRPs9CTLERERKQwJX6JKt6fcSsiIiIVTolfotIzbkVERKQQDQQTERER8QklfiIiEhgeklTydkTzFl7XVETKQV29IiISGB6yaH2JYaPPblMBlRGRWFGLn4iIiIhPKPETERER8QklfiIiIiI+ocRPRERExCeU+ImIiIj4hBI/EREREZ9Q4iciIiLiE0r8RERERHxCiZ+IiIiITyjxExEREfEJJX4iIiIiPqHET0RERMQnlPiJiIiI+IQSPxERERGfUOInIiIi4hNK/ERERER8QomfiIiIiE8o8RMRERHxCSV+IiIiIj6hxE9ERETEJ6p6XQEpm2XLlnpdBREREalklPhVUie263jQ/fOZXUE1ERERkcpCXb0iIiIiPqHET0RERMQnlPiJiIiI+IQSPxERERGfUOInIiIi4hNK/ERERER8QomfiIiIiE8o8RMRERHxCSV+IiIiIj7hWeJnZt3NbIaZrTezn8zsezN7xczah4ntaWYfmNnPZvaDmT1pZilh4mqb2aNmtjUY+5GZ9auQCxIRERGJc162+F0DHAE8AvQG/hh8/6GZdcoLMrMMYAGwGegL3Aj0A94ws8L1nwcMBcYD5wLrgHlmdk4sL0RERESkMvDyWb3XOud+CC0ws0XABuAm4Pxg8f3AGuAi51xuMG4rsAgYBLwYLDsH6AEMdM7NC5YtAY4EHiKQPIqIiIj4lmctfoWTvmDZLuAr4HAAM2sKnALMzkv6gnFvAlv4LTkEOA/IBuaHxDlgJtDGzI6N/lWIiIiIVB5xNbnDzBoC7Qi08BH8mZD3of4dsj8vdl1oghi0utCxRERERHzJy67eAszMgKcIJKMPBosbBF93hvnITuDEkPcNgC+LiQs9VuHz7iqhaskl7BcRERGpFOIm8QMeAAYAI5xznxfa54r5TOHy4uJK2icRWLZsqddVEBERkXKIi8TPzCYDo4HrnXMzQnZlBV/DtdbVp2BLYNZB4iB8qyHOuZQS6rYLtfoBcGK7jiXGzGd2BdREREREysLzMX5mdgdwK3Czc+7RQrvXBl/Djc9rT8Gxf2uBtmGWeMlbFzDcOEERERER3/A08TOzCcBtwG3OuQcK73fOfQ98BAwNTejMrDvQFHglJHwekEJgrb9QlwLrnXProlt7ERERkcrFs65eMxsNTAReB94KXbQZ2O+c+zT48xgCa/bNMbOngCbAfcAK4G8hn1kALAGmmVkDAusBDgc6A/1jeCkiIiIilYKXY/zyWub6BLdQm4AWAM65xWbWB5gEvAHsAV4l0DV8IO8DzjlnZgOAu4NbCoEndwx0zr0Wq4sQERERqSw8S/yccxmliF0ILIwgbjcwMriJiIiISIi4mNUr3tIyLSIiIv6gxE84+qSjSw56Lvb1EBERkdjyfDkXERER8d4RLY4gKSmpxO2IFkd4XVUpB7X4iYhIxJxzJCWV3GbQrNkRbNq0MfYVkqj5/rvveeyTx0qMG3mihtFXZkr8RESkVD6Y/3WJMZ36t6qAmohIaSnxExEpJ02QEpHKQomfxD39UZV4p+dYi0hlocRP4p5mHUuk9I8EEZGDU+InIglDLW+SCI5ocQTff/d9iXGHH3E43238rgJqJIlEiZ+IeEqtdCIFaXatxJISv0rqkzUrvK5CpeVVoqEEJ7wSu/LVjS8iEjVK/CqpI9q29LoKFWZjlNcC86o7UGMVReKHulPFr5T4SdxLa3i4J+eNdgtdtBNYESk7dafGDyXhFUuJn0gxot0y6FUCGyl1RcePSL8LfWeVT/PmLdi8+eDJi3OugmoTH5SEVywlfiIJLtLkINqJrpKSsot0WIBmMVc+mzd/V+KTTzr2Oyqq54wk2QT/JZx+pcRPfMdvE2O8GlsY6XkToQs83n+nop2ER/I4NmfRTSIiTV70jOCiIkk2IfoJp8QnJX4S9T+88d7S46eJMZVBvHeBRyLef6ei3TI4ds6dJcbcPXh8xMeLRKTJi54RLHJwSvwk6n94NXtVIDFa8kT8ZtKkSSXGqEu4clPiJ1JO0W7hjPcW00h50ZIX712uXkmE++LMRbWL2TkXcZKTlJRUYlyidDFfMfi6EmPufjW6rbml+S6k/JT4iZSTWjjjR7x3uUY7qY80oYvn+1K1ejUO/JoTUezYF6LbxRxpkqMu5tjzIuH0KyV+InEm2n+k1eUaP6L9j4R4TugideDXHG594a4S40qT0EXSQhdtziJrGYz2pBev+KkVNNEo8RPPJEqXplcivX+JMHlCpDQ8mcGaC499WvJadNeecG10z+sRtYJWXkr8xDNag6x8dP9EwkhKjIQjmuMPvRobZyR50voqB6fET0Skgviq2z3Jmy5XcqM/FtAL0Rx/6NX6fLnuACv+/k2JcaWpn7qYy0+Jn4hIBfFVt7vPuj4jFs0WyUiPlUCNbupiLj8lfiICJMaSH4nCVy2DfhNBi2TErZEJ0ropFUuJn0glFe1ELdIZopqUE3u+ahkUkQqlxE88oxam8vFqKQ8v1i1UsimeSYTJItG+Bh92MScSJX7imURYg0xEElwidKdG+xoS4Z74mBI/kWJE2iKp8VixF+1/JCyNsAVRLY0icSTClsZEWSQ7VpT4ScKI9h/pSJMNjceqfJpH+N3qcXwicUQtjVGhxE8Shv5IS7SpNVekctJ6f8VT4iciUgy/tebecUfJT4sQqQy03l/xlPhJxDQLVySxXR7B0yIeeu/5CqiJJIREmBGdgJT4ScQ0C1cksU174VGvqyCJRGPy4pISP0kYGo8lUj6dzz+rxJjX1eJXrEmT1FUu8U+JXwLz21IUfhuP5RUl2BIpvyVCV0TQVX73q2rdEm8p8UtgmuUqsaAEWyKlREg8ofX+DkqJn4iIiCQOjS08KD1JT0RERMQnlPiJiIiI+IS6eiXqNPhfREQkPinxk6jT4H8REZH4pK5eEREREZ/wNPEzs8PNbKqZvWdme83MmVlGMbE9zewDM/vZzH4wsyfNLCVMXG0ze9TMtgZjPzKzfjG+FBEREZG453VXbytgCPAJ8DYQNkELJoMLgFeB8UAT4D6gnZmd6ZzLDQmfB5wI3AxsADKBeWbW1zm3IBYXISLiG5bE3UNuiyhOROKP14nfO865RgBmNoBiEj/gfmANcFFekmdmW4FFwCDgxWDZOUAPYKBzbl6wbAlwJPAQgeRRRETKyuXS55G/lhj2+g1XVEBlRKS0PP0nWaGWurDMrClwCjA7NN459yawBTg/JPw8IBuYHxLngJlAGzM7NkpVFxEREal0KkNbfLvg65ow+/4dsj8vdl2YhHJ1oWOJiIiI+I7XXb2RaBB83Rlm304C4/lCY78sJi70WPnMbFcJ508uYb+IiIhIpVAZWvzyFPc05cLlB3vqsj+fyCwiIiJC5Wjxywq+FmmtA+pTsCUw6yBxEKbV0DmXcrCTB1sE1eonIiKSYJKSSm7/atbsCDYl0BOpKkOL39rga7jxee0pOPZvLdDWrMg6Au2Dr+HGCYqIiIjfRJgBfff9ptjWo4LFfYufc+57M/sIGGpmU0KWc+kONAVeCQmfB1wO9CVkZi9wKbDeObeugqotIiIi8SwXxr5wZ4lhdw8eXwGVqTieJ35mdkHwx1OCr13NLBX4n3PuH8GyMQTW7JtjZk/x2wLOK4C/hRxuAbAEmGZmDQgs4Dwc6Az0j+mFiIiIiMQ5zxM/CiZuABODr5uAFgDOucVm1geYBLwB7CHwFI+bnXMH8j7onHPBhaDvDm4pwDoCCzq/FqsLEBEREakMPE/8nHMWYdxCYGEEcbuBkcFNRERERII8T/wkdjYm0CwkERERKT8lfgksreHhXldBRERE4khlWM5FRERERKJAiZ+IiIiITyjxExEREfEJJX4iIiIiPqHET0RERMQnlPiJiIiI+ISWc4kzy5Yt9boKIiIikqCU+MWZE9t1LDFmPrMroCYiIiKSaJT4xZlP1qzwugoiIiKSoJT4xZkj2rb0ugoVZuuP33tdBREREV9R4ieeaVInxesqHJQSUxERSTRK/ESKEe+JqYiISGkp8RORuKfWVxGR6FDiJyJxL95bX5WYikhlocQvgemPUeWk763svLp3XiWmXl1vpKsP/PWFR6N63mgfT8SPlPglsEj/GCnRiC/x3roVz/x277y63pOatigx5j/A1admlBh356tzIz5vtI8n8UNJfcVR4ie++2OZKJSwi0iiUFJfcZT4iVRSSthF4kukrVaRxHnVTR7tOIk/SvwqKS9aexKlhSlRriPaIr0vfrp/0b5WP907L3mVvETaalVSXCQxeXGRilbdShsn8UeJXyXlRWtPorQwJcp1RCrSZCPS++KnsaPR/l3x6t4lwndRGtFOXtS6VXa6d/FHiZ9IgvMq0fVbgh1NXiWcEp5at8pO9y7+KPGThOG3Vg0REZHSUuIncS/aXZXRprFxlY++CxHxKyV+EvfivZsq2mPjJPb0XYQX6cLMIl7RmMHyU+IXZxKhJSIRrkEqjn5f4kekCzNHm/6Yl53f7p0nYwaTICkpqcSww484nO82fhfdc8eAEr84kwgtEYlwDVJx9PsimgBQdrp3FSAXHvv0sRLDRp44sgIqU35K/ERERCSh+K0ltDSU+ImIiEhCUUto8UrutBYRERGRhKDET0RERMQnlPiJiIiI+IQSPxERERGfUOInIiIi4hNK/ERERER8QomfiIiIiE8o8RMRERHxCS3gLCIiInIQkyZNKjHGOVcBNSk/JX4iIiLiS5E+2u2KwdeVGHP3q+PLW50KocRPREREfMmPj3bTGD8RERERn1DiJyIiIuITCZf4mVltM3vUzLaa2c9m9pGZ9fO6XiIiIiJeS7jED5gHDAXGA+cC64B5ZnaOp7USERER8VhCTe4IJnc9gIHOuXnBsiXAkcBDwAIPqyciIiLiqURr8TsPyAbm5xW4wMI6M4E2ZnasVxUTERGRBJYESUlJJW5HtDjC02omVIsf0A5Y55zLLVS+OnR/xVZJREREKrOI1vvLhcc+fazEsJEnjoxCjcrOKstK05Ewsy+BL51zfQqVHw18CfzeOfeXQvt2lXDYZIDk5OQo1jS87OxsqteoUWLcL/v3lxgXSYziFFcZ4uK5bokWV7VGzRLjcvbviziuRs1qJcbt3/er4uLgnIqLTtwhtQ8pMe7nvT/HNKfIzs6GQIdn2F7dREz81jvn+hYqz0v8/s8590ShfbtKOGwy4IDdUaxqceeBQFe1VDzdf2/p/ntL9997+g68lUj3vy6Q65wL26ubaF29WUCDMOX1g687C+9wzqXEskKRyktA46U+fqP77y3df2/p/ntP34G3/HT/E21yx1qgrZkVvq72wdc1FVwfERERkbiRaInfPCAF6Fuo/FICXcCa2CEiIiK+lWhdvQuAJcA0M2sAbACGA52B/l5WTERERMRrCZX4OeecmQ0A7g5uKQSWbxnonHvNw6qJiIiIeC6hZvVWZn4aWBqPdP+9pfvvLd1/7+k78Jaf7n+ijfETERERkWIo8RMRERHxCXX1ioiIiPiEWvxEREREfEKJn4iIiIhPKPETERER8Qklfh4ys9pm9qiZbTWzn83sIzPr53W9KjszO9zMpprZe2a218ycmWUUE9vTzD4I3v8fzOxJM0sJE6fvKkJm1t3MZpjZejP7ycy+N7NXzKx9mFjd/ygzs9PN7J9mtsXM9pnZj2a22Mx6h4nV/a8AZjYx+P+hVWH26TuIIjPLCN7rcFubQrG+vPdK/Lw1DxgKjAfOJbDY9DwzO8fTWlV+rYAhwF7g7eKCgsngAmAzgcf83Qj0A94I87xnfVeRuwY4AngE6A38Mfj+QzPrlBek+x8z9YD1wGjgd8BVwH5ggZkNzgvS/a8YZnYcMAbYHmZfBvoOYmUMcFqhbWPeTl/fe+ecNg824BzAAeeFlBnwHvC51/WrzBuQFPLzgOB9zggTtxL4tFB8z2D8Rfquynz/G4UpSwH+C7ys++/Jd1KVwB+4xbr/FXrfk4APgD8BS4FVhfbrO4j+Pc8I3qsBJcT59t6rxc875wHZwPy8Ahf4jZoJtDGzY72qWGXnnMstKcbMmgKnALND451zbwJbgPNDwvVdlYJz7ocwZbuAr4DDQfe/ojnncgjcw19B978C3UDgd35c4R36Drzj93uvxM877YB1YZKU1SH7JXby7u+aMPv+TcH7r++qnMysIYH7lHe/df9jzMySzKyqmTUxs0nAMQS630H3P+bM7EjgDmCkc253mBB9B7H1pJnlmFm2mb1uZieF7PP1vVfi550GwM4w5TtD9kvs5N3f4r6DBoVi9V2VkZkZ8BSB/988GCzW/Y+9uQRa+LYAo4ALnXMLg/t0/2Mo+Dv/NPBP59yrxYTpO4iNbGAKgbGt3YCbgGOB5WbWMRjj63tf1esK+NzBHpuiR6pUjOLuc+FyfVdl9wCBsZYjnHOfF9qn+x87NwP3AY2Bi4G5ZjbcOTcnJEb3PzauBE4mkHCURN9BFDnnPiUwdi/Pu2b2dwKte5OBHqHhxR2mhPeR7otLSvy8k0X4fynUD76G+xeGRE9W8LW472BnoVh9V2VgZpMJzC693jk3I2SX7n+MOee+Bb4Nvn3NzF4D/mxmL6L7HzNmlgrcD9wD/C9keZCqQJXg+33oO6gwzrltZraIwKxd8Pm9V1evd9YCbcNMG89b6yzc2AOJnrXB13DjM9pT8P7ruyoDM7sDuBW42Tn3aKHduv8VbyWBpV4aovsfS4cDyQQSv/+GbGcQuN//BSai76CiJfFb65yv770SP+/MI7DERd9C5ZcC651z6yq8Rj7inPse+AgYGvoftJl1B5oCr4SE67sqJTObANwG3Oace6Dwft3/ihUcc5YB7AKydP9j6msCY8sKb58B3wR/fkrfQcUxs8YElmr5APT/H8/Xk/HrRmAdoMXADuAyAv8zmAHkAn29rl9l34ALgtt9BP6VNyH4vndIzFlADvAi0B24BPgPgf85VNF3VeZ7Pzp4z18DOhXaTtD9j/n9fw64m8CSFF2BwcA/gt/JSN1/z76XpRRdx0/fQfTv83PAnQSWYckgsKD8BuAn4GTde6fEz9ObD3WBx4BtBMZ8fEIJi05qi/jeumK2jYXifgesCN7/HwnMxKun76pc936p7r+n938k8D6BsUk5wdd/hvsjpftfod9LkcRP30FM7vMtwCoCrdu/Bu/XC0A73fvAZsGLEhEREZEEpzF+IiIiIj6hxE9ERETEJ5T4iYiIiPiEEj8RERERn1DiJyIiIuITSvxEREREfEKJn4hICDPrZmYfmNkeM3NmlmlmGXk/x/C8mcFzZMTqHLFgZhOD9W7hdV1EpGRK/EREgsysHoHHNR1K4AkklwDveFqpOBBMfCeaWYrXdRGR8qnqdQVEROLIKQSey3m5cy7/eZ1mthE4hMCTAPwog8BjD2cQeCKCiFRSavETkYRkZnXK8LHGwdedoYXOuVzn3D7n3IHy10xExDtK/EQkLoWMeesR7GbcZGb7zWy1mQ0uFLvRzJaa2Qlm9k8zywZWh+w/2sxmm9lWM/slGP+AmR0aegxgZvDtkuC5XXBfkTF+ZvaimR0oPCbPzM42s1wzmxWl+1DDzG41s7Vmts/MdpnZa2Z2QqG4/Dqa2Yhg/P7gfbu5mGP/n5mtDx73SzMbWXisoZnNINDaB7Ah776Y2cRCh6thZneb2ffB835mZudE4x6ISPSoq1dE4t19BMbc/QVwwAhgjpnVdM7NCIk7AlgM/A14GagNYGYnBct3AU8CW4AOwHXAGWbW1Tn3KzAK6A1cBdwNfF5Cva4i0DX8rJmlO+d2mFljYBbwNfD7cl11oO7VgIXA6cBsAg+KTwauBJabWRfn3EeFPnYNcBgwjcA1DwPuM7PvnXPPhxx7DHAvgQfO3wrUAm4i8LD6UE8SeEj9ecANwI5g+epCcTMJdIU/CFQncD9fNbNjnHMbS3/1IhITzjlt2rRpi7sNyCSQ6G0CkkPKk4NlO4FDgmUbg7FXhDnOZ8AXQJ1C5ecFP5MZ5pwZhWIzCscGyzsCvwCvEehBeRPYD5xUjuvNCCm7IVh2dqHYusB3wNIwdfwPkBJSXotAMvd+SFl94GcCyVvNkPLGQHaYekwMlrUIU++8fa8DFlJ+SrD8Hq9/l7Rp0/bbpq5eEYl3f3HOZee9Cf78BFCPQLKTZyfwTOgHzaw9cDzwPIGuyNS8DXgP+B/Qq6wVc86tAMYDfQjM/u0B3OKc+7isxyxkGIGk9eNCda9OIMnsbGaHFPrMM865XSF1/An4ADg6JKYnUJPAvd0XErsNeK6MdZ3qnHMhx/oQ2FPovCLiMXX1iki8C9flui74emRI2Teu6OSLtsHXScEtnMPKUTeABwgkfmcCi4Ap5TxeqLYEZhMX7n4NlQpsDnn/bZiYLKBByPuWwdf1YWLDlUUi3Hl3FjqviHhMiZ+IxDsXpszClP10kLiHCIyVC+e/ZalUiBYEWhUBWhEYW7innMfMY8C/gT8eJKZwUhjJzONw96+8ijtvLM4lImWkxE9E4t2xwN8LleW15IVrZQr1VfD1gHPurajWCjCzqsAcAv8vvQ6YSmASyrAoneIroCGw2DmXG6VjAmwIvrYmMPElVOsw8eGSbxGphDTGT0Ti3f+ZWXLem+DP1xCYsbqshM9+CqwBrjGzIwvvNLOqZla/HHW7i8AEj5HOuT8RaFkcambDy3HMULMITLgI2+JnZmXtps6bhPJ/ZlYz5HiNgaFh4vcGX8tzr0QkDqjFT0Ti3Q5ghZlNJ9BtOILA0i1XBCcuFMs558zsEgKtWquDx1hLYKZrK2AgMJbAEylKxcx6ADcDz7vflpW5FegKPGZm/3LOfVXc5yM0lcBEjAfM7KzgdewmcP3dgX1At9Ie1DmXZWaTCCxbs9zMniVwT64CvgROpmAr3wfB1/vM7Lngedc459aU6apExDNK/EQk3o0hMHFiJIGJGF8BQ13ImnQH45xbFVzseCzQj0Br4R4CS8DMAN4ubYXMrBGBdfW+DR4v71y/mtkQAi2Nc8zsdOfcL6U9fqHjnUtgTcBL+G2Cyn+Alfy24HRZjn2Pme0Griewnt93BCaqGIHE7+eQ2OXBdf+uAZ4m8LdjEoHWVBGpRCxk9r2ISNwIPiXjGaCbc26pt7XxDzP7E4Eku4lzbqvX9RGR6NIYPxERHwod2xdSlgZcSqAbV0mfSAJSV6+ISIwEF1dOLikuuHByRcswsweAV4DvCSxLcyWB5Whu8aA+IlIBlPiJiMTORRR6mkgxvFjr7mvgGwLJXgMCEzY+IvCItagvfSMi8UFj/EREYiTYdXpcSXFKtESkoijxExEREfEJTe4QERER8QklfiIiIiI+ocRPRERExCeU+ImIiIj4hBI/EREREZ/4f/BBm3RvzDSLAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 720x576 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"plt.subplots(figsize=(10, 8))\n",
"sns.histplot(data=prediction_df.assign(suite_cat_high=prediction_df.suite_category.str.split(\"_\").str[0]), x=\"prefix_length\", hue=\"suite_cat_high\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "0c72c1bf-bfa8-4a9c-9ffa-a349522ba4c9",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>num_prefix_sentences</th>\n",
" <th>prefix_length</th>\n",
" <th>prefix_length_bin</th>\n",
" <th>canonical_suite</th>\n",
" <th>suite_category</th>\n",
" <th>item_number</th>\n",
" <th>prediction</th>\n",
" <th>correct</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>1</td>\n",
" <td>prediction_0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>2</td>\n",
" <td>prediction_0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>3</td>\n",
" <td>prediction_0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>4</td>\n",
" <td>prediction_0</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>(-0.527, 26.35]</td>\n",
" <td>number_prep</td>\n",
" <td>number</td>\n",
" <td>5</td>\n",
" <td>prediction_0</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125599</th>\n",
" <td>31.0</td>\n",
" <td>494</td>\n",
" <td>(474.3, 500.65]</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>2452</td>\n",
" <td>prediction_2</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125600</th>\n",
" <td>31.0</td>\n",
" <td>502</td>\n",
" <td>(500.65, 527.0]</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>2453</td>\n",
" <td>prediction_2</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125601</th>\n",
" <td>31.0</td>\n",
" <td>497</td>\n",
" <td>(474.3, 500.65]</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>2454</td>\n",
" <td>prediction_2</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125602</th>\n",
" <td>31.0</td>\n",
" <td>501</td>\n",
" <td>(500.65, 527.0]</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>2455</td>\n",
" <td>prediction_2</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>125603</th>\n",
" <td>31.0</td>\n",
" <td>494</td>\n",
" <td>(474.3, 500.65]</td>\n",
" <td>fgd_object</td>\n",
" <td>fgd</td>\n",
" <td>2456</td>\n",
" <td>prediction_2</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>125604 rows × 8 columns</p>\n",
"</div>"
],
"text/plain": [
" num_prefix_sentences prefix_length prefix_length_bin canonical_suite \\\n",
"0 0.0 0 (-0.527, 26.35] number_prep \n",
"1 0.0 0 (-0.527, 26.35] number_prep \n",
"2 0.0 0 (-0.527, 26.35] number_prep \n",
"3 0.0 0 (-0.527, 26.35] number_prep \n",
"4 0.0 0 (-0.527, 26.35] number_prep \n",
"... ... ... ... ... \n",
"125599 31.0 494 (474.3, 500.65] fgd_object \n",
"125600 31.0 502 (500.65, 527.0] fgd_object \n",
"125601 31.0 497 (474.3, 500.65] fgd_object \n",
"125602 31.0 501 (500.65, 527.0] fgd_object \n",
"125603 31.0 494 (474.3, 500.65] fgd_object \n",
"\n",
" suite_category item_number prediction correct \n",
"0 number 1 prediction_0 True \n",
"1 number 2 prediction_0 True \n",
"2 number 3 prediction_0 False \n",
"3 number 4 prediction_0 True \n",
"4 number 5 prediction_0 False \n",
"... ... ... ... ... \n",
"125599 fgd 2452 prediction_2 NaN \n",
"125600 fgd 2453 prediction_2 NaN \n",
"125601 fgd 2454 prediction_2 NaN \n",
"125602 fgd 2455 prediction_2 NaN \n",
"125603 fgd 2456 prediction_2 NaN \n",
"\n",
"[125604 rows x 8 columns]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"prediction_df_melt = prediction_df.reset_index()\n",
"meta_vars = [\"num_prefix_sentences\", \"prefix_length\", \"prefix_length_bin\", \"canonical_suite\", \"suite_category\", \"item_number\"]\n",
"prediction_df_melt = prediction_df_melt.loc[:, prediction_df_melt.columns.str.startswith((\"prediction\",) + tuple(meta_vars))] \\\n",
" .melt(id_vars=meta_vars, var_name=\"prediction\", value_name=\"correct\")\n",
"prediction_df_melt"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "5e5364ca-37cf-4753-8e6d-ec411c5adf44",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Text(1873.3439052734377, 0.5, 'Prediction accuracy')"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
Raw
analyze.nb.html
View raw

(Sorry about that, but we can’t show files that are this big right now.)

Raw
analyze_repetitions.py
View raw

(Sorry about that, but we can’t show files that are this big right now.)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment