{
  "history": {
    "step": [
      50,
      100,
      150,
      200,
      250,
      300,
      350,
      400,
      450,
      500,
      550,
      600,
      650,
      700,
      750,
      800,
      850,
      900,
      950,
      1000,
      1050,
      1100,
      1150,
      1200,
      1250,
      1300,
      1350,
      1400,
      1450,
      1500,
      1550,
      1600,
      1650,
      1700,
      1750,
      1800,
      1850,
      1900,
      1950,
      2000,
      2050,
      2100,
      2150,
      2200,
      2250,
      2300,
      2350,
      2400,
      2450,
      2500,
      2550,
      2600,
      2650,
      2700,
      2750,
      2800,
      2850,
      2900,
      2950,
      3000,
      3050,
      3100,
      3150,
      3200,
      3250,
      3300,
      3350,
      3400,
      3450,
      3500,
      3550,
      3600,
      3650,
      3700,
      3750,
      3800,
      3850,
      3900,
      3950,
      4000,
      4050,
      4100,
      4150,
      4200,
      4250,
      4300,
      4350,
      4400,
      4450,
      4500,
      4550,
      4600,
      4650,
      4700,
      4750,
      4800,
      4850,
      4900,
      4950,
      5000,
      5050,
      5100,
      5150,
      5200,
      5250,
      5300,
      5350,
      5400,
      5450,
      5500,
      5550,
      5600,
      5650,
      5700,
      5750,
      5800,
      5850,
      5900,
      5950,
      6000,
      6050,
      6100,
      6150,
      6200,
      6250,
      6300,
      6350,
      6400,
      6450,
      6500,
      6550,
      6600,
      6650,
      6700,
      6750,
      6800,
      6850,
      6900,
      6950,
      7000,
      7050,
      7100,
      7150,
      7200,
      7250,
      7300,
      7350,
      7400,
      7450,
      7500,
      7550,
      7600,
      7650,
      7700,
      7750,
      7800
    ],
    "loss": [
      11.832879066467285,
      11.36864185333252,
      10.761643409729004,
      10.459257125854492,
      10.175871849060059,
      9.816800117492676,
      9.529030799865723,
      9.260773658752441,
      9.017507553100586,
      8.663485527038574,
      8.371795654296875,
      8.118546485900879,
      7.833754539489746,
      7.534058570861816,
      7.297992706298828,
      6.95162296295166,
      6.682138919830322,
      6.448472023010254,
      6.186150550842285,
      5.851781368255615,
      5.642083168029785,
      5.3920135498046875,
      5.093167781829834,
      4.796841621398926,
      4.534151554107666,
      4.4100446701049805,
      4.1128153800964355,
      3.8507931232452393,
      3.6341888904571533,
      3.4420320987701416,
      3.2039170265197754,
      3.1796298027038574,
      2.970296621322632,
      2.8707118034362793,
      2.7452502250671387,
      2.64337158203125,
      2.503178119659424,
      2.4706099033355713,
      2.370277166366577,
      2.268285036087036,
      2.318305492401123,
      2.2794229984283447,
      2.149564504623413,
      2.1994822025299072,
      2.2138619422912598,
      2.154664993286133,
      2.1253409385681152,
      2.104158401489258,
      2.0960659980773926,
      2.066467523574829,
      2.087989568710327,
      2.050243616104126,
      1.993398666381836,
      2.037946939468384,
      1.9801470041275024,
      2.0205399990081787,
      2.0452141761779785,
      1.938254475593567,
      1.964411973953247,
      1.9696029424667358,
      1.9631099700927734,
      1.9499931335449219,
      1.9534265995025635,
      1.930572271347046,
      1.9394501447677612,
      1.9909099340438843,
      1.8681749105453491,
      1.8672945499420166,
      1.9460028409957886,
      1.8758708238601685,
      1.8508617877960205,
      1.8957277536392212,
      1.89703369140625,
      1.8883601427078247,
      1.9217368364334106,
      1.8644115924835205,
      1.8522149324417114,
      1.8996679782867432,
      1.7724077701568604,
      1.8279880285263062,
      1.8713520765304565,
      1.7754944562911987,
      1.8203290700912476,
      1.8123414516448975,
      1.718186616897583,
      1.7727686166763306,
      1.7215014696121216,
      1.6721407175064087,
      1.7613623142242432,
      1.7179336547851562,
      1.773058533668518,
      1.7030664682388306,
      1.6822694540023804,
      1.708242654800415,
      1.6574541330337524,
      1.7228068113327026,
      1.6863057613372803,
      1.7814959287643433,
      1.6393948793411255,
      1.602840781211853,
      1.6753469705581665,
      1.6772491931915283,
      1.6624025106430054,
      1.6662027835845947,
      1.6071633100509644,
      1.6065205335617065,
      1.6259610652923584,
      1.6235558986663818,
      1.545785665512085,
      1.5571601390838623,
      1.6024692058563232,
      1.5312445163726807,
      1.5064446926116943,
      1.5606070756912231,
      1.5643519163131714,
      1.505636215209961,
      1.542928695678711,
      1.5629281997680664,
      1.5619080066680908,
      1.5639537572860718,
      1.4565213918685913,
      1.4847272634506226,
      1.502737045288086,
      1.4859784841537476,
      1.4928518533706665,
      1.4829928874969482,
      1.4601913690567017,
      1.531474232673645,
      1.4434651136398315,
      1.5020925998687744,
      1.47160804271698,
      1.4648197889328003,
      1.5810593366622925,
      1.4178014993667603,
      1.4882278442382812,
      1.4169243574142456,
      1.434967279434204,
      1.4791187047958374,
      1.4924046993255615,
      1.464010238647461,
      1.355211853981018,
      1.4422768354415894,
      1.4772409200668335,
      1.4855642318725586,
      1.3914214372634888,
      1.5317624807357788,
      1.4767916202545166,
      1.5525057315826416,
      1.4267528057098389,
      1.3921186923980713,
      1.4735260009765625,
      1.4076300859451294,
      1.4896430969238281,
      1.440387487411499,
      1.436928391456604,
      1.4201313257217407
    ],
    "base_loss": [
      11.832879066467285,
      11.36864185333252,
      10.761643409729004,
      10.459257125854492,
      10.175871849060059,
      9.816800117492676,
      9.529030799865723,
      9.260773658752441,
      9.017507553100586,
      8.663485527038574,
      8.371795654296875,
      8.118546485900879,
      7.833754539489746,
      7.534058570861816,
      7.297992706298828,
      6.95162296295166,
      6.682138919830322,
      6.448472023010254,
      6.186150550842285,
      5.851781368255615,
      5.642083168029785,
      5.3920135498046875,
      5.093167781829834,
      4.796841621398926,
      4.534151554107666,
      4.4100446701049805,
      4.1128153800964355,
      3.8507931232452393,
      3.6341888904571533,
      3.4420320987701416,
      3.2039170265197754,
      3.1796298027038574,
      2.970296621322632,
      2.8707118034362793,
      2.7452502250671387,
      2.64337158203125,
      2.503178119659424,
      2.4706099033355713,
      2.370277166366577,
      2.268285036087036,
      2.318305492401123,
      2.2794229984283447,
      2.149564504623413,
      2.1994822025299072,
      2.2138619422912598,
      2.154664993286133,
      2.1253409385681152,
      2.104158401489258,
      2.0960659980773926,
      2.066467523574829,
      2.087989568710327,
      2.050243616104126,
      1.993398666381836,
      2.037946939468384,
      1.9801470041275024,
      2.0205399990081787,
      2.0452141761779785,
      1.938254475593567,
      1.964411973953247,
      1.9696029424667358,
      1.9631099700927734,
      1.9499931335449219,
      1.9534265995025635,
      1.930572271347046,
      1.9394501447677612,
      1.9909099340438843,
      1.8681749105453491,
      1.8672945499420166,
      1.9460028409957886,
      1.8758708238601685,
      1.8508617877960205,
      1.8957277536392212,
      1.89703369140625,
      1.8883601427078247,
      1.9217368364334106,
      1.8644115924835205,
      1.8522149324417114,
      1.8996679782867432,
      1.7724077701568604,
      1.8279880285263062,
      1.8713520765304565,
      1.7754944562911987,
      1.8203290700912476,
      1.8123414516448975,
      1.718186616897583,
      1.7727686166763306,
      1.7215014696121216,
      1.6721407175064087,
      1.7613623142242432,
      1.7179336547851562,
      1.773058533668518,
      1.7030664682388306,
      1.6822694540023804,
      1.708242654800415,
      1.6574541330337524,
      1.7228068113327026,
      1.6863057613372803,
      1.7814959287643433,
      1.6393948793411255,
      1.602840781211853,
      1.6753469705581665,
      1.6772491931915283,
      1.6624025106430054,
      1.6662027835845947,
      1.6071633100509644,
      1.6065205335617065,
      1.6259610652923584,
      1.6235558986663818,
      1.545785665512085,
      1.5571601390838623,
      1.6024692058563232,
      1.5312445163726807,
      1.5064446926116943,
      1.5606070756912231,
      1.5643519163131714,
      1.505636215209961,
      1.542928695678711,
      1.5629281997680664,
      1.5619080066680908,
      1.5639537572860718,
      1.4565213918685913,
      1.4847272634506226,
      1.502737045288086,
      1.4859784841537476,
      1.4928518533706665,
      1.4829928874969482,
      1.4601913690567017,
      1.531474232673645,
      1.4434651136398315,
      1.5020925998687744,
      1.47160804271698,
      1.4648197889328003,
      1.5810593366622925,
      1.4178014993667603,
      1.4882278442382812,
      1.4169243574142456,
      1.434967279434204,
      1.4791187047958374,
      1.4924046993255615,
      1.464010238647461,
      1.355211853981018,
      1.4422768354415894,
      1.4772409200668335,
      1.4855642318725586,
      1.3914214372634888,
      1.5317624807357788,
      1.4767916202545166,
      1.5525057315826416,
      1.4267528057098389,
      1.3921186923980713,
      1.4735260009765625,
      1.4076300859451294,
      1.4896430969238281,
      1.440387487411499,
      1.436928391456604,
      1.4201313257217407
    ],
    "lr": [
      4.188034188034189e-06,
      8.461538461538462e-06,
      1.2735042735042738e-05,
      1.700854700854701e-05,
      1.99998070587026e-05,
      1.9996377198308693e-05,
      1.9988661446190374e-05,
      1.9976663110433913e-05,
      1.996038733525975e-05,
      1.9939841098816945e-05,
      1.9915033210191323e-05,
      1.988597430562865e-05,
      1.9852676843974367e-05,
      1.9815155101331968e-05,
      1.9773425164942158e-05,
      1.972750492628558e-05,
      1.9677414073411902e-05,
      1.9623174082498696e-05,
      1.9564808208643647e-05,
      1.9502341475894044e-05,
      1.943580066651786e-05,
      1.9365214309521018e-05,
      1.929061266841572e-05,
      1.921202772824513e-05,
      1.9129493181869986e-05,
      1.9043044415522936e-05,
      1.895271849363693e-05,
      1.8858554142954e-05,
      1.876059173592142e-05,
      1.8658873273382223e-05,
      1.855344236656757e-05,
      1.844434421839869e-05,
      1.833162560410636e-05,
      1.821533485117628e-05,
      1.8095521818628955e-05,
      1.7972237875642883e-05,
      1.784553587953032e-05,
      1.7715470153074993e-05,
      1.7582096461241506e-05,
      1.7445471987266426e-05,
      1.7305655308141265e-05,
      1.7162706369497937e-05,
      1.7016686459907395e-05,
      1.686765818460248e-05,
      1.671568543863629e-05,
      1.6560833379487507e-05,
      1.6403168399124515e-05,
      1.624275809554018e-05,
      1.60796712437696e-05,
      1.5913977766403167e-05,
      1.5745748703607686e-05,
      1.5575056182668234e-05,
      1.540197338706403e-05,
      1.5226574525091396e-05,
      1.5048934798047377e-05,
      1.4869130367987577e-05,
      1.4687238325072114e-05,
      1.4503336654513631e-05,
      1.4317504203141566e-05,
      1.4129820645597008e-05,
      1.3940366450172631e-05,
      1.3749222844312363e-05,
      1.3556471779785555e-05,
      1.3362195897550621e-05,
      1.3166478492323176e-05,
      1.2969403476863889e-05,
      1.2771055346001343e-05,
      1.2571519140405352e-05,
      1.2370880410126232e-05,
      1.2169225177915685e-05,
      1.1966639902345032e-05,
      1.176321144073656e-05,
      1.1559027011923938e-05,
      1.1354174158857612e-05,
      1.1148740711071244e-05,
      1.0942814747025295e-05,
      1.0736484556343858e-05,
      1.0529838601960978e-05,
      1.032296548219264e-05,
      1.0115953892750747e-05,
      9.908892588715302e-06,
      9.701870346481186e-06,
      9.494975925695724e-06,
      9.288298031203511e-06,
      9.08192527501469e-06,
      8.875946138313021e-06,
      8.670448933520089e-06,
      8.465521766431812e-06,
      8.261252498443584e-06,
      8.057728708880185e-06,
      7.855037657446609e-06,
      7.653266246815977e-06,
      7.452500985370459e-06,
      7.252827950111301e-06,
      7.054332749753787e-06,
      6.85710048802296e-06,
      6.661215727165885e-06,
      6.466762451696027e-06,
      6.273824032385387e-06,
      6.082483190519702e-06,
      5.89282196243218e-06,
      5.70492166433086e-06,
      5.518862857434741e-06,
      5.334725313433595e-06,
      5.152587980286308e-06,
      4.9725289483723496e-06,
      4.794625417010984e-06,
      4.618953661362472e-06,
      4.445588999725514e-06,
      4.274605761244949e-06,
      4.1060772540435276e-06,
      3.940075733791495e-06,
      3.7766723727273204e-06,
      3.6159372291430285e-06,
      3.4579392173470783e-06,
      3.3027460781177336e-06,
      3.15042434965958e-06,
      3.001039339075622e-06,
      2.8546550943672535e-06,
      2.7113343769739875e-06,
      2.57113863486487e-06,
      2.4341279761929824e-06,
      2.300361143524409e-06,
      2.169895488652681e-06,
      2.0427869480095065e-06,
      1.9190900186823515e-06,
      1.7988577350490876e-06,
      1.6821416460398155e-06,
      1.5689917930355325e-06,
      1.4594566884131744e-06,
      1.3535832947461935e-06,
      1.2514170046696128e-06,
      1.1530016214181938e-06,
      1.0583793400460306e-06,
      9.675907293356556e-07,
      8.806747144044004e-07,
      7.976685600154599e-07,
      7.186078546008335e-07,
      6.435264950029818e-07,
      5.724566719417401e-07,
      5.054288562127241e-07,
      4.424717856231486e-07,
      3.836124526706375e-07,
      3.288760929703605e-07,
      2.782861744353793e-07,
      2.318643872149251e-07,
      1.896306343948595e-07,
      1.516030234643362e-07,
      1.1779785855231873e-07,
      8.822963343726454e-08,
      6.291102533301563e-08,
      4.1852889453490864e-08,
      2.5064254358589946e-08,
      1.2552318083239868e-08,
      4.32244505127799e-09,
      3.781637754884937e-10
    ],
    "eval_step": [
      390,
      780,
      1170,
      1560,
      1950,
      2340,
      2730,
      3120,
      3510,
      3900,
      4290,
      4680,
      5070,
      5460,
      5850,
      6240,
      6630,
      7020,
      7410,
      7800
    ],
    "eval_epoch": [
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20
    ],
    "eval_accuracy": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.003157894736842105,
      0.0010526315789473684,
      0.0,
      0.0,
      0.0010526315789473684,
      0.002105263157894737,
      0.010526315789473684,
      0.00631578947368421,
      0.002105263157894737,
      0.004210526315789474,
      0.010526315789473684,
      0.008421052631578947,
      0.011578947368421053,
      0.009473684210526316,
      0.009473684210526316
    ]
  },
  "final_accuracy": 0.011153846153846153,
  "sft_eval": {
    "config": {
      "ops": "add_sub",
      "K": null,
      "mode": "sft",
      "n_digits": 6,
      "n_per_split": 100
    },
    "splits": {
      "add_S0": {
        "full_accuracy": 0.01,
        "digit_accuracy": 0.49142857142857144,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.4396694214876033,
            "count": 605
          },
          "SS": {
            "accuracy": 0.8210526315789474,
            "count": 95
          }
        }
      },
      "add_S1": {
        "full_accuracy": 0.01,
        "digit_accuracy": 0.4471428571428571,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.5245098039215687,
            "count": 204
          },
          "SC": {
            "accuracy": 0.33136094674556216,
            "count": 169
          },
          "SS": {
            "accuracy": 0.6129032258064516,
            "count": 31
          },
          "UC": {
            "accuracy": 0.44256756756756754,
            "count": 296
          }
        }
      },
      "add_S2": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4957142857142857,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.5460122699386503,
            "count": 163
          },
          "SC": {
            "accuracy": 0.38461538461538464,
            "count": 130
          },
          "SS": {
            "accuracy": 0.6091954022988506,
            "count": 87
          },
          "UC": {
            "accuracy": 0.43349753694581283,
            "count": 203
          },
          "US": {
            "accuracy": 0.5726495726495726,
            "count": 117
          }
        }
      },
      "add_S3": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4685714285714286,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.5950413223140496,
            "count": 121
          },
          "SC": {
            "accuracy": 0.3305785123966942,
            "count": 121
          },
          "SS": {
            "accuracy": 0.673469387755102,
            "count": 49
          },
          "UC": {
            "accuracy": 0.3870967741935484,
            "count": 186
          },
          "US": {
            "accuracy": 0.4977578475336323,
            "count": 223
          }
        }
      },
      "add_S4": {
        "full_accuracy": 0.03,
        "digit_accuracy": 0.44142857142857145,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.6057692307692307,
            "count": 104
          },
          "SC": {
            "accuracy": 0.3584905660377358,
            "count": 106
          },
          "SS": {
            "accuracy": 0.7391304347826086,
            "count": 23
          },
          "UC": {
            "accuracy": 0.3875,
            "count": 160
          },
          "US": {
            "accuracy": 0.4201954397394137,
            "count": 307
          }
        }
      },
      "add_S5": {
        "full_accuracy": 0.07,
        "digit_accuracy": 0.39571428571428574,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.59,
            "count": 100
          },
          "SC": {
            "accuracy": 0.29,
            "count": 100
          },
          "UC": {
            "accuracy": 0.33,
            "count": 100
          },
          "US": {
            "accuracy": 0.39,
            "count": 400
          }
        }
      },
      "add_S6": {
        "full_accuracy": 0.13,
        "digit_accuracy": 0.26,
        "n_examples": 100,
        "per_subtask": {
          "SC": {
            "accuracy": 0.29,
            "count": 100
          },
          "UC": {
            "accuracy": 0.25,
            "count": 100
          },
          "US": {
            "accuracy": 0.256,
            "count": 500
          }
        }
      },
      "add_random": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.43857142857142856,
        "n_examples": 200,
        "per_subtask": {
          "SA": {
            "accuracy": 0.4720357941834452,
            "count": 447
          },
          "SC": {
            "accuracy": 0.35625,
            "count": 320
          },
          "SS": {
            "accuracy": 0.625,
            "count": 56
          },
          "UC": {
            "accuracy": 0.42344045368620037,
            "count": 529
          },
          "US": {
            "accuracy": 0.625,
            "count": 48
          }
        }
      },
      "add_C1": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4642857142857143,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.526,
            "count": 500
          },
          "SC": {
            "accuracy": 0.28,
            "count": 100
          },
          "UC": {
            "accuracy": 0.34,
            "count": 100
          }
        }
      },
      "add_C2": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4714285714285714,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.5275,
            "count": 400
          },
          "SC": {
            "accuracy": 0.42,
            "count": 100
          },
          "UC": {
            "accuracy": 0.3141025641025641,
            "count": 156
          },
          "US": {
            "accuracy": 0.6363636363636364,
            "count": 44
          }
        }
      },
      "add_C3": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.45571428571428574,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.5733333333333334,
            "count": 300
          },
          "SC": {
            "accuracy": 0.38,
            "count": 100
          },
          "UC": {
            "accuracy": 0.2914572864321608,
            "count": 199
          },
          "US": {
            "accuracy": 0.504950495049505,
            "count": 101
          }
        }
      },
      "add_C4": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.48142857142857143,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.65,
            "count": 200
          },
          "SC": {
            "accuracy": 0.38,
            "count": 100
          },
          "UC": {
            "accuracy": 0.3333333333333333,
            "count": 264
          },
          "US": {
            "accuracy": 0.5955882352941176,
            "count": 136
          }
        }
      },
      "add_C5": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4928571428571429,
        "n_examples": 100,
        "per_subtask": {
          "SA": {
            "accuracy": 0.7,
            "count": 100
          },
          "SC": {
            "accuracy": 0.43,
            "count": 100
          },
          "UC": {
            "accuracy": 0.3,
            "count": 310
          },
          "US": {
            "accuracy": 0.7315789473684211,
            "count": 190
          }
        }
      },
      "add_C6": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4957142857142857,
        "n_examples": 100,
        "per_subtask": {
          "SC": {
            "accuracy": 0.43,
            "count": 100
          },
          "UC": {
            "accuracy": 0.3621621621621622,
            "count": 370
          },
          "US": {
            "accuracy": 0.7391304347826086,
            "count": 230
          }
        }
      },
      "sub_M0": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.5628571428571428,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.5203252032520326,
            "count": 615
          },
          "ME": {
            "accuracy": 0.8705882352941177,
            "count": 85
          }
        }
      },
      "sub_M1": {
        "full_accuracy": 0.01,
        "digit_accuracy": 0.45,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.5821917808219178,
            "count": 292
          },
          "MB": {
            "accuracy": 0.3541666666666667,
            "count": 144
          },
          "ME": {
            "accuracy": 0.92,
            "count": 25
          },
          "UB": {
            "accuracy": 0.29707112970711297,
            "count": 239
          }
        }
      },
      "sub_M2": {
        "full_accuracy": 0.03,
        "digit_accuracy": 0.5485714285714286,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.7488151658767772,
            "count": 211
          },
          "MB": {
            "accuracy": 0.26956521739130435,
            "count": 115
          },
          "ME": {
            "accuracy": 0.9647058823529412,
            "count": 85
          },
          "UB": {
            "accuracy": 0.4530386740331492,
            "count": 181
          },
          "UD": {
            "accuracy": 0.28703703703703703,
            "count": 108
          }
        }
      },
      "sub_M3": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.43,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.8435754189944135,
            "count": 179
          },
          "MB": {
            "accuracy": 0.14563106796116504,
            "count": 103
          },
          "ME": {
            "accuracy": 0.9642857142857143,
            "count": 56
          },
          "UB": {
            "accuracy": 0.48322147651006714,
            "count": 149
          },
          "UD": {
            "accuracy": 0.04225352112676056,
            "count": 213
          }
        }
      },
      "sub_M4": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.31857142857142856,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.705,
            "count": 200
          },
          "MB": {
            "accuracy": 0.08,
            "count": 100
          },
          "UB": {
            "accuracy": 0.56,
            "count": 100
          },
          "UD": {
            "accuracy": 0.06,
            "count": 300
          }
        }
      },
      "sub_M5": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.24428571428571427,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 1.0,
            "count": 100
          },
          "MB": {
            "accuracy": 0.0,
            "count": 100
          },
          "UB": {
            "accuracy": 0.71,
            "count": 100
          },
          "UD": {
            "accuracy": 0.0,
            "count": 400
          }
        }
      },
      "sub_random": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.4635714285714286,
        "n_examples": 200,
        "per_subtask": {
          "MD": {
            "accuracy": 0.5716666666666667,
            "count": 600
          },
          "MB": {
            "accuracy": 0.3258426966292135,
            "count": 267
          },
          "ME": {
            "accuracy": 0.8113207547169812,
            "count": 53
          },
          "UB": {
            "accuracy": 0.35990888382687924,
            "count": 439
          },
          "UD": {
            "accuracy": 0.43902439024390244,
            "count": 41
          }
        }
      },
      "sub_B3": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.38857142857142857,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.57,
            "count": 300
          },
          "MB": {
            "accuracy": 0.22,
            "count": 100
          },
          "UB": {
            "accuracy": 0.3350253807106599,
            "count": 197
          },
          "UD": {
            "accuracy": 0.1262135922330097,
            "count": 103
          }
        }
      },
      "sub_B4": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.3742857142857143,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 0.69,
            "count": 200
          },
          "MB": {
            "accuracy": 0.18,
            "count": 100
          },
          "UB": {
            "accuracy": 0.3441295546558704,
            "count": 247
          },
          "UD": {
            "accuracy": 0.13725490196078433,
            "count": 153
          }
        }
      },
      "sub_B5": {
        "full_accuracy": 0.0,
        "digit_accuracy": 0.35714285714285715,
        "n_examples": 100,
        "per_subtask": {
          "MD": {
            "accuracy": 1.0,
            "count": 100
          },
          "MB": {
            "accuracy": 0.17,
            "count": 100
          },
          "UB": {
            "accuracy": 0.38926174496644295,
            "count": 298
          },
          "UD": {
            "accuracy": 0.08415841584158416,
            "count": 202
          }
        }
      }
    },
    "summary": {
      "overall_accuracy": 0.011153846153846153,
      "digit_accuracy": 0.43615384615384617,
      "total_examples": 2600,
      "n_splits": 24
    }
  }
}