{
  "execution": "local RTX 5090",
  "method": "TS3",
  "parameters": 11882756,
  "phase": "paused_by_user",
  "precision_order": [
    "FP32",
    "FP64"
  ],
  "epochs_each": 20,
  "train_cases": 800,
  "validation_cases": 100,
  "test_cases": 100,
  "test_labels_read": false,
  "AMP": false,
  "TF32": false,
  "initialization": "same saved FP32 initial values, exactly promoted to FP64 for the second independent arm",
  "finished_arms": 1,
  "current_precision": "fp64",
  "epoch_completed": 6,
  "case_cursor": 200,
  "active_seconds": 24733.596707858003,
  "waiting_seconds": null,
  "remaining_training_seconds_estimate": 52520.94612309611,
  "updated_unix": 1793233733.5992084,
  "error_threshold": null,
  "early_stopping": false,
  "exclusive_GPU_policy": "Wait for the existing compute workflow and its following validation, then three idle observations; yield own run if contention returns",
  "cpu_dtype_check": {
    "fp32": {
      "parameters": 11882756,
      "parameter_dtype": "torch.float32",
      "observed_floating_module_outputs": [
        "torch.float32"
      ],
      "finite_forward_backward": true,
      "scope": "CPU synthetic 19-node dtype check; not GPU timing"
    },
    "fp64": {
      "parameters": 11882756,
      "parameter_dtype": "torch.float64",
      "observed_floating_module_outputs": [
        "torch.float64"
      ],
      "finite_forward_backward": true,
      "scope": "CPU synthetic 19-node dtype check; not GPU timing"
    }
  },
  "source_config_sha256": "47369eb28682d7fa8c44bbe5eadd3ff10791bcdb77c45b25682a44355bc8486d",
  "timers": {
    "train_wall_seconds": 23873.15732868005,
    "forward_backward_gpu_seconds": 24232.90661572266,
    "optimizer_gpu_seconds": 2.858162168741229,
    "checkpoint_seconds": 27.13966263698603,
    "validation_seconds": 823.6099257249989
  },
  "peak_allocated_bytes": 10687886336,
  "optimizer_updates": 625,
  "epoch_curves": {
    "fp32": [
      {
        "epoch": 1,
        "precision": "fp32",
        "active_seconds": 292.53119877399877,
        "online_training_objective": 5.85199445605278,
        "cumulative_train_wall_seconds": 273.14319286299906,
        "cumulative_forward_backward_gpu_seconds": 224.85664277648962,
        "cumulative_optimizer_gpu_seconds": 0.5114034843444824,
        "cumulative_validation_seconds": 16.62708620199919,
        "mean_validation_U_L2": 1.733009358626251,
        "mean_validation_VM_L2": 0.746308618321712
      },
      {
        "epoch": 2,
        "precision": "fp32",
        "active_seconds": 540.7274279059984,
        "online_training_objective": 1.7634325486421585,
        "cumulative_train_wall_seconds": 505.7502845580002,
        "cumulative_forward_backward_gpu_seconds": 449.76951963806226,
        "cumulative_optimizer_gpu_seconds": 0.85177746796608,
        "cumulative_validation_seconds": 28.847337418999814,
        "mean_validation_U_L2": 1.064205498875448,
        "mean_validation_VM_L2": 0.7068550219802887
      },
      {
        "epoch": 3,
        "precision": "fp32",
        "active_seconds": 784.8695073529998,
        "online_training_objective": 0.9274752035737037,
        "cumulative_train_wall_seconds": 734.4019976949858,
        "cumulative_forward_backward_gpu_seconds": 674.57175883484,
        "cumulative_optimizer_gpu_seconds": 1.1958190996646885,
        "cumulative_validation_seconds": 41.031675648999226,
        "mean_validation_U_L2": 0.8182536862582954,
        "mean_validation_VM_L2": 0.6697711931748402
      },
      {
        "epoch": 4,
        "precision": "fp32",
        "active_seconds": 1031.6661074489984,
        "online_training_objective": 0.7081887927651406,
        "cumulative_train_wall_seconds": 964.953777895973,
        "cumulative_forward_backward_gpu_seconds": 899.1040238952663,
        "cumulative_optimizer_gpu_seconds": 1.5360314836502076,
        "cumulative_validation_seconds": 53.994581671999185,
        "mean_validation_U_L2": 0.9643217306644749,
        "mean_validation_VM_L2": 0.6459519773315605
      },
      {
        "epoch": 5,
        "precision": "fp32",
        "active_seconds": 1283.187164467001,
        "online_training_objective": 0.5377564929425717,
        "cumulative_train_wall_seconds": 1201.1879159169875,
        "cumulative_forward_backward_gpu_seconds": 1123.711544174197,
        "cumulative_optimizer_gpu_seconds": 1.8796969234943386,
        "cumulative_validation_seconds": 66.10173192099683,
        "mean_validation_U_L2": 0.7900960169999902,
        "mean_validation_VM_L2": 0.5647513553911255
      },
      {
        "epoch": 6,
        "precision": "fp32",
        "active_seconds": 1532.4083910289974,
        "online_training_objective": 0.497676345705986,
        "cumulative_train_wall_seconds": 1434.1131140549805,
        "cumulative_forward_backward_gpu_seconds": 1348.4294615173358,
        "cumulative_optimizer_gpu_seconds": 2.2127704906463608,
        "cumulative_validation_seconds": 79.14614343099674,
        "mean_validation_U_L2": 0.6914699146892446,
        "mean_validation_VM_L2": 0.5266991549117348
      },
      {
        "epoch": 7,
        "precision": "fp32",
        "active_seconds": 1799.7331050489975,
        "online_training_objective": 0.36839810565114023,
        "cumulative_train_wall_seconds": 1684.3849849329854,
        "cumulative_forward_backward_gpu_seconds": 1573.1617207946804,
        "cumulative_optimizer_gpu_seconds": 2.550084955930708,
        "cumulative_validation_seconds": 92.57851941799527,
        "mean_validation_U_L2": 0.5027115711919294,
        "mean_validation_VM_L2": 0.427611270827836
      },
      {
        "epoch": 8,
        "precision": "fp32",
        "active_seconds": 2067.5156015400007,
        "online_training_objective": 0.2938681323081255,
        "cumulative_train_wall_seconds": 1935.1829326069892,
        "cumulative_forward_backward_gpu_seconds": 1797.7607106628463,
        "cumulative_optimizer_gpu_seconds": 2.888717787265775,
        "cumulative_validation_seconds": 105.9309363969951,
        "mean_validation_U_L2": 0.4663555118814145,
        "mean_validation_VM_L2": 0.43070002668307306
      },
      {
        "epoch": 9,
        "precision": "fp32",
        "active_seconds": 2332.5115981679974,
        "online_training_objective": 0.29484805881977083,
        "cumulative_train_wall_seconds": 2183.2306714759725,
        "cumulative_forward_backward_gpu_seconds": 2022.2656664428782,
        "cumulative_optimizer_gpu_seconds": 3.229331386089322,
        "cumulative_validation_seconds": 119.28678625399334,
        "mean_validation_U_L2": 0.5047419305858243,
        "mean_validation_VM_L2": 0.461407290155371
      },
      {
        "epoch": 10,
        "precision": "fp32",
        "active_seconds": 2599.660450112,
        "online_training_objective": 0.25855292394757273,
        "cumulative_train_wall_seconds": 2433.5460435469613,
        "cumulative_forward_backward_gpu_seconds": 2246.823277328499,
        "cumulative_optimizer_gpu_seconds": 3.5572209858894306,
        "cumulative_validation_seconds": 132.54189673799556,
        "mean_validation_U_L2": 0.39586891090100773,
        "mean_validation_VM_L2": 0.37425027920805987
      },
      {
        "epoch": 11,
        "precision": "fp32",
        "active_seconds": 2864.0560051480006,
        "online_training_objective": 0.18018072865903378,
        "cumulative_train_wall_seconds": 2681.749611212941,
        "cumulative_forward_backward_gpu_seconds": 2471.418041656504,
        "cumulative_optimizer_gpu_seconds": 3.899299449920648,
        "cumulative_validation_seconds": 145.3282159429982,
        "mean_validation_U_L2": 0.40305144550463984,
        "mean_validation_VM_L2": 0.3899514379743141
      },
      {
        "epoch": 12,
        "precision": "fp32",
        "active_seconds": 3129.078870585001,
        "online_training_objective": 0.1893819984793663,
        "cumulative_train_wall_seconds": 2929.901071027951,
        "cumulative_forward_backward_gpu_seconds": 2696.212697128295,
        "cumulative_optimizer_gpu_seconds": 4.2359742026328995,
        "cumulative_validation_seconds": 158.801896889996,
        "mean_validation_U_L2": 0.3261349466788588,
        "mean_validation_VM_L2": 0.3496485607482014
      },
      {
        "epoch": 13,
        "precision": "fp32",
        "active_seconds": 3394.535914713999,
        "online_training_objective": 0.1719182702153921,
        "cumulative_train_wall_seconds": 3178.773502584936,
        "cumulative_forward_backward_gpu_seconds": 2920.8180158996593,
        "cumulative_optimizer_gpu_seconds": 4.564655738830556,
        "cumulative_validation_seconds": 171.79206392599735,
        "mean_validation_U_L2": 0.4936299740974587,
        "mean_validation_VM_L2": 0.3464852719221316
      },
      {
        "epoch": 14,
        "precision": "fp32",
        "active_seconds": 3661.262500302999,
        "online_training_objective": 0.13557964641600848,
        "cumulative_train_wall_seconds": 3428.4897555749412,
        "cumulative_forward_backward_gpu_seconds": 3145.449134674072,
        "cumulative_optimizer_gpu_seconds": 4.897818236350998,
        "cumulative_validation_seconds": 185.03787995699895,
        "mean_validation_U_L2": 0.24695282047944972,
        "mean_validation_VM_L2": 0.33255336528591894
      },
      {
        "epoch": 15,
        "precision": "fp32",
        "active_seconds": 3926.8336481829992,
        "online_training_objective": 0.09613803081214428,
        "cumulative_train_wall_seconds": 3677.148695092921,
        "cumulative_forward_backward_gpu_seconds": 3370.1230749053975,
        "cumulative_optimizer_gpu_seconds": 5.2351259176730975,
        "cumulative_validation_seconds": 198.26886785700117,
        "mean_validation_U_L2": 0.20987754187315522,
        "mean_validation_VM_L2": 0.3050815062054776
      },
      {
        "epoch": 16,
        "precision": "fp32",
        "active_seconds": 4192.919113821999,
        "online_training_objective": 0.08161027144640684,
        "cumulative_train_wall_seconds": 3926.2513418159215,
        "cumulative_forward_backward_gpu_seconds": 3595.153029678346,
        "cumulative_optimizer_gpu_seconds": 5.5782046060561985,
        "cumulative_validation_seconds": 211.80242432199884,
        "mean_validation_U_L2": 0.16923121801586358,
        "mean_validation_VM_L2": 0.29321090072144407
      },
      {
        "epoch": 17,
        "precision": "fp32",
        "active_seconds": 4459.289055790998,
        "online_training_objective": 0.07254318341612816,
        "cumulative_train_wall_seconds": 4175.807539112906,
        "cumulative_forward_backward_gpu_seconds": 3820.026637084958,
        "cumulative_optimizer_gpu_seconds": 5.919293373346313,
        "cumulative_validation_seconds": 225.2002318959967,
        "mean_validation_U_L2": 0.19121843364036736,
        "mean_validation_VM_L2": 0.29043317547706826
      },
      {
        "epoch": 18,
        "precision": "fp32",
        "active_seconds": 4725.175091797,
        "online_training_objective": 0.06513093959540128,
        "cumulative_train_wall_seconds": 4424.958118525912,
        "cumulative_forward_backward_gpu_seconds": 4044.8551472625722,
        "cumulative_optimizer_gpu_seconds": 6.260276733398419,
        "cumulative_validation_seconds": 238.47175554299974,
        "mean_validation_U_L2": 0.16550072819835995,
        "mean_validation_VM_L2": 0.28392326025561965
      },
      {
        "epoch": 19,
        "precision": "fp32",
        "active_seconds": 4990.284453186998,
        "online_training_objective": 0.06121941054239869,
        "cumulative_train_wall_seconds": 4673.46288712191,
        "cumulative_forward_backward_gpu_seconds": 4269.620746231085,
        "cumulative_optimizer_gpu_seconds": 6.598021884918189,
        "cumulative_validation_seconds": 251.7117106350015,
        "mean_validation_U_L2": 0.15021361370329311,
        "mean_validation_VM_L2": 0.27986405192648606
      },
      {
        "epoch": 20,
        "precision": "fp32",
        "active_seconds": 5256.675693495001,
        "online_training_objective": 0.05855873852968216,
        "cumulative_train_wall_seconds": 4923.018869042902,
        "cumulative_forward_backward_gpu_seconds": 4494.2936603546195,
        "cumulative_optimizer_gpu_seconds": 6.9325837092399345,
        "cumulative_validation_seconds": 265.03405312700124,
        "mean_validation_U_L2": 0.14407541536861024,
        "mean_validation_VM_L2": 0.27737759340475754
      }
    ],
    "fp64": [
      {
        "epoch": 1,
        "precision": "fp64",
        "active_seconds": 4033.2218505029996,
        "online_training_objective": 5.85198136816027,
        "cumulative_train_wall_seconds": 3886.741387455015,
        "cumulative_forward_backward_gpu_seconds": 3881.407590332035,
        "cumulative_optimizer_gpu_seconds": 0.5894270639419554,
        "cumulative_validation_seconds": 141.597910212,
        "mean_validation_U_L2": 1.7329939664268215,
        "mean_validation_VM_L2": 0.746308475856751
      },
      {
        "epoch": 2,
        "precision": "fp64",
        "active_seconds": 8064.789096313001,
        "online_training_objective": 1.7631230467164392,
        "cumulative_train_wall_seconds": 7772.407190723014,
        "cumulative_forward_backward_gpu_seconds": 7760.789523681642,
        "cumulative_optimizer_gpu_seconds": 1.0245093052387235,
        "cumulative_validation_seconds": 281.7740598090022,
        "mean_validation_U_L2": 1.0592703911244354,
        "mean_validation_VM_L2": 0.7068280667324878
      },
      {
        "epoch": 3,
        "precision": "fp64",
        "active_seconds": 12047.922923311999,
        "online_training_objective": 0.9661809003343365,
        "cumulative_train_wall_seconds": 11612.538147613024,
        "cumulative_forward_backward_gpu_seconds": 11633.313697998039,
        "cumulative_optimizer_gpu_seconds": 1.4601357662677763,
        "cumulative_validation_seconds": 418.94442857500326,
        "mean_validation_U_L2": 0.7448157893693892,
        "mean_validation_VM_L2": 0.7221103561907841
      },
      {
        "epoch": 4,
        "precision": "fp64",
        "active_seconds": 15985.916017665,
        "online_training_objective": 0.6708300020479983,
        "cumulative_train_wall_seconds": 15409.190982893022,
        "cumulative_forward_backward_gpu_seconds": 15506.186522460941,
        "cumulative_optimizer_gpu_seconds": 1.8885630304813383,
        "cumulative_validation_seconds": 554.6962912960043,
        "mean_validation_U_L2": 0.7635234727240156,
        "mean_validation_VM_L2": 0.6599414697093744
      },
      {
        "epoch": 5,
        "precision": "fp64",
        "active_seconds": 19913.744458094003,
        "online_training_objective": 0.6241455049976213,
        "cumulative_train_wall_seconds": 19195.477900967002,
        "cumulative_forward_backward_gpu_seconds": 19378.97030908209,
        "cumulative_optimizer_gpu_seconds": 2.3191226789951327,
        "cumulative_validation_seconds": 690.1484406580021,
        "mean_validation_U_L2": 0.6866762056340682,
        "mean_validation_VM_L2": 0.5372744462963331
      },
      {
        "epoch": 6,
        "precision": "fp64",
        "active_seconds": 23796.389159351,
        "online_training_objective": 0.5081765485082549,
        "cumulative_train_wall_seconds": 22939.03599232604,
        "cumulative_forward_backward_gpu_seconds": 23249.87496166991,
        "cumulative_optimizer_gpu_seconds": 2.75207276105881,
        "cumulative_validation_seconds": 823.6099257249989,
        "mean_validation_U_L2": 0.8024834755435609,
        "mean_validation_VM_L2": 0.5582116859123153
      }
    ]
  },
  "completed_train_case_passes": 5000
}