{
  "n_matched_performance_genes": 110,
  "n_truncated_inputs_in_matched_set": 0,
  "n_first_rater_scores_seen_before_rescore": 25,
  "same_rater": {
    "note": "x = SFT (rater B), y = RL (rater B)",
    "all": {
      "correctness": {
        "n": 110,
        "mean_x": 3.964,
        "mean_y": 4.045,
        "mean_diff_x_minus_y": -0.082,
        "x_higher": 24,
        "y_higher": 38,
        "tied": 48,
        "exact_agreement": 48,
        "within_one": 84,
        "quadratic_weighted_kappa": 0.431,
        "wilcoxon_p": 0.624,
        "x_distribution": {
          "1": 3,
          "2": 11,
          "3": 14,
          "4": 41,
          "5": 41
        },
        "y_distribution": {
          "1": 7,
          "2": 9,
          "3": 19,
          "4": 12,
          "5": 63
        }
      },
      "completeness": {
        "n": 110,
        "mean_x": 3.882,
        "mean_y": 2.745,
        "mean_diff_x_minus_y": 1.136,
        "x_higher": 80,
        "y_higher": 1,
        "tied": 29,
        "exact_agreement": 29,
        "within_one": 72,
        "quadratic_weighted_kappa": 0.329,
        "wilcoxon_p": 2.53e-15,
        "x_distribution": {
          "1": 2,
          "2": 13,
          "3": 12,
          "4": 52,
          "5": 31
        },
        "y_distribution": {
          "1": 12,
          "2": 28,
          "3": 48,
          "4": 20,
          "5": 2
        }
      }
    },
    "by_reference_status": {
      "COMPLETE": {
        "correctness": {
          "n": 62,
          "mean_x": 4.177,
          "mean_y": 4.081,
          "mean_diff_x_minus_y": 0.097,
          "x_higher": 15,
          "y_higher": 16,
          "tied": 31,
          "exact_agreement": 31,
          "within_one": 50,
          "quadratic_weighted_kappa": 0.488,
          "wilcoxon_p": 0.419,
          "x_distribution": {
            "1": 1,
            "2": 3,
            "3": 6,
            "4": 26,
            "5": 26
          },
          "y_distribution": {
            "1": 4,
            "2": 7,
            "3": 7,
            "4": 6,
            "5": 38
          }
        },
        "completeness": {
          "n": 62,
          "mean_x": 4.097,
          "mean_y": 2.758,
          "mean_diff_x_minus_y": 1.339,
          "x_higher": 52,
          "y_higher": 0,
          "tied": 10,
          "exact_agreement": 10,
          "within_one": 36,
          "quadratic_weighted_kappa": 0.301,
          "wilcoxon_p": 1.31e-10,
          "x_distribution": {
            "1": 1,
            "2": 4,
            "3": 5,
            "4": 30,
            "5": 22
          },
          "y_distribution": {
            "1": 8,
            "2": 14,
            "3": 26,
            "4": 13,
            "5": 1
          }
        }
      },
      "DRAFT": {
        "correctness": {
          "n": 35,
          "mean_x": 3.543,
          "mean_y": 3.943,
          "mean_diff_x_minus_y": -0.4,
          "x_higher": 5,
          "y_higher": 16,
          "tied": 14,
          "exact_agreement": 14,
          "within_one": 28,
          "quadratic_weighted_kappa": 0.545,
          "wilcoxon_p": 0.0452,
          "x_distribution": {
            "1": 2,
            "2": 6,
            "3": 7,
            "4": 11,
            "5": 9
          },
          "y_distribution": {
            "1": 3,
            "2": 1,
            "3": 9,
            "4": 4,
            "5": 18
          }
        },
        "completeness": {
          "n": 35,
          "mean_x": 3.371,
          "mean_y": 2.686,
          "mean_diff_x_minus_y": 0.686,
          "x_higher": 19,
          "y_higher": 1,
          "tied": 15,
          "exact_agreement": 15,
          "within_one": 29,
          "quadratic_weighted_kappa": 0.559,
          "wilcoxon_p": 0.000141,
          "x_distribution": {
            "1": 1,
            "2": 9,
            "3": 5,
            "4": 16,
            "5": 4
          },
          "y_distribution": {
            "1": 3,
            "2": 12,
            "3": 14,
            "4": 5,
            "5": 1
          }
        }
      },
      "INITIALIZED": {
        "correctness": {
          "n": 3,
          "mean_x": 3.667,
          "mean_y": 5,
          "mean_diff_x_minus_y": -1.333,
          "x_higher": 0,
          "y_higher": 2,
          "tied": 1,
          "exact_agreement": 1,
          "within_one": 2,
          "quadratic_weighted_kappa": 0.0,
          "wilcoxon_p": 0.5,
          "x_distribution": {
            "2": 1,
            "4": 1,
            "5": 1
          },
          "y_distribution": {
            "5": 3
          }
        },
        "completeness": {
          "n": 3,
          "mean_x": 4.333,
          "mean_y": 3,
          "mean_diff_x_minus_y": 1.333,
          "x_higher": 2,
          "y_higher": 0,
          "tied": 1,
          "exact_agreement": 1,
          "within_one": 1,
          "quadratic_weighted_kappa": 0.0,
          "wilcoxon_p": 0.5,
          "x_distribution": {
            "3": 1,
            "5": 2
          },
          "y_distribution": {
            "3": 3
          }
        }
      },
      "IN_PROGRESS": {
        "correctness": {
          "n": 10,
          "mean_x": 4.2,
          "mean_y": 3.9,
          "mean_diff_x_minus_y": 0.3,
          "x_higher": 4,
          "y_higher": 4,
          "tied": 2,
          "exact_agreement": 2,
          "within_one": 4,
          "quadratic_weighted_kappa": -0.449,
          "wilcoxon_p": 0.57,
          "x_distribution": {
            "2": 1,
            "3": 1,
            "4": 3,
            "5": 5
          },
          "y_distribution": {
            "2": 1,
            "3": 3,
            "4": 2,
            "5": 4
          }
        },
        "completeness": {
          "n": 10,
          "mean_x": 4.2,
          "mean_y": 2.8,
          "mean_diff_x_minus_y": 1.4,
          "x_higher": 7,
          "y_higher": 0,
          "tied": 3,
          "exact_agreement": 3,
          "within_one": 6,
          "quadratic_weighted_kappa": -0.104,
          "wilcoxon_p": 0.0156,
          "x_distribution": {
            "3": 1,
            "4": 6,
            "5": 3
          },
          "y_distribution": {
            "1": 1,
            "2": 2,
            "3": 5,
            "4": 2
          }
        }
      }
    }
  },
  "rater_calibration": {
    "note": "x = RL rater B re-score, y = RL rater A (first rater); same RL text",
    "all": {
      "correctness": {
        "n": 110,
        "mean_x": 4.045,
        "mean_y": 3.955,
        "mean_diff_x_minus_y": 0.091,
        "x_higher": 13,
        "y_higher": 5,
        "tied": 92,
        "exact_agreement": 92,
        "within_one": 108,
        "quadratic_weighted_kappa": 0.934,
        "wilcoxon_p": 0.0409,
        "x_distribution": {
          "1": 7,
          "2": 9,
          "3": 19,
          "4": 12,
          "5": 63
        },
        "y_distribution": {
          "1": 7,
          "2": 11,
          "3": 19,
          "4": 16,
          "5": 57
        }
      },
      "completeness": {
        "n": 110,
        "mean_x": 2.745,
        "mean_y": 2.873,
        "mean_diff_x_minus_y": -0.127,
        "x_higher": 12,
        "y_higher": 27,
        "tied": 71,
        "exact_agreement": 71,
        "within_one": 109,
        "quadratic_weighted_kappa": 0.782,
        "wilcoxon_p": 0.0308,
        "x_distribution": {
          "1": 12,
          "2": 28,
          "3": 48,
          "4": 20,
          "5": 2
        },
        "y_distribution": {
          "1": 7,
          "2": 33,
          "3": 38,
          "4": 31,
          "5": 1
        }
      }
    },
    "unseen_only": {
      "correctness": {
        "n": 85,
        "mean_x": 4.271,
        "mean_y": 4.141,
        "mean_diff_x_minus_y": 0.129,
        "x_higher": 12,
        "y_higher": 3,
        "tied": 70,
        "exact_agreement": 70,
        "within_one": 83,
        "quadratic_weighted_kappa": 0.915,
        "wilcoxon_p": 0.0165,
        "x_distribution": {
          "1": 6,
          "2": 1,
          "3": 12,
          "4": 11,
          "5": 55
        },
        "y_distribution": {
          "1": 6,
          "2": 4,
          "3": 11,
          "4": 15,
          "5": 49
        }
      },
      "completeness": {
        "n": 85,
        "mean_x": 2.941,
        "mean_y": 3.035,
        "mean_diff_x_minus_y": -0.094,
        "x_higher": 10,
        "y_higher": 19,
        "tied": 56,
        "exact_agreement": 56,
        "within_one": 84,
        "quadratic_weighted_kappa": 0.775,
        "wilcoxon_p": 0.157,
        "x_distribution": {
          "1": 7,
          "2": 15,
          "3": 41,
          "4": 20,
          "5": 2
        },
        "y_distribution": {
          "1": 5,
          "2": 18,
          "3": 32,
          "4": 29,
          "5": 1
        }
      }
    }
  },
  "cross_rater_confounded": {
    "note": "x = SFT (rater B), y = RL (rater A); rater and model are confounded",
    "all": {
      "correctness": {
        "n": 110,
        "mean_x": 3.964,
        "mean_y": 3.955,
        "mean_diff_x_minus_y": 0.009,
        "x_higher": 27,
        "y_higher": 33,
        "tied": 50,
        "exact_agreement": 50,
        "within_one": 85,
        "quadratic_weighted_kappa": 0.498,
        "wilcoxon_p": 0.738,
        "x_distribution": {
          "1": 3,
          "2": 11,
          "3": 14,
          "4": 41,
          "5": 41
        },
        "y_distribution": {
          "1": 7,
          "2": 11,
          "3": 19,
          "4": 16,
          "5": 57
        }
      },
      "completeness": {
        "n": 110,
        "mean_x": 3.882,
        "mean_y": 2.873,
        "mean_diff_x_minus_y": 1.009,
        "x_higher": 74,
        "y_higher": 2,
        "tied": 34,
        "exact_agreement": 34,
        "within_one": 76,
        "quadratic_weighted_kappa": 0.361,
        "wilcoxon_p": 3.94e-14,
        "x_distribution": {
          "1": 2,
          "2": 13,
          "3": 12,
          "4": 52,
          "5": 31
        },
        "y_distribution": {
          "1": 7,
          "2": 33,
          "3": 38,
          "4": 31,
          "5": 1
        }
      }
    }
  }
}