Summarise-herg-export output, not for merging just for testing

pcpostprocess/scripts/summarise_herg_export.py

@@ -59,7 +59,7 @@ def main():
     parser.add_argument('--protocols', type=str, default=[], nargs='+')
     parser.add_argument('-r', '--reversal', type=float, default=np.nan)
     # parser.add_argument('--selection_file', default=None, type=str)
-    parser.add_argument('--experiment_name', default='newtonrun4')
+    parser.add_argument('--experiment_name', default='13112023_MW2')
     parser.add_argument('--figsize', type=int, nargs=2, default=[5, 3])
     parser.add_argument('--output_all', action='store_true')
     parser.add_argument('--log_level', default='INFO')
@@ -85,11 +85,11 @@ def main():
 
     qc_styled_df = create_qc_table(qc_df)
     qc_styled_df = qc_styled_df.pivot(columns='protocol', index='crit')
-    qc_styled_df.to_excel(os.path.join(output_dir, 'qc_table.xlsx'))
+    # qc_styled_df.to_excel(os.path.join(output_dir, 'qc_table.xlsx'))
     qc_styled_df.to_latex(os.path.join(output_dir, 'qc_table.tex'))
 
-    qc_estimates_file = os.path.join(args.save_dir, f"{args.experiment_name}_subtraction_qc.csv")
-    qc_vals_df = pd.read_csv(os.path.join(qc_estimates_file))
+    # qc_estimates_file = os.path.join(args.save_dir, f"{args.experiment_name}_subtraction_qc.csv")
+    # qc_vals_df = pd.read_csv(os.path.join(qc_estimates_file))
 
     qc_df.protocol = ['staircaseramp1' if protocol == 'staircaseramp' else protocol
                       for protocol in qc_df.protocol]
@@ -131,9 +131,8 @@ def main():
                                                         categories=protocol_order,
                                                         ordered=True)
 
-        qc_vals_df['protocol'] = pd.Categorical(qc_vals_df['protocol'],
-                                                categories=protocol_order,
-                                                ordered=True)
+        # qc_vals_df['protocol'] = pd.Categorical(
+        #     qc_vals_df['protocol'], categories=protocol_order, ordered=True)
 
         leak_parameters_df.sort_values(['protocol', 'sweep'], inplace=True)
     except FileNotFoundError as exc:
@@ -161,10 +160,10 @@ def main():
         plot_spatial_Erev(leak_parameters_df)
 
     leak_parameters_df['passed QC'] = [well in passed_wells for well in leak_parameters_df.well]
-    qc_vals_df['passed QC'] = [well in passed_wells for well in qc_vals_df.well]
+    # qc_vals_df['passed QC'] = [well in passed_wells for well in qc_vals_df.well]
 
     # do_scatter_matrices(leak_parameters_df, qc_vals_df)
-    plot_histograms(leak_parameters_df, qc_vals_df)
+    plot_histograms(leak_parameters_df)
 
     # Very resource intensive
     # overlay_reversal_plots(leak_parameters_df)
@@ -235,7 +234,7 @@ def scatterplot_timescale_E_obs(df):
     ax.set_ylabel(r'$\tau$ (ms)')
     ax.set_xlabel(r'$E_\mathrm{obs}$')
 
-    fig.savefig(os.path.join(output_dir, "decay_timescale_vs_E_rev_scatter.pdf"))
+    fig.savefig(os.path.join(output_dir, "decay_timescale_vs_E_rev_scatter.png"))
     ax.cla()
 
     sns.lineplot(data=plot_df, y='40mV decay time constant',
@@ -245,7 +244,7 @@ def scatterplot_timescale_E_obs(df):
     ax.set_ylabel(r'$\tau$ (ms)')
     ax.set_xlabel(r'$E_\mathrm{obs}$')
     ax.spines[['top', 'right']].set_visible(False)
-    fig.savefig(os.path.join(output_dir, "decay_timescale_vs_E_rev_line.pdf"))
+    fig.savefig(os.path.join(output_dir, "decay_timescale_vs_E_rev_line.png"))
     ax.cla()
 
     plot_df['E_rev'] = (plot_df.set_index('well')['E_rev'] - plot_df.groupby('well')
@@ -257,7 +256,7 @@ def scatterplot_timescale_E_obs(df):
     ax.set_ylabel(r'$E_\mathrm{leak} - \bar E_\mathrm{leak}$ (ms)')
     ax.set_xlabel(r'$E_\mathrm{obs} - \bar E_\mathrm{obs}$')
 
-    fig.savefig(os.path.join(output_dir, "E_leak_vs_E_rev_scatter.pdf"))
+    fig.savefig(os.path.join(output_dir, "E_leak_vs_E_rev_scatter.png"))
     ax.cla()
 
 
@@ -336,8 +335,7 @@ def label_func(p, s):
         legend_handles, _ = ax.get_legend_handles_labels()
         ax.legend(legend_handles, ['failed QC', 'passed QC'], bbox_to_anchor=(1.26, 1))
 
-        fig.savefig(os.path.join(sub_dir, f"{var.replace(' ', '_')}.pdf"),
-                    format='pdf')
+        fig.savefig(os.path.join(sub_dir, f"{var.replace(' ', '_')}.png"))
         ax.cla()
 
     plt.close(fig)
@@ -477,8 +475,7 @@ def do_scatter_matrices(df, qc_df):
     grid = sns.pairplot(data=df, hue='hue', diag_kind='hist',
                         plot_kws={'alpha': 0.4, 'edgecolor': None},
                         hue_order=[True, False])
-    grid.savefig(os.path.join(output_dir, 'scatter_matrix_by_reversal.pdf'),
-                 format='pdf')
+    grid.savefig(os.path.join(output_dir, 'scatter_matrix_by_reversal.png'))
 
     # Now do artefact parameters only
     if 'drug' in qc_df:
@@ -656,8 +653,7 @@ def func(protocol, sweep):
         # Put 'A' row at the top
         ax.invert_yaxis()
 
-        fig.savefig(os.path.join(output_dir, f"{protocol}_sweep{sweep}_E_Kr_map.pdf"),
-                    format='pdf')
+        fig.savefig(os.path.join(output_dir, f"{protocol}_sweep{sweep}_E_Kr_map.png"))
         plt.close(fig)
 
     protocol = 'staircaseramp1'
@@ -696,12 +692,12 @@ def plot_spatial_passed(df):
     ax.set_yticklabels(string.ascii_uppercase[:16])
 
     ax.invert_yaxis()
-    fig.savefig(os.path.join(output_dir, "QC_map.pdf"), format='pdf')
+    fig.savefig(os.path.join(output_dir, "QC_map.png"))
 
     plt.close(fig)
 
 
-def plot_histograms(df, qc_df):
+def plot_histograms(df):
     fig = plt.figure(figsize=args.figsize, constrained_layout=True)
     ax = fig.subplots()
 
@@ -896,20 +892,22 @@ def create_attrition_table(qc_df, subtraction_df):
                             'qc6.subtracted', 'qc6.1.subtracted',
                             'qc6.2.subtracted']
 
-    subtraction_df_sc = subtraction_df[subtraction_df.protocol.isin(['staircaseramp1',
-                                                                     'staircaseramp1_2'])]
-    R_leftover_qc = subtraction_df_sc.groupby('well')['R_leftover'].max() < 0.4
+    # subtraction_df_sc = subtraction_df[
+    #    subtraction_df.protocol.isin(['staircaseramp1', 'staircaseramp1_2'])]
 
-    qc_df['QC.R_leftover'] = [R_leftover_qc.loc[well] for well in subtraction_df.well.unique()]
+    # R_leftover_qc = subtraction_df_sc.groupby('well')['R_leftover'].max() < 0.4
+
+    # This line doesn't work: "Length of values does not match length of index"
+    # qc_df['QC.R_leftover'] = [R_leftover_qc.loc[well] for well in subtraction_df.well.unique()]
 
     stage_3_criteria = original_qc_criteria + ['QC1.all_protocols', 'QC4.all_protocols',
                                                'QC6.all_protocols']
     stage_4_criteria = stage_3_criteria + ['qc3.bookend']
     stage_5_criteria = stage_4_criteria + ['QC.Erev.all_protocols', 'QC.Erev.spread']
 
-    stage_6_criteria = stage_5_criteria + ['QC.R_leftover']
+    # stage_6_criteria = stage_5_criteria + ['QC.R_leftover']
 
-    agg_dict = {crit: 'min' for crit in stage_6_criteria}
+    agg_dict = {crit: 'min' for crit in stage_5_criteria}
 
     qc_df_sc1 = qc_df[qc_df.protocol == 'staircaseramp1']
     print(qc_df_sc1.values.shape)
@@ -935,11 +933,11 @@ def create_attrition_table(qc_df, subtraction_df):
                                     .agg(agg_dict)[stage_5_criteria].values,
                                     axis=1))
 
-    n_stage_6_wells = np.sum(np.all(qc_df.groupby('well')
-                                    .agg(agg_dict)[stage_6_criteria].values,
-                                    axis=1))
+    # n_stage_6_wells = np.sum(np.all(qc_df.groupby('well')
+    #                                 .agg(agg_dict)[stage_6_criteria].values,
+    #                                 axis=1))
 
-    passed_qc_df = qc_df.groupby('well').agg(agg_dict)[stage_6_criteria]
+    passed_qc_df = qc_df.groupby('well').agg(agg_dict)[stage_5_criteria]
     print(passed_qc_df)
     passed_wells = [well for well, row in passed_qc_df.iterrows() if np.all(row.values)]
 
@@ -951,7 +949,7 @@ def create_attrition_table(qc_df, subtraction_df):
         'stage3': [n_stage_3_wells],
         'stage4': [n_stage_4_wells],
         'stage5': [n_stage_5_wells],
-        'stage6': [n_stage_6_wells],
+        # 'stage6': [n_stage_6_wells],
     }
 
     res_df = pd.DataFrame.from_records(res_dict)

summarise-all/summarise_herg_export/attrition.tex

@@ -0,0 +1,7 @@
+\begin{tabular}{lrrrrr}
+\toprule
+ & stage1 & stage2 & stage3 & stage4 & stage5 \\
+\midrule
+0 & 58 & 26 & 26 & 14 & 14 \\
+\bottomrule
+\end{tabular}

summarise-all/summarise_herg_export/pcpostprocess_info.txt

@@ -0,0 +1,6 @@
+pcpostprocess output https://github.com/CardiacModelling/pcpostprocess
+Date: 2025-11-12 15:10:51.794995
+Version: 0.2.1.dev1
+Build type: Develop
+Commit: g62a0bd564
+Command: pcpostprocess summarise_herg_export output-all --experiment_name=13112023_MW2 --output_dir=summarise-all

summarise-all/summarise_herg_export/qc_table.tex

@@ -0,0 +1,31 @@
+\begin{tabular}{lrrr}
+\toprule
+ & \multicolumn{3}{r}{wells failing} \\
+protocol & staircaseramp1 & staircaseramp1_2 & all \\
+crit &  &  &  \\
+\midrule
+QC.Erev.all\_protocols & 384 & 384 & 0 \\
+QC.Erev.spread & 384 & 384 & 0 \\
+QC1.all\_protocols & 384 & 384 & 0 \\
+QC4.all\_protocols & 384 & 384 & 0 \\
+QC6.all\_protocols & 384 & 384 & 0 \\
+all & 384 & 384 & 370 \\
+qc1.cm & 384 & 384 & 140 \\
+qc1.rseal & 384 & 384 & 188 \\
+qc1.rseries & 384 & 384 & 199 \\
+qc2.raw & 384 & 384 & 4 \\
+qc2.subtracted & 384 & 384 & 47 \\
+qc3.E4031 & 384 & 384 & 73 \\
+qc3.bookend & 384 & 384 & 287 \\
+qc3.raw & 384 & 384 & 135 \\
+qc3.subtracted & 384 & 384 & 183 \\
+qc4.cm & 384 & 384 & 142 \\
+qc4.rseal & 384 & 384 & 72 \\
+qc4.rseries & 384 & 384 & 152 \\
+qc5.1.staircase & 384 & 384 & 212 \\
+qc5.staircase & 384 & 384 & 80 \\
+qc6.1.subtracted & 384 & 384 & 226 \\
+qc6.2.subtracted & 384 & 384 & 215 \\
+qc6.subtracted & 384 & 384 & 239 \\
+\bottomrule
+\end{tabular}