From 4b235425151990e40221eade36da4ad995e9c4e5 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 18:38:24 +0000 Subject: [PATCH 01/17] Moving tests into pcpostprocess module --- .gitignore | 28 +++++++++++++++--- {tests => pcpostprocess/tests}/__init__.py | 0 .../tests}/test_directory_builder.py | 0 .../tests}/test_herg_qc.py | 0 .../tests}/test_infer_reversal.py | 0 .../tests}/test_leak_correct.py | 0 .../tests}/test_scripts.py | 0 .../tests}/test_subtraction_plots.py | 0 tests/__pycache__/__init__.cpython-39.pyc | Bin 148 -> 0 bytes .../test_leak_correct.cpython-39.pyc | Bin 1893 -> 0 bytes .../test_trace_class.cpython-39.pyc | Bin 1884 -> 0 bytes 11 files changed, 24 insertions(+), 4 deletions(-) rename {tests => pcpostprocess/tests}/__init__.py (100%) rename {tests => pcpostprocess/tests}/test_directory_builder.py (100%) mode change 100755 => 100644 rename {tests => pcpostprocess/tests}/test_herg_qc.py (100%) rename {tests => pcpostprocess/tests}/test_infer_reversal.py (100%) rename {tests => pcpostprocess/tests}/test_leak_correct.py (100%) rename {tests => pcpostprocess/tests}/test_scripts.py (100%) rename {tests => pcpostprocess/tests}/test_subtraction_plots.py (100%) delete mode 100644 tests/__pycache__/__init__.cpython-39.pyc delete mode 100644 tests/__pycache__/test_leak_correct.cpython-39.pyc delete mode 100644 tests/__pycache__/test_trace_class.cpython-39.pyc diff --git a/.gitignore b/.gitignore index e4bda13c..5624d80c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,28 @@ +# Autogenerated by setuptools-wcm pcpostprocess/_version.py -/tests/test_data + +# Tests and test data +.coverage +/test_data /test_output /output -*__pycache__* + +# Compiled python +*.pyc +__pycache__ + +# Installation files *.egg-info -*.DS_Store -.coverage + +# Jupyter notebooks +.ipynb_checkpoints + +# Virtual environments +venv +env + +# DS Store +.DS_Store + +# VS code config +.vscode diff --git a/tests/__init__.py b/pcpostprocess/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to pcpostprocess/tests/__init__.py diff --git a/tests/test_directory_builder.py b/pcpostprocess/tests/test_directory_builder.py old mode 100755 new mode 100644 similarity index 100% rename from tests/test_directory_builder.py rename to pcpostprocess/tests/test_directory_builder.py diff --git a/tests/test_herg_qc.py b/pcpostprocess/tests/test_herg_qc.py similarity index 100% rename from tests/test_herg_qc.py rename to pcpostprocess/tests/test_herg_qc.py diff --git a/tests/test_infer_reversal.py b/pcpostprocess/tests/test_infer_reversal.py similarity index 100% rename from tests/test_infer_reversal.py rename to pcpostprocess/tests/test_infer_reversal.py diff --git a/tests/test_leak_correct.py b/pcpostprocess/tests/test_leak_correct.py similarity index 100% rename from tests/test_leak_correct.py rename to pcpostprocess/tests/test_leak_correct.py diff --git a/tests/test_scripts.py b/pcpostprocess/tests/test_scripts.py similarity index 100% rename from tests/test_scripts.py rename to pcpostprocess/tests/test_scripts.py diff --git a/tests/test_subtraction_plots.py b/pcpostprocess/tests/test_subtraction_plots.py similarity index 100% rename from tests/test_subtraction_plots.py rename to pcpostprocess/tests/test_subtraction_plots.py diff --git a/tests/__pycache__/__init__.cpython-39.pyc b/tests/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 14f7afb5b743717f2804d56a289ddf0755e32668..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmYe~<>g`k0;z3%sUZ3>h(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vKKO;XkRX-~~ zwNgJhKP6SaAR|TJFEKAOKQG=Tu_Q4*peR2%wYXTnB(=DtSU)~KGcU6wK3=b&@)n0p RZhlH>PO2Tq$j?B`00700B9Z_A diff --git a/tests/__pycache__/test_leak_correct.cpython-39.pyc b/tests/__pycache__/test_leak_correct.cpython-39.pyc deleted file mode 100644 index 8f6fb2e084b9c19b693e6fa5e57d7c11a75e707b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1893 zcmah~&2Jk;6rb6hU9Z=65)vZu-B7iVMPSFN5SIvr8U+quwQ3O3UPjxUNwUt`Yi7nR zwUSeN?G16ZB`*Cx_$%hZffN4%2f};1CUv4njJ0pxn~(S2?|qDuRx2bh{Q@wA<%&T~aKp^VJc*ZxiGkWGcM?QevU{DvKXuPB#hbYe) za{_ZHntM10v*to>E#wx;d_@p|vC_Am0UX-aapEE|iCyj9>vnf{cJJ};f86B<2h&^H zB$-MREmblcZ*A{JeE0O*Der#X-s!e?zL?(mp9AfIE=IOdnkXh_JTdWFDHu!>PxD0U z>^BHl_IsU>1ISwg^R`GExR?#4j)NyfZjxiEqgCq^+OlzCPAnS~*~o^KS}rnWefe{S zkb+@yDq%s}2Kvn(7n70D)-lS4udUmaV`+Gj=e%?VuB8*GmJ!ATcS_r~f$YM*-9(`n zJuVU@_`@E5lH?N!iXZkO*D@{hqqub$&eEki27n)BxqL;`dr%L5-|w6hL)jS=@>wS> zgzSt@M5mvOGC21&WPv|WMJl!KprVTMh&GE^nsGC5yd)Wm}f)o4`QvkQtqmb84KKdq(HbVy|dFVrm0+)mupL zLKPbbgUC`7ISO4-ZzDmV6?q@eRVl~VEe^j6s~R5yLPI*;y6$5c!lmO^+(pZXymWUP z7(t<@oT{h5bqBiyW@uxeOt6s1K2Rgd{@px;RS? zD^y`E!k-n06MC^HDjZj94$q0PAJ)pivP>?DP_Z0pK) p72F1H1PHH(4=S2epCGxxAsg&fh`z*PEsH0BCU4TfUH>eg{{mBHVYIL$~z?aCr%S0{2{WtltjBA=p;lhsB+RJT0M!YBlpn>9wws7Z-v ziqC!L~9}(cFK1+5$j^@nl2e7UlZ>0+H1mVf(;@VRg9&9(>b26-ry1 zM_Hm&9}T`44ls}(as*SI0MTSfOKSTQhF6n ziT3Ao&aP3b#tHbZ9UD(W0Ls*HiFu+Ev(|gm>mBSLJc_^l;UIqcboQRoNhzrjN9LcybqcA`|;jIJ`#Jw zTuk@+ITw4y1>ZYP#wqAMfm_8VGVcqe_RwxcdLH(BBs=Om+tdp-syEahc0H ztPT8Z^Yl)scCGrIf;He&0Mi!iV<;sgo-F8`mUKxL3|0oK1FHiol&EyISNbq(bGD@T zQ8=?Ag%aLM-7;8`C*-GxKeC0l@b75(Z?s;12OjIcQ2Bul%LcaGl7W5=X0WKwgR;rN zBKV+Xi@O7`jgVM`^I+JXI}pd(JOmklfd@akpcj-KUbiO#fDHTtSsko2ZaFvwkpqN4 z+TqPx>8&Qju)ZYFc44`pH;4DiEsT!0mT*@qy+5xZXIrs-e1o`=2>x07t1IZVOzb>+ zB}AblqQW?j_j|@2NHHD27RH}ncA4?|d6rA#C0TKiM75)dlwzzS_cR@eS?774>0}^w zp!zxo&8T)OiCUs^QFTFgTW)|w(-;UHUsi`q9XA^6DAPtZ$+AjYnGpI~NC56c7&b19 z1OH5rs`GT5YGFJeBPG|+q*G+t1Zr}IF9-k)+6Ij3Oqh;yPgT+c$`2eu?gdNl=Df3d^3(;PLizp zBOick`56*?5>^OCCh;Z3{3sU)1lO>V?mgp;C!=C2-v_Y?CQv_gZ&PW)QKGRVS$ZaI zcYX*;QEMdhMb6c(O_{`Jx=OC2rpnatg~9{q2jj+P3StF0cBV44uq85nI>sI{4tO&G mrr-#lSd4Qec6$|@@8I+5-d`XZ6jswF8IswAWH%w1>(0L Date: Wed, 12 Nov 2025 18:54:04 +0000 Subject: [PATCH 02/17] Updated flake8 and other configs --- .flake8 | 3 ++- pyproject.toml | 10 +++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.flake8 b/.flake8 index 1878446e..ec5b2da4 100644 --- a/.flake8 +++ b/.flake8 @@ -16,4 +16,5 @@ ignore = exclude= .git, venv, - tests/test_data, + test_data, + test_output, diff --git a/pyproject.toml b/pyproject.toml index 5ec2471f..fc30cf1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [build-system] requires = [ - 'setuptools>=61', - 'wheel', - 'setuptools_scm[toml]>=8.0', + 'setuptools>=61', + 'wheel', + 'setuptools_scm[toml]>=8.0', ] build-backend = 'setuptools.build_meta' @@ -64,8 +64,6 @@ pcpostprocess = 'pcpostprocess.scripts.__main__:main' [tool.setuptools.packages.find] include = [ 'pcpostprocess', - 'pcpostprocess.*', - 'pcpostprocess.scripts', ] [tool.isort] @@ -74,7 +72,5 @@ skip = ['_version.py'] [tool.coverage.run] source = [ 'pcpostprocess', - 'pcpostprocess.scripts', - 'tests', ] From a4cef475b7eae28753e2433ce031da536494848f Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 18:54:14 +0000 Subject: [PATCH 03/17] Simplified directory builder tests --- pcpostprocess/tests/test_directory_builder.py | 60 +++++++------------ 1 file changed, 21 insertions(+), 39 deletions(-) mode change 100644 => 100755 pcpostprocess/tests/test_directory_builder.py diff --git a/pcpostprocess/tests/test_directory_builder.py b/pcpostprocess/tests/test_directory_builder.py old mode 100644 new mode 100755 index 8096d20e..875f708e --- a/pcpostprocess/tests/test_directory_builder.py +++ b/pcpostprocess/tests/test_directory_builder.py @@ -3,57 +3,39 @@ import re import tempfile import unittest -from contextlib import ContextDecorator -from unittest.mock import patch from pcpostprocess import directory_builder -class temp_cwd(ContextDecorator): - def __enter__(self): - self.old = os.getcwd() - self.tmpdir = tempfile.TemporaryDirectory() - os.chdir(self.tmpdir.name) - return self.tmpdir +store_output = False - def __exit__(self, *exc): - os.chdir(self.old) - self.tmpdir.cleanup() - return False - -def make_info_dict(lines): - return {l.split(": ", 1)[0]: l.split(": ", 1)[1] - for l in lines - if len(l.split(": ", 1)) == 2} +def read_info_dict(path): + """ Reads a pcpostproces_info.txt and returns it as a dict. """ + with open(path, 'r') as f: + items = [line.strip().split(': ') for line in f.readlines()] + return dict(line for line in items if len(line) == 2) class TestDirectoryBuilder(unittest.TestCase): - def test_directory_with_git(self): - test_dir = directory_builder.setup_output_directory("test_output", - self.__class__.__name__) - - # Check that git commit is there - with open(os.path.join(test_dir, "pcpostprocess_info.txt"), "r") as fin: - info_file_contents = [l.strip() for l in fin.readlines()] + """ Tests the DirectoryBuilder class. """ - info_dict = make_info_dict(info_file_contents) + def test_directory_builder(self): + # Test that a pcpostprocess_info is written with a real git commit - # REGEX to check if the relevant line contains a git commit - self.assertTrue(bool(re.fullmatch(r"g[0-9a-fA-F]{9}", - info_dict["Commit"]))) - return + with tempfile.TemporaryDirectory() as d: + if store_output: # pragma: no cover + d = 'test_output' + path = directory_builder.setup_output_directory( + d, 'TestDirectoryBuilder') - @temp_cwd() - def test_directory_with_dev_version(self): - with patch("pcpostprocess.directory_builder.__commit_id__", "a000"): - test_dir = directory_builder.setup_output_directory("test_output", - self.__class__.__name__) + # Check that git commit is written + info_dict = read_info_dict( + os.path.join(path, 'pcpostprocess_info.txt')) + self.assertTrue( + re.fullmatch(r'g[0-9a-fA-F]{9}', info_dict['Commit'])) - with open(os.path.join(test_dir, "pcpostprocess_info.txt"), "r") as fin: - info_file_contents = [l.strip() for l in fin.readlines()] - info_dict = make_info_dict(info_file_contents) - self.assertEqual(info_dict["Commit"], "a000") - return +if __name__ == '__main__': + unittest.main() From 667dabe64b24d1e1b1f7ef1c85546b7b8ddad35c Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 22:54:10 +0000 Subject: [PATCH 04/17] Tidied up tests --- CHANGELOG.md | 3 +- pcpostprocess/hergQC.py | 3 +- pcpostprocess/infer_reversal.py | 4 - pcpostprocess/leak_correct.py | 22 +- pcpostprocess/scripts/run_herg_qc.py | 208 +++----- pcpostprocess/tests/test_directory_builder.py | 14 +- pcpostprocess/tests/test_herg_qc.py | 503 ++++++++---------- pcpostprocess/tests/test_infer_reversal.py | 104 ++-- pcpostprocess/tests/test_leak_correct.py | 90 ++-- pcpostprocess/tests/test_scripts.py | 59 +- pcpostprocess/tests/test_subtraction_plots.py | 68 ++- pull_request_template.md | 20 - 12 files changed, 508 insertions(+), 590 deletions(-) delete mode 100644 pull_request_template.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 685a6de4..6b0fef3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,10 @@ This page lists the main changes made to pcpostprocess in each release. ## Unreleased - Added - [#81](https://github.com/CardiacModelling/pcpostprocess/pull/81) Added docstrings to the `hERGQC` class. - - [#104](https://github.com/CardiacModelling/pcpostprocess/pull/104) Added a CHANGELOG.md and CONTRIBUTING.md + - [#104](https://github.com/CardiacModelling/pcpostprocess/pull/104) Added a CHANGELOG.md and CONTRIBUTING.md. - Changed - [#81](https://github.com/CardiacModelling/pcpostprocess/pull/81) Changed the constructor arguments for `hERGQC`. + - [#122](https://github.com/CardiacModelling/pcpostprocess/pull/122) `fit_linear_leak` no longer accepts an `output_dir` argument. - Deprecated - Removed - [#81](https://github.com/CardiacModelling/pcpostprocess/pull/81) Removed `hERGQC.plot_dir`, `hERGQC.set_trace` and `hERGQC.set_debug`. diff --git a/pcpostprocess/hergQC.py b/pcpostprocess/hergQC.py index 9f4f9311..5740f3c6 100644 --- a/pcpostprocess/hergQC.py +++ b/pcpostprocess/hergQC.py @@ -499,8 +499,7 @@ def qc6(self, recording1, win, label=None): i, f = win val = np.mean(recording1[i:f]) valc = self.negative_tolc * np.std(recording1[:self.noise_len]) - if (val < valc) or not (np.isfinite(val) - and np.isfinite(valc)): + if (val < valc) or not (np.isfinite(val) and np.isfinite(valc)): self.logger.debug(f'qc6_{label} val: {val}, valc: {valc}') result = False else: diff --git a/pcpostprocess/infer_reversal.py b/pcpostprocess/infer_reversal.py index 26fcf285..48761e03 100644 --- a/pcpostprocess/infer_reversal.py +++ b/pcpostprocess/infer_reversal.py @@ -1,5 +1,4 @@ import logging -import os import matplotlib.pyplot as plt import numpy as np @@ -65,9 +64,6 @@ def infer_reversal_potential(current, times, voltage_segments, voltages, # Optional plot if output_path is not None: - dirname = os.path.dirname(output_path) - if not os.path.exists(dirname): - os.makedirs(dirname) fig = plt.figure(figsize=figsize) ax = fig.subplots() diff --git a/pcpostprocess/leak_correct.py b/pcpostprocess/leak_correct.py index c1fc76d1..2b9319f1 100644 --- a/pcpostprocess/leak_correct.py +++ b/pcpostprocess/leak_correct.py @@ -1,6 +1,6 @@ -import logging -import os - +# +# Leak correction methods +# import numpy as np from matplotlib import pyplot as plt @@ -55,7 +55,7 @@ def get_leak_corrected(current, voltages, times, ramp_start_index, def fit_linear_leak(current, voltage, times, ramp_start_index, ramp_end_index, - save_fname=None, output_dir=None, figsize=(5.54, 7)): + save_fname=None, figsize=(5.54, 7)): """ Fits linear leak to a leak ramp, returning @@ -64,8 +64,6 @@ def fit_linear_leak(current, voltage, times, ramp_start_index, ramp_end_index, @param ramp_start_index: the index of the observation where the leak ramp begins @param ramp_end_index: the index of the observation where the leak ramp ends @param save_fname: if set, a debugging figure will be made and stored with this name - @param output_dir: if ``save_fname`` is set, this directory will be used to store - the figure, and created if it does not exist @param figsize: if ``save_fname`` is set, the figure size. @return: the linear regression parameters obtained from fitting the leak @@ -152,14 +150,8 @@ def fit_linear_leak(current, voltage, times, ramp_start_index, ramp_end_index, alpha=0.5, label=r'$I_\mathrm{obs} - I_\mathrm{L}$') ax4.legend(frameon=False) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - if output_dir: - try: - fig.savefig(os.path.join(output_dir, save_fname)) - plt.close(fig) - except Exception as exc: - logging.warning(str(exc)) + if save_fname is not None: + fig.savefig(save_fname) + plt.close(fig) return (b_0, b_1), I_leak diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index ff37ff9c..a971f347 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -28,6 +28,18 @@ from pcpostprocess.subtraction_plots import do_subtraction_plot +def starmap(n, func, iterable): + """ + Like ``multiprocessing.Pool.starmap``, but does not use subprocesses when + n=1. + """ + if n == 1: + return [func(*args) for args in iterable] + else: + with multiprocessing.Pool(n, maxtasksperchild=1) as pool: + return pool.starmap(func, iterable) + + def run_from_command_line(): """ Reads arguments from the command line and an ``export_config.py`` and then @@ -65,7 +77,6 @@ def run_from_command_line(): parser.add_argument('--figsize', nargs=2, type=int, default=[16, 18]) - parser.add_argument('--debug', action='store_true') parser.add_argument('--log_level', default='INFO') args = parser.parse_args() @@ -93,7 +104,6 @@ def run_from_command_line(): reversal_spread_threshold=args.reversal_spread_threshold, max_processes=args.no_cpus, figure_size=args.figsize, - debug=args.debug, save_id=export_config.saveID ) @@ -101,8 +111,7 @@ def run_from_command_line(): def run(data_path, output_path, qc_map, wells=None, write_traces=False, write_failed_traces=False, write_map={}, reversal_potential=-90, reversal_spread_threshold=10, - max_processes=1, figure_size=None, - debug=False, save_id=None): + max_processes=1, figure_size=None, save_id=None): """ Imports traces and runs QC. @@ -226,21 +235,18 @@ def run(data_path, output_path, qc_map, wells=None, if not readnames: raise ValueError('No compatible protocols specified.') - n = min(1, max_processes, len(readnames)) - with multiprocessing.Pool(n, maxtasksperchild=1) as pool: - - pool_argument_list = zip( - readnames, - savenames, - times_list, - [output_path for i in readnames], - [data_path for i in readnames], - [wells for i in readnames], - [write_traces for i in readnames], - [save_id] * len(readnames), - ) - well_selections, qc_dfs = list(zip( - *pool.starmap(run_qc_for_protocol, pool_argument_list))) + n = min(max_processes, len(readnames)) + args = zip( + readnames, + savenames, + times_list, + [output_path] * len(readnames), + [data_path] * len(readnames), + [wells] * len(readnames), + [write_traces] * len(readnames), + [save_id] * len(readnames), + ) + well_selections, qc_dfs = list(zip(*starmap(n, run_qc_for_protocol, args))) qc_df = pd.concat(qc_dfs, ignore_index=True) @@ -251,7 +257,7 @@ def run(data_path, output_path, qc_map, wells=None, if len(times) == 4: qc3_bookend_dict = qc3_bookend( protocol, savename, times, wells, output_path, data_path, - debug, figure_size, save_id) + figure_size, save_id) else: qc3_bookend_dict = {well: True for well in qc_df.well.unique()} @@ -274,8 +280,7 @@ def run(data_path, output_path, qc_map, wells=None, overall_selection = [] for well in qc_df.well.unique(): failed = False - for well_selection, protocol in zip(well_selections, - list(savenames)): + for well_selection, protocol in zip(well_selections, list(savenames)): logging.debug(f"{well_selection} selected from protocol {protocol}") fname = os.path.join(output_path, 'selected-%s-%s.txt' % @@ -330,7 +335,8 @@ def run(data_path, output_path, qc_map, wells=None, no_protocols = len(res_dict) - args_list = list(zip( + n = min(max_processes, no_protocols) + args = zip( readnames, savenames, times_list, @@ -341,11 +347,8 @@ def run(data_path, output_path, qc_map, wells=None, [figure_size for i in readnames], [reversal_potential for i in readnames], [save_id for i in readnames], - )) - - n = min(1, max_processes, no_protocols) - with multiprocessing.Pool(n, maxtasksperchild=1) as pool: - dfs = list(pool.starmap(extract_protocol, args_list)) + ) + dfs = starmap(n, extract_protocol, args) if dfs: extract_df = pd.concat(dfs, ignore_index=True) @@ -554,22 +557,12 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, logging.info(f"extracting {savename}") traces_dir = os.path.join(savedir, 'traces') - - if not os.path.exists(traces_dir): - try: - os.makedirs(traces_dir) - except FileExistsError: - pass + os.makedirs(traces_dir, exist_ok=True) row_dict = {} subtraction_plots_dir = os.path.join(savedir, 'subtraction_plots') - - if not os.path.isdir(subtraction_plots_dir): - try: - os.makedirs(subtraction_plots_dir) - except FileExistsError: - pass + os.makedirs(subtraction_plots_dir, exist_ok=True) logging.info(f"Exporting {readname} as {savename}") @@ -659,13 +652,21 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, # plot subtraction fig = plt.figure(figsize=figure_size, layout='constrained') - reversal_plot_dir = os.path.join(savedir, 'reversal_plots') + plot_dir = os.path.join(savedir, 'reversal_plots') + os.makedirs(plot_dir, exist_ok=True) rows = [] before_leak_current_dict = {} after_leak_current_dict = {} + out1 = os.path.join(savedir, 'leak_correction', + f'{save_id}-{savename}-leak_fit-before') + out2 = os.path.join(savedir, 'leak_correction', + f'{save_id}-{savename}-leak_fit-after') + os.makedirs(out1, exist_ok=True) + os.makedirs(out2, exist_ok=True) + for well in selected_wells: before_current = before_trace.get_trace_sweeps()[well] after_current = after_trace.get_trace_sweeps()[well] @@ -673,9 +674,6 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, before_leak_currents = [] after_leak_currents = [] - out_path = os.path.join(savedir, "leak_correction", - f"{save_id}-{savename}-leak_fit-before") - for sweep in range(before_current.shape[0]): row_dict = { 'well': well, @@ -693,26 +691,18 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, row_dict['Cm'] = qc_vals[1] row_dict['Rseries'] = qc_vals[2] - before_params, before_leak = fit_linear_leak(before_current[sweep, :], - voltages, times, - *ramp_bounds, - output_dir=out_path, - save_fname=f"{well}_sweep{sweep}.png" - ) - + before_params, before_leak = fit_linear_leak( + before_current[sweep], voltages, times, *ramp_bounds, + save_fname=os.path.join(out1, f'{well}_sweep{sweep}.png')) before_leak_currents.append(before_leak) - out_path = os.path.join(savedir, - f"{save_id}-{savename}-leak_fit-after") # Convert linear regression parameters into conductance and reversal row_dict['gleak_before'] = before_params[1] row_dict['E_leak_before'] = -before_params[0] / before_params[1] - after_params, after_leak = fit_linear_leak(after_current[sweep, :], - voltages, times, - *ramp_bounds, - save_fname=f"{well}_sweep{sweep}.png", - output_dir=out_path) + after_params, after_leak = fit_linear_leak( + after_current[sweep, :], voltages, times, *ramp_bounds, + save_fname=os.path.join(out2, f'{well}_sweep{sweep}.png')) after_leak_currents.append(after_leak) @@ -727,17 +717,20 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, E_rev_before = infer_reversal_potential( before_corrected, times, desc, voltages, - output_path=os.path.join(reversal_plot_dir, f"{well}_{savename}_sweep{sweep}_before"), + output_path=os.path.join( + plot_dir, f'{well}_{savename}_sweep{sweep}_before'), known_Erev=reversal_potential) E_rev_after = infer_reversal_potential( after_corrected, times, desc, voltages, - output_path=os.path.join(reversal_plot_dir, f"{well}_{savename}_sweep{sweep}_after"), + output_path=os.path.join( + plot_dir, f'{well}_{savename}_sweep{sweep}_after'), known_Erev=reversal_potential) E_rev = infer_reversal_potential( subtracted_trace, times, desc, voltages, - output_path=os.path.join(reversal_plot_dir, f"{well}_{savename}_sweep{sweep}_subtracted"), + output_path=os.path.join( + plot_dir, f'{well}_{savename}_sweep{sweep}_subtracted'), known_Erev=reversal_potential) row_dict['R_leftover'] =\ @@ -753,15 +746,8 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, row_dict['QC.Erev'] = E_rev < -50 and E_rev > -120 # Check QC6 for each protocol (not just the staircase) - plot_dir = os.path.join(savedir, "QC", "debug", - f"debug_{well}_{savename}") - if not os.path.exists(plot_dir): - os.makedirs(plot_dir) - - hergqc = hERGQC(sampling_rate=before_trace.sampling_rate, - voltage=voltage, - plot_dir=plot_dir) + hergqc = hERGQC(voltage, before_trace.sampling_rate) times = before_trace.get_times() voltage = before_trace.get_voltage() @@ -774,9 +760,7 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, current = hergqc.filter_capacitive_spikes(before_corrected - after_corrected, times, voltage_steps) - row_dict['QC6'] = hergqc.qc6(current, - win=hergqc.qc6_win, - label='0')[0] + row_dict['QC6'] = hergqc.qc6(current, win=hergqc.qc6_win)[0] #  Assume there is only one sweep for all non-QC protocols rseal_before, cm_before, rseries_before = qc_before[well][0] @@ -857,11 +841,7 @@ def extract_protocol(readname, savename, time_strs, selected_wells, savedir, plt.close(fig) protocol_dir = os.path.join(traces_dir, 'protocols') - if not os.path.exists(protocol_dir): - try: - os.makedirs(protocol_dir) - except FileExistsError: - pass + os.makedirs(protocol_dir, exist_ok=True) # extract protocol protocol = before_trace.get_voltage_protocol() @@ -903,10 +883,8 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, sampling_rate = before_trace.sampling_rate savedir = os.path.join(output_path, 'QC') - leak_correction_dir = os.path.join(savedir, "leak_correction") - - if not os.path.exists(savedir): - os.makedirs(savedir) + plot_dir = os.path.join(savedir, 'leak_correction') + os.makedirs(plot_dir, exist_ok=True) before_voltage = before_trace.get_voltage() after_voltage = after_trace.get_voltage() @@ -921,26 +899,17 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, raw_after_all = after_trace.get_trace_sweeps(sweeps) selected_wells = [] - for well in wells: - - plot_dir = os.path.join(savedir, "debug", f"debug_{well}_{savename}") - - if not os.path.exists(plot_dir): - os.makedirs(plot_dir) - # Setup QC instance. We could probably just do this inside the loop - hergqc = hERGQC(sampling_rate=sampling_rate, - plot_dir=plot_dir, - voltage=before_voltage) + hergqc = hERGQC(before_voltage, sampling_rate) - qc_before = before_trace.get_onboard_QC_values() - qc_after = after_trace.get_onboard_QC_values() + qc_before = before_trace.get_onboard_QC_values() + qc_after = after_trace.get_onboard_QC_values() + for well in wells: # Check if any cell first! if (None in qc_before[well][0]) or (None in qc_after[well][0]): no_cell = True continue - else: no_cell = False @@ -954,8 +923,8 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, after_currents = np.empty((nsweeps, after_trace.NofSamples)) # Get ramp times from protocol description - voltage_protocol = VoltageProtocol.from_voltage_trace(voltage, - before_trace.get_times()) + voltage_protocol = VoltageProtocol.from_voltage_trace( + voltage, before_trace.get_times()) #  Find start of leak section desc = voltage_protocol.get_all_sections() @@ -973,19 +942,15 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, before_raw = np.array(raw_before_all[well])[sweep, :] after_raw = np.array(raw_after_all[well])[sweep, :] - before_params1, before_leak = fit_linear_leak(before_raw, - voltage, - times, - *ramp_bounds, - save_fname=f"{well}-sweep{sweep}-before.png", - output_dir=leak_correction_dir) + before_params1, before_leak = fit_linear_leak( + before_raw, voltage, times, *ramp_bounds, + save_fname=os.path.join( + plot_dir, f'{well}-sweep{sweep}-before.png')) - after_params1, after_leak = fit_linear_leak(after_raw, - voltage, - times, - *ramp_bounds, - save_fname=f"{well}-sweep{sweep}-after.png", - output_dir=leak_correction_dir) + after_params1, after_leak = fit_linear_leak( + after_raw, voltage, times, *ramp_bounds, + save_fname=os.path.join( + plot_dir, f'{well}-sweep{sweep}-after.png')) before_currents_corrected[sweep, :] = before_raw - before_leak after_currents_corrected[sweep, :] = after_raw - after_leak @@ -1027,8 +992,7 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, subtracted_current = before_currents_corrected[i, :] - after_currents_corrected[i, :] if write_traces: - if not os.path.exists(savedir): - os.makedirs(savedir) + os.makedirs(savedir, exist_ok=True) np.savetxt(savepath, subtracted_current, delimiter=',', comments='', header=header) @@ -1057,7 +1021,7 @@ def run_qc_for_protocol(readname, savename, time_strs, output_path, def qc3_bookend(readname, savename, time_strs, wells, output_path, - data_path, debug, figure_size, save_id): + data_path, figure_size, save_id): filepath_first_before = os.path.join(data_path, f'{readname}_{time_strs[0]}') filepath_last_before = os.path.join(data_path, f'{readname}_{time_strs[1]}') @@ -1116,20 +1080,6 @@ def qc3_bookend(readname, savename, time_strs, wells, output_path, save_fname = f"{well}_{savename}_before0.pdf" - if debug: - qc3_output_dir = os.path.join( - output_path, 'QC', 'debug', f'{well}_{savename}', 'qc3_bookend') - - if not os.path.exists(qc3_output_dir): - os.makedirs(qc3_output_dir) - - leak_correct_dir = os.path.join(qc3_output_dir, 'leak_correction') - - #  Plot subtraction - get_leak_corrected( - first_before_current, voltage, times, *ramp_bounds, - save_fname=save_fname, output_dir=leak_correct_dir) - before_traces_first[well] = get_leak_corrected(first_before_current, voltage, times, *ramp_bounds) @@ -1153,8 +1103,7 @@ def qc3_bookend(readname, savename, time_strs, wells, output_path, plot_dir = os.path.join(output_path, output_path, f'{save_id}-{savename}-qc3-bookend') - if not os.path.exists(plot_dir): - os.makedirs(plot_dir) + os.makedirs(plot_dir, exist_ok=True) hergqc = hERGQC(sampling_rate=first_before_trace.sampling_rate, plot_dir=plot_dir, voltage=voltage) @@ -1198,8 +1147,7 @@ def get_time_constant_of_first_decay( """ ??? """ - if not os.path.exists(os.path.dirname(output_path)): - os.makedirs(os.path.dirname(output_path)) + os.makedirs(os.path.dirname(output_path), exist_ok=True) first_120mV_step_index = [ i for i, line in enumerate(protocol_desc) if line[2] == 40][0] + 1 @@ -1245,7 +1193,7 @@ def fit_func(x, args=None): ] # TESTING ONLY - # np.random.seed(1) + np.random.seed(1) #  Repeat optimisation with different starting guesses x0s = [[np.random.uniform(lower_b, upper_b) for lower_b, upper_b in bounds] for i in range(100)] diff --git a/pcpostprocess/tests/test_directory_builder.py b/pcpostprocess/tests/test_directory_builder.py index 875f708e..9ba7d857 100755 --- a/pcpostprocess/tests/test_directory_builder.py +++ b/pcpostprocess/tests/test_directory_builder.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import os -import re +import sys import tempfile import unittest @@ -21,21 +21,25 @@ class TestDirectoryBuilder(unittest.TestCase): """ Tests the DirectoryBuilder class. """ def test_directory_builder(self): - # Test that a pcpostprocess_info is written with a real git commit + # Test that a pcpostprocess_info.txt is written, with a commit hash with tempfile.TemporaryDirectory() as d: if store_output: # pragma: no cover d = 'test_output' path = directory_builder.setup_output_directory( - d, 'TestDirectoryBuilder') + d, 'directory_builder') # Check that git commit is written info_dict = read_info_dict( os.path.join(path, 'pcpostprocess_info.txt')) - self.assertTrue( - re.fullmatch(r'g[0-9a-fA-F]{9}', info_dict['Commit'])) + self.assertRegex(info_dict['Commit'], r'^g[0-9a-fA-F]{9}$') if __name__ == '__main__': + if '-store' in sys.argv: + store_output = True + sys.argv.remove('-store') + else: + print('Add -store to store output') unittest.main() diff --git a/pcpostprocess/tests/test_herg_qc.py b/pcpostprocess/tests/test_herg_qc.py index e2a4da54..8986a66e 100755 --- a/pcpostprocess/tests/test_herg_qc.py +++ b/pcpostprocess/tests/test_herg_qc.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os import string +import sys +import tempfile import unittest import numpy as np @@ -9,17 +11,25 @@ from pcpostprocess.hergQC import hERGQC +store_output = False + + def all_passed(result): + """ Checks a tuple-list (pass, message) is all passes. """ return all([x for x, _ in result]) class TestHergQC(unittest.TestCase): + """ + Tests the hERGQC class methods using syncropatch test data. + """ - def setUp(self): - base_path = os.path.join("tests", "test_data", "13112023_MW2_FF") + @classmethod + def setUpClass(self): + base_path = os.path.join('test_data', '13112023_MW2_FF') - label_before = "staircaseramp (2)_2kHz_15.01.07" - label_after = "staircaseramp (2)_2kHz_15.11.33" + label_before = 'staircaseramp (2)_2kHz_15.01.07' + label_after = 'staircaseramp (2)_2kHz_15.11.33' path_before = os.path.join(base_path, label_before) path_after = os.path.join(base_path, label_after) @@ -47,7 +57,7 @@ def setUp(self): # in kHz self.sampling_rate = int(1 / (self.times[1] - self.times[0])) - #  Assume that there are no discontinuities at the start or end of ramps + #  Assume no discontinuities at the start or end of ramps voltage_protocol = trace_before.get_voltage_protocol() self.voltage_steps = [ tstart @@ -55,50 +65,49 @@ def setUp(self): if vend == vstart ] - def create_hergqc(self, plot_dir=None): - """ - Creates and returns a hERGQC object. - - If a ``plot_dir`` is set this will be used for debug output - """ - # Set this to True to generate plot directories - debug = False - - if debug and plot_dir is not None: - plot_dir = os.path.join('test_output', plot_dir) + # Store, or test plot generation in a temporary directory + if store_output: # pragma: no cover + self.temp_dir = None + plot_dir = os.path.join('test_output', 'qc') os.makedirs(plot_dir, exist_ok=True) else: - plot_dir = None + self.temp_dir = tempfile.TemporaryDirectory() + plot_dir = self.temp_dir.name + + self.qc = hERGQC(self.voltage, self.sampling_rate, plot_dir=plot_dir) - return hERGQC( - sampling_rate=self.sampling_rate, - plot_dir=plot_dir, - voltage=self.voltage, - ) + @classmethod + def tearDownClass(self): + if self.temp_dir is not None: + self.temp_dir.cleanup() def test_qc_inputs(self): - hergqc = self.create_hergqc() - times = self.times + # Tests... that the test data is what we think it should be? + # That hardcoded properties are what they are??? + # This test _MIGHT_ make sense once windows are changeable, but at the + # moment is pointless # TODO: This should work some nicer way, without accessing what should # really be private properties. But first we probably need to # stop hardcoding these windows - voltage = hergqc.voltage - qc6_win = hergqc.qc6_win - qc6_1_win = hergqc.qc6_1_win - qc6_2_win = hergqc.qc6_2_win + voltage = self.qc.voltage + qc6_win = self.qc.qc6_win + qc6_1_win = self.qc.qc6_1_win + qc6_2_win = self.qc.qc6_2_win - self.assertTrue(np.all(np.isfinite(times))) + self.assertTrue(np.all(np.isfinite(self.times))) self.assertTrue(np.all(np.isfinite(voltage))) # Ensures that the windows are correct by checking the voltage trace - assert np.all(np.abs(voltage[qc6_win[0]: qc6_win[1]] - 40.0)) < 1e-8 - assert np.all(np.abs(voltage[qc6_1_win[0]: qc6_1_win[1]] - 40.0)) < 1e-8 - assert np.all(np.abs(voltage[qc6_2_win[0]: qc6_2_win[1]] - 40.0)) < 1e-8 + self.assertTrue( + np.all(np.abs(voltage[qc6_win[0]: qc6_win[1]] - 40.0) < 1e-8)) + self.assertTrue( + np.all(np.abs(voltage[qc6_1_win[0]: qc6_1_win[1]] - 40.0) < 1e-8)) + self.assertTrue( + np.all(np.abs(voltage[qc6_2_win[0]: qc6_2_win[1]] - 40.0) < 1e-8)) def test_qc1(self): # qc1 checks that rseal, cm, rseries are within range - hergqc = self.create_hergqc('qc1') rseal_lo, rseal_hi = 1e8, 1e12 rseal_mid = (rseal_lo + rseal_hi) / 2 @@ -134,13 +143,13 @@ def test_qc1(self): for (rseal, cm, rseries), ex_pass in test_matrix: self.assertEqual( - all_passed(hergqc.qc1(rseal, cm, rseries)), + all_passed(self.qc.qc1(rseal, cm, rseries)), ex_pass, - f"QC1: {rseal}, {cm}, {rseries}", + f'QC1: {rseal}, {cm}, {rseries}', ) # Test on data - values before - failed_wells_rseal_before = [ + failed_wells_rseal_before = { 'A10', 'A12', 'A13', 'A16', 'A19', 'A20', 'B05', 'B07', 'B11', 'B12', 'B13', 'B15', 'B19', 'B21', 'B23', 'C02', 'C04', 'C07', 'C09', 'C11', 'C12', 'C14', 'C18', 'C19', 'C20', 'D02', 'D03', 'D05', 'D09', 'D10', @@ -158,9 +167,9 @@ def test_qc1(self): 'N16', 'N18', 'N21', 'N24', 'O01', 'O02', 'O03', 'O05', 'O07', 'O10', 'O11', 'O17', 'O19', 'O22', 'O24', 'P01', 'P03', 'P06', 'P07', 'P08', 'P09', 'P12', 'P13', 'P14', 'P15', 'P17', 'P18', 'P21', 'P22', 'P24' - ] + } - failed_wells_cm_before = [ + failed_wells_cm_before = { 'A12', 'A13', 'A16', 'A19', 'B07', 'B11', 'B13', 'B15', 'B21', 'B23', 'C02', 'C04', 'C07', 'C11', 'C12', 'C14', 'C18', 'C20', 'D03', 'D10', 'D14', 'E04', 'E07', 'E10', 'E15', 'E16', 'E17', 'E22', 'E23', 'F01', @@ -173,9 +182,9 @@ def test_qc1(self): 'N11', 'N14', 'N18', 'N21', 'N24', 'O01', 'O03', 'O07', 'O10', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P07', 'P08', 'P12', 'P13', 'P14', 'P15', 'P16', 'P18', 'P21', 'P22' - ] + } - failed_wells_rseries_before = [ + failed_wells_rseries_before = { 'A12', 'A13', 'A16', 'A19', 'A24', 'B07', 'B11', 'B13', 'B15', 'B21', 'B23', 'C02', 'C04', 'C07', 'C11', 'C12', 'C14', 'C18', 'C20', 'C23', 'D03', 'D09', 'D10', 'D14', 'D15', 'D16', 'E04', 'E06', 'E07', 'E10', @@ -189,18 +198,17 @@ def test_qc1(self): 'N06', 'N08', 'N11', 'N14', 'N18', 'N21', 'N24', 'O01', 'O03', 'O07', 'O10', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P07', 'P08', 'P12', 'P13', 'P14', 'P15', 'P16', 'P18', 'P21', 'P22' - ] + } for well in self.all_wells: - qc_vals_before = np.array(self.qc_vals_before[well])[0, :] - result = hergqc.qc1(*qc_vals_before) + result = self.qc.qc1(*self.qc_vals_before[well][0]) pass_rseal_before, rseal_before = result[0] ex_pass_rseal_before = well not in failed_wells_rseal_before self.assertEqual( pass_rseal_before, ex_pass_rseal_before, - f"QC1: {well} (rseal before) {rseal_before}", + f'QC1: {well} (rseal before) {rseal_before}', ) pass_cm_before, cm_before = result[1] @@ -208,7 +216,7 @@ def test_qc1(self): self.assertEqual( pass_cm_before, ex_pass_cm_before, - f"QC1: {well} (cm before) {cm_before}", + f'QC1: {well} (cm before) {cm_before}', ) pass_rseries_before, rseries_before = result[2] @@ -216,11 +224,11 @@ def test_qc1(self): self.assertEqual( pass_rseries_before, ex_pass_rseries_before, - f"QC1: {well} (rseries before) {rseries_before}", + f'QC1: {well} (rseries before) {rseries_before}', ) # Test on data - values after - failed_wells_rseal_after = [ + failed_wells_rseal_after = { 'A10', 'A12', 'A13', 'A16', 'A19', 'A20', 'A24', 'B02', 'B05', 'B07', 'B11', 'B12', 'B13', 'B15', 'B21', 'B23', 'C02', 'C04', 'C07', 'C09', 'C11', 'C12', 'C14', 'C18', 'C20', 'C22', 'D03', 'D05', 'D10', 'D14', @@ -238,9 +246,9 @@ def test_qc1(self): 'O03', 'O07', 'O08', 'O10', 'O11', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P07', 'P08', 'P09', 'P12', 'P13', 'P14', 'P15', 'P17', 'P18', 'P21', 'P22' - ] + } - failed_wells_cm_after = [ + failed_wells_cm_after = { 'A12', 'A13', 'A19', 'A20', 'B07', 'B11', 'B13', 'B15', 'B19', 'B21', 'B23', 'C02', 'C04', 'C11', 'C12', 'C14', 'C18', 'C20', 'D10', 'D14', 'E03', 'E09', 'E10', 'E15', 'E16', 'E17', 'E19', 'E22', 'E23', 'F01', @@ -254,9 +262,9 @@ def test_qc1(self): 'O01', 'O03', 'O07', 'O10', 'O15', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P08', 'P12', 'P13', 'P14', 'P15', 'P16', 'P17', 'P18', 'P21', 'P22' - ] + } - failed_wells_rseries_after = [ + failed_wells_rseries_after = { 'A06', 'A08', 'A12', 'A13', 'A19', 'A20', 'A24', 'B07', 'B11', 'B13', 'B15', 'B19', 'B21', 'B23', 'C01', 'C02', 'C04', 'C09', 'C11', 'C12', 'C14', 'C18', 'C20', 'C22', 'C23', 'D09', 'D10', 'D14', 'D15', 'D19', @@ -274,18 +282,17 @@ def test_qc1(self): 'O03', 'O05', 'O07', 'O10', 'O11', 'O14', 'O15', 'O17', 'O19', 'O22', 'O24', 'P01', 'P03', 'P06', 'P08', 'P11', 'P12', 'P13', 'P14', 'P15', 'P16', 'P17', 'P18', 'P19', 'P21', 'P22' - ] + } for well in self.all_wells: - qc_vals_after = np.array(self.qc_vals_after[well])[0, :] - result = hergqc.qc1(*qc_vals_after) + result = self.qc.qc1(*self.qc_vals_after[well][0]) pass_rseal_after, rseal_after = result[0] ex_pass_rseal_after = well not in failed_wells_rseal_after self.assertEqual( pass_rseal_after, ex_pass_rseal_after, - f"QC1: {well} (rseal after) {rseal_after}", + f'QC1: {well} (rseal after) {rseal_after}', ) pass_cm_after, cm_after = result[1] @@ -293,7 +300,7 @@ def test_qc1(self): self.assertEqual( pass_cm_after, ex_pass_cm_after, - f"QC1: {well} (cm after) {cm_after}", + f'QC1: {well} (cm after) {cm_after}', ) pass_rseries_after, rseries_after = result[2] @@ -301,13 +308,11 @@ def test_qc1(self): self.assertEqual( pass_rseries_after, ex_pass_rseries_after, - f"QC1: {well} (rseries after) {rseries_after}", + f'QC1: {well} (rseries after) {rseries_after}', ) def test_qc2(self): # qc2 checks that raw and subtracted SNR are above a minimum threshold - hergqc = self.create_hergqc('qc2') - test_matrix = [ (10, True, 8082.1), (1, True, 74.0), @@ -319,46 +324,33 @@ def test_qc2(self): ] for i, ex_pass, ex_snr in test_matrix: - recording = np.asarray([0, 0.1] * 100 + [i] * 500) - pass_, snr = hergqc.qc2(recording) + recording = np.array([0, 0.1] * 100 + [i] * 500) + pass_, snr = self.qc.qc2(recording) self.assertAlmostEqual( - snr, ex_snr, 1, f"QC2: ({i}) {snr} != {ex_snr}" - ) - self.assertEqual(pass_, ex_pass, f"QC2: ({i}) {pass_} != {ex_pass}") + snr, ex_snr, 1, f'QC2: ({i}) {snr} != {ex_snr}') + self.assertEqual(pass_, ex_pass, f'QC2: ({i}) {pass_} != {ex_pass}') # Test on data - failed_wells_raw = ["P16"] + failed_wells_raw = ['P16'] failed_wells_subtracted = [ - "B09", "C11", "H19", "H24", "K22", "O16", "P16" - ] + 'B09', 'C11', 'H19', 'H24', 'K22', 'O16', 'P16'] for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) + before = self.trace_sweeps_before[well] + after = self.trace_sweeps_after[well] - raw = [] - subtracted = [] - for i in range(self.n_sweeps): - raw.append(hergqc.qc2(before[i])) - subtracted.append(hergqc.qc2(before[i] - after[i])) + raw = [self.qc.qc2(b) for b in before] + self.assertEqual(all_passed(raw), well not in failed_wells_raw, + f'QC2: {well} (raw) {raw}') - ex_pass_raw = well not in failed_wells_raw - self.assertEqual( - all_passed(raw), - ex_pass_raw, - f"QC2: {well} (raw) {raw}", - ) - - ex_pass_subtracted = well not in failed_wells_subtracted + subtracted = [self.qc.qc2(b - a) for b, a in zip(before, after)] self.assertEqual( all_passed(subtracted), - ex_pass_subtracted, - f"QC2: {well} (subtracted) {subtracted}", - ) + well not in failed_wells_subtracted, + f'QC2: {well} (subtracted) {subtracted}') def test_qc3(self): # qc3 checks that rmsd of two sweeps are similar - hergqc = self.create_hergqc('qc3') # Test with same noise, different signal test_matrix = [ @@ -372,19 +364,14 @@ def test_qc3(self): (10, False, -0.8), ] - recording1 = np.asarray([0, 0.1] * 100 + [40] * 500) + recording1 = np.array([0, 0.1] * 100 + [40] * 500) for i, ex_pass, ex_d_rmsd in test_matrix: - recording2 = np.asarray( - [0, 0.1] * 100 + [40 + i] * 500 - ) - pass_, d_rmsd = hergqc.qc3(recording1, recording2) - self.assertAlmostEqual( - d_rmsd, - ex_d_rmsd, - 1, - f"QC3: ({i}) {d_rmsd} != {ex_d_rmsd}", - ) - self.assertEqual(pass_, ex_pass, f"QC3: ({i}) {pass_} != {ex_pass}") + recording2 = np.array([0, 0.1] * 100 + [40 + i] * 500) + ob_pass, ob_d_rmsd = self.qc.qc3(recording1, recording2) + self.assertAlmostEqual(ob_d_rmsd, ex_d_rmsd, 1, + f'QC3: ({i}) {ob_d_rmsd} != {ex_d_rmsd}') + self.assertEqual(ob_pass, ex_pass, + f'QC3: ({i}) {ob_pass} != {ex_pass}') # Test with same signal, different noise test_matrix = [ @@ -397,19 +384,14 @@ def test_qc3(self): (100, True, 11.4), ] - recording1 = np.asarray([0, 0.1] * 100 + [40] * 500) + recording1 = np.array([0, 0.1] * 100 + [40] * 500) for i, ex_pass, ex_d_rmsd in test_matrix: - recording2 = np.asarray( - [0, 0.1 * i] * 100 + [40] * 500 - ) - pass_, d_rmsd = hergqc.qc3(recording1, recording2) - self.assertAlmostEqual( - d_rmsd, - ex_d_rmsd, - 1, - f"QC3: ({i}) {d_rmsd} != {ex_d_rmsd}", - ) - self.assertEqual(pass_, ex_pass, f"QC3: ({i}) {pass_} != {ex_pass}") + recording2 = np.array([0, 0.1 * i] * 100 + [40] * 500) + ob_pass, ob_d_rmsd = self.qc.qc3(recording1, recording2) + self.assertAlmostEqual(ob_d_rmsd, ex_d_rmsd, 1, + f'QC3: ({i}) {ob_d_rmsd} != {ex_d_rmsd}') + self.assertEqual(ob_pass, ex_pass, + f'QC3: ({i}) {ob_pass} != {ex_pass}') # Test on data failed_wells_raw = [ @@ -433,39 +415,23 @@ def test_qc3(self): ] for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) + before = self.trace_sweeps_before[well] + after = self.trace_sweeps_after[well] - pass_raw, d_rmsd_raw = hergqc.qc3(before[0, :], before[1, :]) - ex_pass_raw = well not in failed_wells_raw - self.assertEqual( - pass_raw, - ex_pass_raw, - f"QC3: {well} (raw) {d_rmsd_raw}", - ) + obt, rmsd = self.qc.qc3(before[0], before[1]) + exp = well not in failed_wells_raw + self.assertEqual(obt, exp, f'QC3: {well} (raw) {rmsd}') - pass_E4031, d_rmsd_E4031 = hergqc.qc3(after[0, :], after[1, :]) - ex_pass_E4031 = well not in failed_wells_E4031 - self.assertEqual( - pass_E4031, - ex_pass_E4031, - f"QC3: {well} (E4031) {d_rmsd_E4031}", - ) + obt, rmsd = self.qc.qc3(after[0], after[1]) + exp = well not in failed_wells_E4031 + self.assertEqual(obt, exp, f'QC3: {well} (E4031) {rmsd}') - pass_subtracted, d_rmsd_subtracted = hergqc.qc3( - before[0, :] - after[0, :], - before[1, :] - after[1, :], - ) - ex_pass_subtracted = well not in failed_wells_subtracted - self.assertEqual( - pass_subtracted, - ex_pass_subtracted, - f"QC3: {well} (subtracted) {d_rmsd_subtracted}", - ) + obt, rmsd = self.qc.qc3(before[0] - after[0], before[1] - after[1]) + exp = well not in failed_wells_subtracted + self.assertEqual(obt, exp, f'QC3: {well} (subtracted) {rmsd}') def test_qc4(self): - # qc4 checks that rseal, cm, rseries are similar before/after E-4031 change - hergqc = self.create_hergqc('qc4') + # qc4 checks that rseal, cm, rseries are similar before/after block r_lo, r_hi = 1e6, 3e7 c_lo, c_hi = 1e-12, 1e-10 @@ -484,16 +450,16 @@ def test_qc4(self): for i, ex_pass in test_matrix: rseals = [r_lo, i * r_lo] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) rseals = [r_hi, i * r_hi] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) # Test cms @@ -510,16 +476,16 @@ def test_qc4(self): for i, ex_pass in test_matrix: cms = [c_lo, i * c_lo] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) cms = [c_hi, i * c_hi] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) # Test rseriess @@ -536,28 +502,28 @@ def test_qc4(self): for i, ex_pass in test_matrix: rseriess = [r_lo, i * r_lo] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) rseriess = [r_hi, i * r_hi] self.assertEqual( - all_passed(hergqc.qc4(rseals, cms, rseriess)), + all_passed(self.qc.qc4(rseals, cms, rseriess)), ex_pass, - f"({i}: {rseals}, {cms}, {rseriess})", + f'({i}: {rseals}, {cms}, {rseriess})', ) # Test on data - failed_wells_rseals = [ + failed_wells_rseals = { 'A04', 'A05', 'A07', 'A16', 'A21', 'A23', 'B02', 'B04', 'B11', 'B16', 'C10', 'C19', 'C22', 'C23', 'D03', 'D23', 'E01', 'E02', 'E03', 'E07', 'F23', 'H01', 'H09', 'H17', 'I06', 'I11', 'J11', 'K01', 'K09', 'K12', 'K14', 'K23', 'M05', 'M10', 'N02', 'N09', 'N17', 'O08', 'O14', 'P16', 'P24' - ] + } - failed_wells_cms = [ + failed_wells_cms = { 'A12', 'A13', 'A16', 'A19', 'A20', 'B07', 'B11', 'B13', 'B15', 'B19', 'B21', 'B23', 'C02', 'C04', 'C07', 'C11', 'C12', 'C14', 'C18', 'D03', 'D10', 'D14', 'E03', 'E04', 'E07', 'E09', 'E10', 'E15', 'E16', 'E17', @@ -571,9 +537,9 @@ def test_qc4(self): 'N18', 'N19', 'N21', 'N24', 'O01', 'O03', 'O07', 'O10', 'O15', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P07', 'P08', 'P12', 'P13', 'P14', 'P15', 'P16', 'P17', 'P18', 'P21', 'P22' - ] + } - failed_wells_rseriess = [ + failed_wells_rseriess = { 'A12', 'A13', 'A16', 'A19', 'A20', 'B07', 'B11', 'B13', 'B15', 'B19', 'B21', 'B23', 'C02', 'C04', 'C07', 'C11', 'C12', 'C14', 'C18', 'C22', 'D03', 'D10', 'D14', 'E01', 'E03', 'E04', 'E07', 'E09', 'E10', 'E15', @@ -588,7 +554,7 @@ def test_qc4(self): 'N18', 'N19', 'N21', 'N24', 'O01', 'O03', 'O07', 'O10', 'O14', 'O15', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P07', 'P08', 'P12', 'P13', 'P14', 'P15', 'P16', 'P17', 'P18', 'P21', 'P22' - ] + } for well in self.all_wells: qc_vals_before = np.array(self.qc_vals_before[well])[0, :] @@ -597,14 +563,14 @@ def test_qc4(self): rseals = [qc_vals_before[0], qc_vals_after[0]] cms = [qc_vals_before[1], qc_vals_after[1]] rseriess = [qc_vals_before[2], qc_vals_after[2]] - result = hergqc.qc4(rseals, cms, rseriess) + result = self.qc.qc4(rseals, cms, rseriess) pass_rseals, d_rseal = result[0] ex_pass_rseals = well not in failed_wells_rseals self.assertEqual( pass_rseals, ex_pass_rseals, - f"QC4: {well} (rseals) {d_rseal} {rseals}", + f'QC4: {well} (rseals) {d_rseal} {rseals}', ) pass_cms, d_cm = result[1] @@ -612,7 +578,7 @@ def test_qc4(self): self.assertEqual( pass_cms, ex_pass_cms, - f"QC4: {well} (cms) {d_cm} {cms}", + f'QC4: {well} (cms) {d_cm} {cms}', ) pass_rseriess, d_rseries = result[2] @@ -620,14 +586,12 @@ def test_qc4(self): self.assertEqual( pass_rseriess, ex_pass_rseriess, - f"QC4: {well} (rseriess) {d_rseries} {rseriess}", + f'QC4: {well} (rseriess) {d_rseries} {rseriess}', ) def test_qc5(self): # qc5 checks that the maximum current during the second half of the # staircase changes by at least 75% of the raw trace after E-4031 addition - hergqc = self.create_hergqc('qc5') - test_matrix = [ (-1.0, True, -12.5), (0.1, True, -1.5), @@ -640,22 +604,18 @@ def test_qc5(self): (1.0, False, 7.5), ] - recording1 = np.asarray([0, 0.1] * 100 + [10] * 500) + recording1 = np.array([0, 0.1] * 100 + [10] * 500) for i, ex_pass, ex_d_max_diff in test_matrix: - recording2 = np.asarray( - [0, 0.1] * 100 + [10 * i] * 500 - ) - pass_, d_max_diff = hergqc.qc5(recording1, recording2, (0, -1)) + recording2 = np.array([0, 0.1] * 100 + [10 * i] * 500) + pass_, d_max_diff = self.qc.qc5(recording1, recording2, (0, -1)) self.assertAlmostEqual( - d_max_diff, - ex_d_max_diff, - 1, - f"QC5: ({i}) {d_max_diff} != {ex_d_max_diff}", - ) - self.assertEqual(pass_, ex_pass, f"QC5: ({i}) {pass_} != {ex_pass}") + d_max_diff, ex_d_max_diff, 1, + f'QC5: ({i}) {d_max_diff} != {ex_d_max_diff}') + self.assertEqual( + pass_, ex_pass, f'QC5: ({i}) {pass_} != {ex_pass}') # Test on data - failed_wells = [ + failed_wells = { 'A10', 'A12', 'A13', 'A15', 'A19', 'A20', 'A24', 'B05', 'B07', 'B09', 'B11', 'B12', 'B13', 'B15', 'B18', 'B19', 'B21', 'B23', 'C02', 'C04', 'C05', 'C07', 'C08', 'C09', 'C11', 'C12', 'C14', 'C17', 'C18', 'C19', @@ -679,27 +639,23 @@ def test_qc5(self): 'O17', 'O18', 'O19', 'O20', 'O21', 'O22', 'O24', 'P01', 'P03', 'P05', 'P06', 'P07', 'P08', 'P09', 'P10', 'P12', 'P13', 'P14', 'P15', 'P17', 'P18', 'P20', 'P21', 'P22', 'P24' - ] + } for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) + # Test plotting, but only once + label = 'qc5-A03.png' if well == 'A03' else None - pass_, d_max_diff = hergqc.qc5( - before[0, :], after[0, :], hergqc.qc5_win - ) + before = self.trace_sweeps_before[well][0] + after = self.trace_sweeps_after[well][0] + + pass_, d_max_diff = self.qc.qc5( + before, after, self.qc.qc5_win, label=label) ex_pass = well not in failed_wells - self.assertEqual( - pass_, - ex_pass, - f"QC5: {well} {d_max_diff}", - ) + self.assertEqual(pass_, ex_pass, f'QC5: {well} {d_max_diff}') def test_qc5_1(self): # qc5_1 checks that the RMSD to zero of staircase protocol changes # by at least 50% of the raw trace after E-4031 addition. - hergqc = self.create_hergqc('qc5_1') - test_matrix = [ (-1.0, False, 4.23), (-0.5, False, 0), @@ -713,24 +669,18 @@ def test_qc5_1(self): (1.0, False, 4.23), ] - recording1 = np.asarray([0, 0.1] * 100 + [10] * 500) + recording1 = np.array([0, 0.1] * 100 + [10] * 500) for i, ex_pass, ex_d_max_diff in test_matrix: - recording2 = np.asarray( - [0, 0.1] * 100 + [10 * i] * 500 - ) - pass_, d_max_diff = hergqc.qc5_1(recording1, recording2) + recording2 = np.array([0, 0.1] * 100 + [10 * i] * 500) + pass_, d_max_diff = self.qc.qc5_1(recording1, recording2) self.assertAlmostEqual( - d_max_diff, - ex_d_max_diff, - 2, - f"QC5_1: ({i}) {d_max_diff} != {ex_d_max_diff}", - ) + d_max_diff, ex_d_max_diff, 2, + f'QC5_1: ({i}) {d_max_diff} != {ex_d_max_diff}') self.assertEqual( - pass_, ex_pass, f"QC5_1: ({i}) {pass_} != {ex_pass}" - ) + pass_, ex_pass, f'QC5_1: ({i}) {pass_} != {ex_pass}') # Test on data - failed_wells = [ + failed_wells = { 'A05', 'A10', 'A12', 'A13', 'A15', 'A19', 'A20', 'A24', 'B02', 'B05', 'B07', 'B09', 'B10', 'B12', 'B13', 'B14', 'B15', 'B18', 'B19', 'B21', 'B23', 'C02', 'C04', 'C05', 'C07', 'C08', 'C09', 'C11', 'C12', 'C14', @@ -757,24 +707,22 @@ def test_qc5_1(self): 'O18', 'O19', 'O20', 'O21', 'O22', 'O24', 'P01', 'P03', 'P05', 'P06', 'P07', 'P08', 'P09', 'P10', 'P12', 'P13', 'P14', 'P15', 'P16', 'P17', 'P18', 'P20', 'P21', 'P22' - ] + } for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) + # Test plotting, but only once + label = 'qc5_1-A03.png' if well == 'A03' else None - pass_, d_rmsd = hergqc.qc5_1(before[0, :], after[0, :], label='1') - ex_pass = well not in failed_wells - self.assertEqual( - pass_, - ex_pass, - f"QC5_1: {well} {d_rmsd}", - ) + before = self.trace_sweeps_before[well][0] + after = self.trace_sweeps_after[well][0] + + obt, d_rmsd = self.qc.qc5_1(before, after, label=label) + exp = well not in failed_wells + self.assertEqual(obt, exp, f'QC5_1: {well} {d_rmsd}') def test_qc6(self): # qc6 checks that the first step up to +40 mV, before the staircase, in # the subtracted trace is bigger than -2 x estimated noise level. - hergqc = self.create_hergqc('qc6') test_matrix = [ (-100, False, 9.9), @@ -787,20 +735,15 @@ def test_qc6(self): ] for i, ex_pass, ex_d_val in test_matrix: - recording = np.asarray( - [0, 0.1] * 100 + [0.1 * i] * 500 - ) - pass_, d_val = hergqc.qc6(recording, win=[200, -1]) + recording = np.array([0, 0.1] * 100 + [0.1 * i] * 500) + pass_, d_val = self.qc.qc6(recording, win=[200, -1]) self.assertAlmostEqual( - d_val, - ex_d_val, - 1, - f"QC6: ({i}) {d_val} != {ex_d_val}", - ) - self.assertEqual(pass_, ex_pass, f"QC6: ({i}) {pass_} != {ex_pass}") + d_val, ex_d_val, 1, f'QC6: ({i}) {d_val} != {ex_d_val}') + self.assertEqual( + pass_, ex_pass, f'QC6: ({i}) {pass_} != {ex_pass}') # Test on data - failed_wells_0 = [ + failed_0 = { 'A05', 'A07', 'A11', 'A12', 'A13', 'A20', 'A22', 'A24', 'B02', 'B05', 'B07', 'B09', 'B10', 'B15', 'B23', 'C02', 'C04', 'C14', 'C18', 'C22', 'D01', 'D12', 'E04', 'E09', 'E10', 'E15', 'E16', 'E17', 'E19', 'E21', @@ -813,9 +756,9 @@ def test_qc6(self): 'M11', 'M12', 'M17', 'N06', 'N11', 'N14', 'N20', 'N24', 'O01', 'O07', 'O08', 'O16', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P08', 'P12', 'P14', 'P15', 'P17', 'P18', 'P21' - ] + } - failed_wells_1 = [ + failed_1 = { 'A05', 'A07', 'A11', 'A12', 'A13', 'A20', 'A22', 'A24', 'B02', 'B05', 'B07', 'B10', 'B15', 'C02', 'C04', 'C14', 'C18', 'C22', 'D01', 'D12', 'E04', 'E09', 'E10', 'E14', 'E15', 'E16', 'E19', 'E21', 'F02', 'F03', @@ -827,9 +770,9 @@ def test_qc6(self): 'L24', 'M01', 'M04', 'M06', 'M07', 'M11', 'M12', 'M17', 'N06', 'N11', 'N14', 'N20', 'N24', 'O01', 'O06', 'O07', 'O08', 'O17', 'O19', 'O22', 'O24', 'P01', 'P06', 'P08', 'P12', 'P14', 'P15', 'P17', 'P18', 'P21' - ] + } - failed_wells_2 = [ + failed_2 = { 'A05', 'A07', 'A10', 'A11', 'A12', 'A13', 'A20', 'A22', 'A24', 'B02', 'B05', 'B07', 'B10', 'B15', 'C02', 'C04', 'C14', 'C22', 'D01', 'D12', 'E04', 'E09', 'E10', 'E14', 'E15', 'E16', 'E21', 'F02', 'F03', 'F07', @@ -841,63 +784,36 @@ def test_qc6(self): 'M06', 'M07', 'M11', 'M12', 'M17', 'N06', 'N11', 'N14', 'N24', 'O01', 'O06', 'O07', 'O08', 'O17', 'O19', 'O24', 'P01', 'P06', 'P08', 'P12', 'P14', 'P15', 'P17', 'P18', 'P21' - ] + } for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) - - subtracted_0 = [] - subtracted_1 = [] - subtracted_2 = [] - - for i in range(before.shape[0]): - subtracted_0.append( - hergqc.qc6( - (before[i, :] - after[i, :]), hergqc.qc6_win, label="0" - ) - ) - - subtracted_1.append( - hergqc.qc6( - (before[i, :] - after[i, :]), - hergqc.qc6_1_win, - label="1", - ) - ) - - subtracted_2.append( - hergqc.qc6( - (before[i, :] - after[i, :]), - hergqc.qc6_2_win, - label="2", - ) - ) - - ex_pass_0 = well not in failed_wells_0 - self.assertEqual( - all_passed(subtracted_0), - ex_pass_0, - f"QC6 (0): {well} {subtracted_0}", - ) - - ex_pass_1 = well not in failed_wells_1 - self.assertEqual( - all_passed(subtracted_1), - ex_pass_1, - f"QC6 (1): {well} {subtracted_1}", - ) - - ex_pass_2 = well not in failed_wells_2 - self.assertEqual( - all_passed(subtracted_2), - ex_pass_2, - f"QC6 (2): {well} {subtracted_2}", - ) + # Test plotting, once + l0 = l1 = l2 = None + if well == 'A03': + l0, l1, l2 = ('qc6-0', 'qc6-1', 'ac6-2') + + before = self.trace_sweeps_before[well] + after = self.trace_sweeps_after[well] + sub0, sub1, sub2 = [], [], [] + for b, a in zip(before, after): + c = b - a + sub0.append(self.qc.qc6(c, self.qc.qc6_win, label=l0)) + sub1.append(self.qc.qc6(c, self.qc.qc6_1_win, label=l1)) + sub2.append(self.qc.qc6(c, self.qc.qc6_2_win, label=l2)) + l0 = l1 = l2 = None + + self.assertEqual(all_passed(sub0), well not in failed_0, + f'QC6 (0): {well} {sub0}') + self.assertEqual(all_passed(sub1), well not in failed_1, + f'QC6 (1): {well} {sub1}') + self.assertEqual(all_passed(sub2), well not in failed_2, + f'QC6 (2): {well} {sub2}') def test_run_qc(self): # Test all wells - hergqc = self.create_hergqc('run_qc') + + # Create a QC that doesn't store (already tested) + qc = hERGQC(self.voltage, self.sampling_rate) failed_wells = [ 'A04', 'A05', 'A06', 'A07', 'A08', 'A10', 'A11', 'A12', 'A13', 'A15', @@ -934,27 +850,30 @@ def test_run_qc(self): ] for well in self.all_wells: - before = np.array(self.trace_sweeps_before[well]) - after = np.array(self.trace_sweeps_after[well]) - - # Take values from the first sweep only - qc_vals_before = np.array(self.qc_vals_before[well])[0, :] - qc_vals_after = np.array(self.qc_vals_after[well])[0, :] - - QC = hergqc.run_qc( + QC = qc.run_qc( voltage_steps=self.voltage_steps, times=self.times, - before=before, - after=after, - qc_vals_before=qc_vals_before, - qc_vals_after=qc_vals_after, + before=self.trace_sweeps_before[well], + after=self.trace_sweeps_after[well], + qc_vals_before=self.qc_vals_before[well][0], + qc_vals_after=self.qc_vals_after[well][0], n_sweeps=self.n_sweeps, ) - trace = "" + trace = [] for label, result in QC.items(): if not QC.qc_passed(label): - trace += f"{well} {label}: {result}\n" + trace.append(f'{well} {label}: {result}') + trace = '\n'.join(trace) ex_pass = well not in failed_wells self.assertEqual(QC.all_passed(), ex_pass, trace) + + +if __name__ == '__main__': + if '-store' in sys.argv: + sys.argv.remove('-store') + store_output = True + else: + print('Add -store to store output') + unittest.main() diff --git a/pcpostprocess/tests/test_infer_reversal.py b/pcpostprocess/tests/test_infer_reversal.py index 75070e9d..6cb1a83e 100755 --- a/pcpostprocess/tests/test_infer_reversal.py +++ b/pcpostprocess/tests/test_infer_reversal.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os import unittest +import sys +import tempfile from syncropatch_export.trace import Trace @@ -8,55 +10,63 @@ from pcpostprocess.detect_ramp_bounds import detect_ramp_bounds -class TestInferReversal(unittest.TestCase): - def setUp(self): - test_data_dir = os.path.join('tests', 'test_data', '13112023_MW2_FF', - "staircaseramp (2)_2kHz_15.01.07") - json_file = "staircaseramp (2)_2kHz_15.01.07.json" - - self.output_dir = os.path.join('test_output', self.__class__.__name__) - - if not os.path.exists(self.output_dir): - os.makedirs(self.output_dir) - - self.test_trace = Trace(test_data_dir, json_file) - - # get currents and QC from trace object - self.currents = self.test_trace.get_all_traces(leakcorrect=False) - self.currents['times'] = self.test_trace.get_times() - self.currents['voltages'] = self.test_trace.get_voltage() - - self.protocol_desc = self.test_trace.get_voltage_protocol().get_all_sections() - self.leak_ramp_bound_indices = detect_ramp_bounds(self.currents['times'], - self.protocol_desc, - ramp_index=0) +store_output = False - self.voltages = self.test_trace.get_voltage() - self.correct_Erev = -89.57184330525791438049054704606533050537109375 - def test_plot_leak_fit(self): - well = "A03" - sweep = 0 - - voltage = self.test_trace.get_voltage() - times = self.test_trace.get_times() - - current = self.test_trace.get_trace_sweeps(sweeps=[sweep])[well][0, :] - params, Ileak = leak_correct.fit_linear_leak(current, voltage, times, - *self.leak_ramp_bound_indices, - output_dir=self.output_dir, - save_fname=f"{well}_sweep{sweep}_leak_correction") - - I_corrected = current - Ileak - - E_rev = infer_reversal.infer_reversal_potential( - I_corrected, times, self.protocol_desc, - self.voltages, - output_path=os.path.join(self.output_dir, - f"{well}_staircase"), - known_Erev=self.correct_Erev) - self.assertLess(abs(E_rev - self.correct_Erev), 1e-5) +class TestInferReversal(unittest.TestCase): + """ + Tests the `infer_reversal` method. + """ + @classmethod + def setUpClass(self): + if store_output: # pragma: no cover + self.temp_dir = None + self.plot_dir = os.path.join('test_output', 'infer_reversal') + os.makedirs(self.plot_dir, exist_ok=True) + else: + self.temp_dir = tempfile.TemporaryDirectory() + self.plot_dir = self.temp_dir.name + + @classmethod + def tearDownClass(self): + if self.temp_dir: + self.temp_dir.cleanup() + + def test_infer_reversal(self): + # Test infer_reversal_potential, including plot + + # Load test data + data = os.path.join('test_data', '13112023_MW2_FF', + 'staircaseramp (2)_2kHz_15.01.07') + trace = Trace(data, 'staircaseramp (2)_2kHz_15.01.07.json') + + # Get times and voltages + times = trace.get_times() + voltages = trace.get_voltage() + + # Get protocol and leak ramp indices + protocol = trace.get_voltage_protocol().get_all_sections() + leak_indices = detect_ramp_bounds(times, protocol, ramp_index=0) + + # Load current for one well, one sweep, and leak correct + well, sweep = 'A03', 0 + current = trace.get_trace_sweeps(sweeps=[sweep])[well][0] + _, leak = leak_correct.fit_linear_leak( + current, voltages, times, *leak_indices) + current -= leak + + # Estimate reversal potential + fpath = os.path.join(self.plot_dir, f'{well}-{sweep}.png') + erev = infer_reversal.infer_reversal_potential( + current, times, protocol, voltages, fpath, -89.57184) + self.assertAlmostEqual(erev, -89.57184, 5) if __name__ == "__main__": - pass + if '-store' in sys.argv: + store_output = True + sys.argv.remove('-store') + else: + print('Add -store to store output') + + unittest.main() diff --git a/pcpostprocess/tests/test_leak_correct.py b/pcpostprocess/tests/test_leak_correct.py index 4b4ed399..1b7677a1 100755 --- a/pcpostprocess/tests/test_leak_correct.py +++ b/pcpostprocess/tests/test_leak_correct.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import os +import sys +import tempfile import unittest from syncropatch_export.trace import Trace @@ -8,57 +10,79 @@ from pcpostprocess.detect_ramp_bounds import detect_ramp_bounds -class TestLeakCorrect(unittest.TestCase): - def setUp(self): - test_data_dir = os.path.join('tests', 'test_data', '13112023_MW2_FF', - "staircaseramp (2)_2kHz_15.01.07") - json_file = "staircaseramp (2)_2kHz_15.01.07.json" - - self.output_dir = os.path.join('test_output', self.__class__.__name__) +store_output = False - os.makedirs(self.output_dir, exist_ok=True) - self.test_trace = Trace(test_data_dir, json_file) +class TestLeakCorrect(unittest.TestCase): + """ + Test the leak correction methods + """ + + @classmethod + def setUpClass(self): + if store_output: # pragma: no cover + self.temp_dir = None + self.plot_dir = os.path.join('test_output', 'leak_correct') + os.makedirs(self.plot_dir, exist_ok=True) + else: + self.temp_dir = tempfile.TemporaryDirectory() + self.plot_dir = self.temp_dir.name + + test_data_dir = os.path.join( + 'test_data', '13112023_MW2_FF', 'staircaseramp (2)_2kHz_15.01.07') + json_file = 'staircaseramp (2)_2kHz_15.01.07.json' + + self.trace = Trace(test_data_dir, json_file) # get currents and QC from trace object - self.currents = self.test_trace.get_all_traces(leakcorrect=False) - self.currents['times'] = self.test_trace.get_times() - self.currents['voltages'] = self.test_trace.get_voltage() + self.currents = self.trace.get_all_traces(leakcorrect=False) + self.currents['times'] = self.trace.get_times() + self.currents['voltages'] = self.trace.get_voltage() - self.QC = self.test_trace.get_onboard_QC_values() + self.QC = self.trace.get_onboard_QC_values() # Find first times ahead of these times - voltage_protocol = self.test_trace.get_voltage_protocol().get_all_sections() + voltage_protocol = self.trace.get_voltage_protocol().get_all_sections() times = self.currents['times'].flatten() self.ramp_bound_indices = detect_ramp_bounds(times, voltage_protocol, ramp_index=0) - def test_plot_leak_fit(self): - well = 'A01' - sweep = 0 + @classmethod + def tearDownClass(self): + if self.temp_dir: + self.temp_dir.cleanup() - voltage = self.test_trace.get_voltage() - times = self.test_trace.get_times() + def test_fit_linear_leak(self): + # Test fit_linear_leak, and plotting + well, sweep = 'A01', 0 - current = self.test_trace.get_trace_sweeps(sweeps=[sweep])[well][0, :] + current = self.trace.get_trace_sweeps(sweeps=[sweep])[well][0] + voltage = self.trace.get_voltage() + time = self.trace.get_times() + fname = os.path.join(self.plot_dir, f'{well}-{sweep}.png') - leak_correct.fit_linear_leak(current, voltage, times, - *self.ramp_bound_indices, - output_dir=self.output_dir, - save_fname=f"{well}_sweep{sweep}_leak_correction") + leak_correct.fit_linear_leak( + current, voltage, time, *self.ramp_bound_indices, save_fname=fname) def test_get_leak_correct(self): - trace = self.test_trace - currents = self.currents - well = 'A01' - sweep = 0 + # Test get_leak_corrected + well, sweep = 'A01', 0 + + trace = self.trace + current = self.currents[well][sweep] voltage = trace.get_voltage() - times = trace.get_times() + time = trace.get_times() - current = currents[well][sweep, :] - x = leak_correct.get_leak_corrected(current, voltage, times, - *self.ramp_bound_indices) + x = leak_correct.get_leak_corrected( + current, voltage, time, *self.ramp_bound_indices) self.assertEqual(x.shape, (30784,)) if __name__ == "__main__": - pass + if '-store' in sys.argv: + sys.argv.remove('-store') + store_output = True + else: + print('Add -store to store output') + + unittest.main() + diff --git a/pcpostprocess/tests/test_scripts.py b/pcpostprocess/tests/test_scripts.py index 9fe79dfb..6a3e50e3 100755 --- a/pcpostprocess/tests/test_scripts.py +++ b/pcpostprocess/tests/test_scripts.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import os +import sys import tempfile import unittest from pcpostprocess.scripts.run_herg_qc import run as run_herg_qc from pcpostprocess.scripts.summarise_herg_export import run as run_summarise + store_output = False @@ -14,33 +16,50 @@ class TestScripts(unittest.TestCase): Tests the scripts bundled with pcpostprocess. """ + @classmethod + def setUpClass(self): + if store_output: # pragma: no cover + self.temp_dir = None + self.plot_dir = 'test_output' + os.makedirs(self.plot_dir, exist_ok=True) + else: + self.temp_dir = tempfile.TemporaryDirectory() + self.plot_dir = self.temp_dir.name + + @classmethod + def tearDownClass(self): + if self.temp_dir: + self.temp_dir.cleanup() + def test_run_herg_qc_and_summarise_herg_export(self): # Test run_herg_qc_, then summarise_herg_export - data = os.path.join('tests', 'test_data', '13112023_MW2_FF') - with tempfile.TemporaryDirectory() as d: - if store_output: - d = 'test_output' - d1 = os.path.join(d, 'run_herg_qc') - d2 = os.path.join(d, 'summarise_herg_export') + data = os.path.join('test_data', '13112023_MW2_FF') + d1 = os.path.join(self.plot_dir, 'run_herg_qc') + d2 = os.path.join(self.plot_dir, 'summarise_herg_export') + + # Test run herg qc + erev = -90.71 + qc_map = {'staircaseramp (2)_2kHz': 'staircaseramp'} + write_map = {'staircaseramp2': 'staircaseramp2'} + run_herg_qc( + data, d1, qc_map, ('A03', 'A20', 'D16'), + write_traces=True, write_map=write_map, + save_id='13112023_MW2', reversal_potential=erev) - # Test run herg qc - erev = -90.71 - qc_map = {'staircaseramp (2)_2kHz': 'staircaseramp'} - write_map = {'staircaseramp2': 'staircaseramp2'} - run_herg_qc( - data, d1, qc_map, ('A03', 'A20', 'D16'), - write_traces=True, write_map=write_map, - save_id='13112023_MW2', reversal_potential=erev) + with open(os.path.join(d1, 'passed_wells.txt'), 'r') as f: + self.assertEqual(f.read().strip(), 'A03') - with open(os.path.join(d1, 'passed_wells.txt'), 'r') as f: - self.assertEqual(f.read().strip(), 'A03') + # Test summarise herg export + run_summarise(d1, d2, '13112023_MW2', reversal_potential=erev) - # Test summarise herg export - run_summarise(d1, d2, '13112023_MW2', reversal_potential=erev) +if __name__ == "__main__": + if '-store' in sys.argv: + sys.argv.remove('-store') + store_output = True + else: + print('Add -store to store output') -if __name__ == '__main__': - store_output = True unittest.main() diff --git a/pcpostprocess/tests/test_subtraction_plots.py b/pcpostprocess/tests/test_subtraction_plots.py index 1904d51d..b06b6c29 100755 --- a/pcpostprocess/tests/test_subtraction_plots.py +++ b/pcpostprocess/tests/test_subtraction_plots.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 import os +import sys +import tempfile import unittest import matplotlib.pyplot as plt @@ -9,44 +11,68 @@ from pcpostprocess.subtraction_plots import do_subtraction_plot -class TestSubtractionPlots(unittest.TestCase): - def setUp(self): - test_data_dir_before = os.path.join('tests', 'test_data', '13112023_MW2_FF', - "staircaseramp (2)_2kHz_15.01.07") - - test_data_dir_after = os.path.join('tests', 'test_data', '13112023_MW2_FF', - "staircaseramp (2)_2kHz_15.11.33") - - json_file_before = "staircaseramp (2)_2kHz_15.01.07.json" - json_file_after = "staircaseramp (2)_2kHz_15.11.33.json" +store_output = False - self.output_dir = os.path.join("test_output", self.__class__.__name__) - os.makedirs(self.output_dir, exist_ok=True) - # Use identical traces for purpose of the test - self.before_trace = Trace(test_data_dir_before, json_file_before) - self.after_trace = Trace(test_data_dir_after, json_file_after) +class TestSubtractionPlots(unittest.TestCase): + """ + Tests the subtraction_plots module. + """ + + @classmethod + def setUpClass(self): + if store_output: # pragma: no cover + self.temp_dir = None + self.plot_dir = os.path.join('test_output', 'subtraction_plots') + os.makedirs(self.plot_dir, exist_ok=True) + else: + self.temp_dir = tempfile.TemporaryDirectory() + self.plot_dir = self.temp_dir.name + + # TODO: Only one test, why are we doing this? + dir_before = os.path.join( + 'test_data', '13112023_MW2_FF', 'staircaseramp (2)_2kHz_15.01.07') + dir_after = os.path.join( + 'test_data', '13112023_MW2_FF', 'staircaseramp (2)_2kHz_15.11.33') + + json_file_before = 'staircaseramp (2)_2kHz_15.01.07.json' + json_file_after = 'staircaseramp (2)_2kHz_15.11.33.json' + + self.before_trace = Trace(dir_before, json_file_before) + self.after_trace = Trace(dir_after, json_file_after) + + @classmethod + def tearDownClass(self): + if self.temp_dir: + self.temp_dir.cleanup() def test_do_subtraction_plot(self): + # Tests do_subtraction_plot + fig = plt.figure(figsize=(16, 18), layout='constrained') times = self.before_trace.get_times() well = 'A01' before_current = self.before_trace.get_trace_sweeps()[well] after_current = self.after_trace.get_trace_sweeps()[well] - voltage_protocol = self.before_trace.get_voltage_protocol() - - ramp_bounds = detect_ramp_bounds(times, - voltage_protocol.get_all_sections()) + ramp_bounds = detect_ramp_bounds( + times, voltage_protocol.get_all_sections()) sweeps = [0, 1] voltages = self.before_trace.get_voltage() do_subtraction_plot(fig, times, sweeps, before_current, after_current, voltages, ramp_bounds, well=well) - fig.savefig(os.path.join(self.output_dir, f"subtraction_plot_{well}")) + fig.savefig(os.path.join(self.plot_dir, f'{well}.png')) if __name__ == "__main__": - pass + if '-store' in sys.argv: + sys.argv.remove('-store') + store_output = True + else: + print('Add -store to store output') + + unittest.main() + diff --git a/pull_request_template.md b/pull_request_template.md deleted file mode 100644 index 809c58ad..00000000 --- a/pull_request_template.md +++ /dev/null @@ -1,20 +0,0 @@ -## Description - - - - -## Types of changes - -- [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] New feature (non-breaking change which adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to change) - -## Testing -- [X] Testing is done automatically and codecov shows test coverage -- [ ] This cannot be tested automatically - -## Documentation checklist - -- [ ] I have updated all documentation in the code where necessary. -- [ ] I have checked spelling in all (new) comments and documentation. -- [ ] I have added a note to RELEASE.md if relevant (new feature, breaking change, or notable bug fix). From f1a7788ea42eecaa58cedcf92455c9fa85d752cc Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 22:55:49 +0000 Subject: [PATCH 05/17] No seeding in run-herg-qc --- pcpostprocess/scripts/run_herg_qc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index a971f347..4936232d 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -1193,7 +1193,7 @@ def fit_func(x, args=None): ] # TESTING ONLY - np.random.seed(1) + # np.random.seed(1) #  Repeat optimisation with different starting guesses x0s = [[np.random.uniform(lower_b, upper_b) for lower_b, upper_b in bounds] for i in range(100)] From aac356ab4eb6b3b266ddf8f752f60618e158783e Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 22:58:13 +0000 Subject: [PATCH 06/17] Fix to test workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1eefdb00..d3d21ba1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: - name: Check code style with flake8 if: ${{ matrix.python-version == env.python-latest }} run: | - python -m flake8 pcpostprocess/*.py tests/*.py pcpostprocess/scripts/*.py + python -m flake8 pcpostprocess - name: Check import ordering with isort if: ${{ matrix.python-version == env.python-latest }} From 74056ecb56f0a798b15cbaf73afdc4045ffad617 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:03:04 +0000 Subject: [PATCH 07/17] Removing isort, causes more issues than it solves --- .github/workflows/tests.yml | 5 ----- CONTRIBUTING.md | 11 ----------- pyproject.toml | 4 ---- 3 files changed, 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3d21ba1..76ca4484 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,11 +38,6 @@ jobs: run: | python -m flake8 pcpostprocess - - name: Check import ordering with isort - if: ${{ matrix.python-version == env.python-latest }} - run: | - python -m isort --verbose --check-only --diff pcpostprocess tests setup.py - - name: Extract test data run: | wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz -P tests/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ea00af3..4d94ecc8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,17 +62,6 @@ In addition to the rules checked by flake8, we try to use single quotes (`'`) fo Class, method, and argument names are in UK english. -### Import ordering - -Import ordering is tested with [isort](https://pycqa.github.io/isort/index.html). - -To run locally, use -``` -isort --check-only --verbose ./pcpostprocess ./tests/ -``` - -Isort is configured in [pyproject.toml](./pyproject.toml) under the section `tool.isort`. - ## Documentation Every method and every class should have a [docstring](https://www.python.org/dev/peps/pep-0257/) that describes in plain terms what it does, and what the expected input and output is. diff --git a/pyproject.toml b/pyproject.toml index fc30cf1f..286ad492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ test = [ 'coverage', # For coverage testing 'codecov>=2.1.3', # To upload coverage reports 'flake8>=3', # For code style checking - 'isort', ] [tools.setuptools.package-data] @@ -66,9 +65,6 @@ include = [ 'pcpostprocess', ] -[tool.isort] -skip = ['_version.py'] - [tool.coverage.run] source = [ 'pcpostprocess', From 32e1695e16bed806db4860cfaf3a5d262f05f5ac Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:11:39 +0000 Subject: [PATCH 08/17] Dropped excel export. Closes #113 --- .github/workflows/tests.yml | 2 +- README.md | 6 +++--- pcpostprocess/scripts/run_herg_qc.py | 1 - pcpostprocess/scripts/summarise_herg_export.py | 1 - pyproject.toml | 1 - 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 76ca4484..5a77bc4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,7 @@ jobs: - name: Extract test data run: | wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz -P tests/ - tar xvf tests/test_data.tar.xz -C tests/ + tar xvf test_data.tar.xz -C tests/ - name: Install TeX dependencies for run_herg_qc test timeout-minutes: 5 diff --git a/README.md b/README.md index 625d3625..1106afd3 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ To run the tests you must first download some test data. Test data is available at [cardiac.nottingham.ac.uk/syncropatch\_export](https://cardiac.nottingham.ac.uk/syncropatch_export) ```sh -wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz -P tests/ -tar xvf tests/test_data.tar.xz -C tests/ -rm tests/test_data.tar.xz +wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz +tar xvf test_data.tar.xz +rm test_data.tar.xz ``` Then you can run the tests. diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index 4936232d..5a178865 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -452,7 +452,6 @@ def run(data_path, output_path, qc_map, wells=None, qc_styled_df = create_qc_table(qc_df) logging.info(qc_styled_df) - # qc_styled_df.to_excel(os.path.join(output_path, 'qc_table.xlsx')) qc_styled_df.to_latex(os.path.join(output_path, 'qc_table.tex')) # Save in csv format diff --git a/pcpostprocess/scripts/summarise_herg_export.py b/pcpostprocess/scripts/summarise_herg_export.py index a9664a50..4eb5b7f7 100644 --- a/pcpostprocess/scripts/summarise_herg_export.py +++ b/pcpostprocess/scripts/summarise_herg_export.py @@ -84,7 +84,6 @@ def run(data_path, output_path, experiment_name, reversal_potential=None, qc_styled_df = create_qc_table(qc_df) qc_styled_df = qc_styled_df.pivot(columns='protocol', index='crit') - # qc_styled_df.to_excel(os.path.join(output_path, 'qc_table.xlsx')) qc_styled_df.to_latex(os.path.join(output_path, 'qc_table.tex')) qc_df.protocol = ['staircaseramp1' if protocol == 'staircaseramp' else protocol diff --git a/pyproject.toml b/pyproject.toml index 286ad492..e0317cb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dependencies = [ 'pandas>=1.3', 'regex>=2023.12.25', 'seaborn>=0.12.2', - 'openpyxl>=3.1.2', 'jinja2>=3.1.0', 'syncropatch_export @ git+https://github.com/CardiacModelling/syncropatch_export.git' ] From 2f2c816ba8f70c9cdc372bc924fef3f08271fa86 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:13:57 +0000 Subject: [PATCH 09/17] Fix to test workflow --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a77bc4d..36e829df 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,8 +40,9 @@ jobs: - name: Extract test data run: | - wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz -P tests/ - tar xvf test_data.tar.xz -C tests/ + wget https://cardiac.nottingham.ac.uk/syncropatch_export/test_data.tar.xz + tar xvf test_data.tar.xz + rm test_data.tar.xz - name: Install TeX dependencies for run_herg_qc test timeout-minutes: 5 From cb4f9c4d845ea792ede9dd3a54cc6b9eefb452ab Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:18:58 +0000 Subject: [PATCH 10/17] Trying without latex deps --- .github/workflows/tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36e829df..3526af71 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,10 +44,6 @@ jobs: tar xvf test_data.tar.xz rm test_data.tar.xz - - name: Install TeX dependencies for run_herg_qc test - timeout-minutes: 5 - run: sudo apt-get install dvipng texlive-latex-extra texlive-fonts-recommended cm-super -y - - name: Run unit tests (without coverage testing) if: ${{ success() && matrix.python-version != env.python-latest }} run: python -m unittest From 334f924886fa460e88efb20c91a8bd5162197e32 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:23:35 +0000 Subject: [PATCH 11/17] Removed need for latex as dependency --- pcpostprocess/leak_correct.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pcpostprocess/leak_correct.py b/pcpostprocess/leak_correct.py index 2b9319f1..c4238832 100644 --- a/pcpostprocess/leak_correct.py +++ b/pcpostprocess/leak_correct.py @@ -104,26 +104,26 @@ def fit_linear_leak(current, voltage, times, ramp_start_index, ramp_end_index, time_range = (0, times.max() / 5) #  Current vs time - ax1.set_title(r'\textbf{a}', loc='left', usetex=True) + ax1.set_title(r'\textbf{a}', loc='left') ax1.set_xlabel(r'$t$ (ms)') ax1.set_ylabel(r'$I_\mathrm{obs}$ (pA)') ax1.set_xticklabels([]) ax1.set_xlim(*time_range) # Voltage vs time - ax2.set_title(r'\textbf{b}', loc='left', usetex=True) + ax2.set_title(r'\textbf{b}', loc='left') ax2.set_xlabel(r'$t$ (ms)') ax2.set_ylabel(r'$V_\mathrm{cmd}$ (mV)') ax2.set_xlim(*time_range) # Current vs voltage - ax3.set_title(r'\textbf{c}', loc='left', usetex=True) + ax3.set_title(r'\textbf{c}', loc='left') ax3.set_xlabel(r'$V_\mathrm{cmd}$ (mV)') ax3.set_ylabel(r'$I_\mathrm{obs}$ (pA)') ax4.set_xlabel(r'$t$ (ms)') ax4.set_ylabel(r'current (pA)') - ax4.set_title(r'\textbf{d}', loc='left', usetex=True) + ax4.set_title(r'\textbf{d}', loc='left') start_t = times[ramp_start_index] end_t = times[ramp_end_index] From 20fbd4f21d468b3f641e9523fe1be5bbad8af051 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:27:39 +0000 Subject: [PATCH 12/17] Added cover pragmas to command line handling --- pcpostprocess/scripts/run_herg_qc.py | 2 +- pcpostprocess/scripts/summarise_herg_export.py | 2 +- pcpostprocess/tests/test_directory_builder.py | 2 +- pcpostprocess/tests/test_herg_qc.py | 2 +- pcpostprocess/tests/test_infer_reversal.py | 2 +- pcpostprocess/tests/test_leak_correct.py | 2 +- pcpostprocess/tests/test_scripts.py | 2 +- pcpostprocess/tests/test_subtraction_plots.py | 2 +- pyproject.toml | 3 +++ 9 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index 5a178865..2cf250f4 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -40,7 +40,7 @@ def starmap(n, func, iterable): return pool.starmap(func, iterable) -def run_from_command_line(): +def run_from_command_line(): # pragma: no cover """ Reads arguments from the command line and an ``export_config.py`` and then runs herg QC. diff --git a/pcpostprocess/scripts/summarise_herg_export.py b/pcpostprocess/scripts/summarise_herg_export.py index 4eb5b7f7..7e0a2eac 100644 --- a/pcpostprocess/scripts/summarise_herg_export.py +++ b/pcpostprocess/scripts/summarise_herg_export.py @@ -37,7 +37,7 @@ def get_protocol_list(input_dir, experiment_name): return list(np.unique(protocols)) -def run_from_command_line(): +def run_from_command_line(): # pragma: no cover """ Parses arguments from the command line and then ??? """ diff --git a/pcpostprocess/tests/test_directory_builder.py b/pcpostprocess/tests/test_directory_builder.py index 9ba7d857..e0a72fd9 100755 --- a/pcpostprocess/tests/test_directory_builder.py +++ b/pcpostprocess/tests/test_directory_builder.py @@ -35,7 +35,7 @@ def test_directory_builder(self): self.assertRegex(info_dict['Commit'], r'^g[0-9a-fA-F]{9}$') -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover if '-store' in sys.argv: store_output = True sys.argv.remove('-store') diff --git a/pcpostprocess/tests/test_herg_qc.py b/pcpostprocess/tests/test_herg_qc.py index 8986a66e..a4d6f63f 100755 --- a/pcpostprocess/tests/test_herg_qc.py +++ b/pcpostprocess/tests/test_herg_qc.py @@ -870,7 +870,7 @@ def test_run_qc(self): self.assertEqual(QC.all_passed(), ex_pass, trace) -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover if '-store' in sys.argv: sys.argv.remove('-store') store_output = True diff --git a/pcpostprocess/tests/test_infer_reversal.py b/pcpostprocess/tests/test_infer_reversal.py index 6cb1a83e..218d8b01 100755 --- a/pcpostprocess/tests/test_infer_reversal.py +++ b/pcpostprocess/tests/test_infer_reversal.py @@ -62,7 +62,7 @@ def test_infer_reversal(self): self.assertAlmostEqual(erev, -89.57184, 5) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover if '-store' in sys.argv: store_output = True sys.argv.remove('-store') diff --git a/pcpostprocess/tests/test_leak_correct.py b/pcpostprocess/tests/test_leak_correct.py index 1b7677a1..04024db7 100755 --- a/pcpostprocess/tests/test_leak_correct.py +++ b/pcpostprocess/tests/test_leak_correct.py @@ -77,7 +77,7 @@ def test_get_leak_correct(self): self.assertEqual(x.shape, (30784,)) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover if '-store' in sys.argv: sys.argv.remove('-store') store_output = True diff --git a/pcpostprocess/tests/test_scripts.py b/pcpostprocess/tests/test_scripts.py index 6a3e50e3..9683952f 100755 --- a/pcpostprocess/tests/test_scripts.py +++ b/pcpostprocess/tests/test_scripts.py @@ -54,7 +54,7 @@ def test_run_herg_qc_and_summarise_herg_export(self): run_summarise(d1, d2, '13112023_MW2', reversal_potential=erev) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover if '-store' in sys.argv: sys.argv.remove('-store') store_output = True diff --git a/pcpostprocess/tests/test_subtraction_plots.py b/pcpostprocess/tests/test_subtraction_plots.py index b06b6c29..23c46837 100755 --- a/pcpostprocess/tests/test_subtraction_plots.py +++ b/pcpostprocess/tests/test_subtraction_plots.py @@ -67,7 +67,7 @@ def test_do_subtraction_plot(self): fig.savefig(os.path.join(self.plot_dir, f'{well}.png')) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover if '-store' in sys.argv: sys.argv.remove('-store') store_output = True diff --git a/pyproject.toml b/pyproject.toml index e0317cb7..61634039 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,4 +68,7 @@ include = [ source = [ 'pcpostprocess', ] +omit = [ + 'pcpostprocess/__main__.py', +] From 56b91c8f482f0e11cbbe3905f2285a5193d72d96 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:33:20 +0000 Subject: [PATCH 13/17] Removed unused option from directory builder --- pcpostprocess/directory_builder.py | 24 ++++--------------- pcpostprocess/tests/test_directory_builder.py | 2 +- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/pcpostprocess/directory_builder.py b/pcpostprocess/directory_builder.py index 4d145e76..290f1356 100644 --- a/pcpostprocess/directory_builder.py +++ b/pcpostprocess/directory_builder.py @@ -10,20 +10,15 @@ def get_git_revision_hash(): Get the hash for the git commit currently being used. @return The most recent commit hash or a suitable message - """ - return __commit_id__ def get_build_type(): - if "dev" in __version__: - return "Develop" - else: - return "Release" + return 'Develop' if 'dev' in __version__ else 'Release' -def setup_output_directory(dirname: str = None, subdir_name: str = None): +def setup_output_directory(dirname: str): """ Create an output directory if one doesn't already exist. Place an info file in this directory which lists the date/time created, the version of @@ -31,22 +26,11 @@ def setup_output_directory(dirname: str = None, subdir_name: str = None): commit. The two parameters allow for a user specified top-level directory and a script-defined name for a subdirectory. - @param Optional directory name - @param Optional subdirectory name + @param Directory name @return The path to the created file directory (String) """ - - if dirname is None: - if subdir_name: - dirname = os.path.join("output", f"{subdir_name}") - else: - dirname = os.path.join("output", "output") - - if subdir_name is not None: - dirname = os.path.join(dirname, subdir_name) - if not os.path.exists(dirname): - os.makedirs(dirname) + os.makedirs(dirname, exist_ok=True) with open(os.path.join(dirname, "pcpostprocess_info.txt"), "w") as description_fout: git_hash = get_git_revision_hash() diff --git a/pcpostprocess/tests/test_directory_builder.py b/pcpostprocess/tests/test_directory_builder.py index e0a72fd9..90ab0016 100755 --- a/pcpostprocess/tests/test_directory_builder.py +++ b/pcpostprocess/tests/test_directory_builder.py @@ -27,7 +27,7 @@ def test_directory_builder(self): if store_output: # pragma: no cover d = 'test_output' path = directory_builder.setup_output_directory( - d, 'directory_builder') + os.path.join(d, 'directory_builder')) # Check that git commit is written info_dict = read_info_dict( From 5247d7e5d867875255d59b4fb725e2f8e75a0dde Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:37:18 +0000 Subject: [PATCH 14/17] Removed unused parts of summarise_herg_export.py --- pcpostprocess/scripts/run_herg_qc.py | 4 +- .../scripts/summarise_herg_export.py | 309 +----------------- 2 files changed, 3 insertions(+), 310 deletions(-) diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index 2cf250f4..fd754bfe 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -35,7 +35,7 @@ def starmap(n, func, iterable): """ if n == 1: return [func(*args) for args in iterable] - else: + else: # pragma: no cover with multiprocessing.Pool(n, maxtasksperchild=1) as pool: return pool.starmap(func, iterable) @@ -1300,5 +1300,5 @@ def fit_func(x, args=None): return (d, b), f, peak_current if res else (np.nan, np.nan), np.nan, peak_current -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover run_from_command_line() diff --git a/pcpostprocess/scripts/summarise_herg_export.py b/pcpostprocess/scripts/summarise_herg_export.py index 7e0a2eac..c9ce182c 100644 --- a/pcpostprocess/scripts/summarise_herg_export.py +++ b/pcpostprocess/scripts/summarise_herg_export.py @@ -16,27 +16,6 @@ from pcpostprocess.scripts.run_herg_qc import create_qc_table -def get_wells_list(input_dir, experiment_name): - regex = re.compile(f"{experiment_name}-([a-z|A-Z|0-9]*)-([A-Z][0-9][0-9])-after") - wells = [] - - for f in filter(regex.match, os.listdir(input_dir)): - well = re.search(regex, f).groups(2)[1] - if well not in wells: - wells.append(well) - return list(np.unique(wells)) - - -def get_protocol_list(input_dir, experiment_name): - regex = re.compile(f"{experiment_name}-([a-z|A-Z|0-9]*)-([A-Z][0-9][0-9])-after") - protocols = [] - for f in filter(regex.match, os.listdir(input_dir)): - well = re.search(regex, f).groups(3)[0] - if protocols not in protocols: # TODO This has GOT to be a bug - protocols.append(well) - return list(np.unique(protocols)) - - def run_from_command_line(): # pragma: no cover """ Parses arguments from the command line and then ??? @@ -147,7 +126,6 @@ def run(data_path, output_path, experiment_name, reversal_potential=None, leak_parameters_df['passed QC'] = [ well in passed_wells for well in leak_parameters_df.well] - # do_scatter_matrices(leak_parameters_df, qc_vals_df, output_path, reversal_potential) plot_histograms(leak_parameters_df, output_path, reversal_potential, figsize) @@ -342,192 +320,6 @@ def label_func(p, s): plt.close(fig) -def do_combined_plots(data_path, output_path, experiment_name, - leak_parameters_df, passed_wells, figsize=None): - """ - ??? - - @param data_path - @param output_path - @param experiment_name - @param leak_parameters_df - @param passed_wells - @param figsize - - """ - fig = plt.figure(figsize=figsize, constrained_layout=True) - ax = fig.subplots() - - wells = [well for well in leak_parameters_df.well.unique() if well in passed_wells] - - protocol_overlaid_dir = os.path.join(output_path, 'overlaid_by_protocol') - if not os.path.exists(protocol_overlaid_dir): - os.makedirs(protocol_overlaid_dir) - - leak_parameters_df = leak_parameters_df[leak_parameters_df.well.isin(passed_wells)] - - palette = sns.color_palette('husl', len(leak_parameters_df.groupby(['well', 'sweep']))) - for protocol in leak_parameters_df.protocol.unique(): - pname = protocol - if pname == 'staircaseramp1': - pname = 'staircaseramp' - elif pname == 'staircaseramp1_2': - pname = 'staircaseramp_2' - - times_fname = os.path.join(data_path, 'traces', - f'{experiment_name}-{pname}-times.csv') - try: - times = np.loadtxt(times_fname).astype(np.float64).flatten() - except FileNotFoundError: - continue - - times = times.flatten().astype(np.float64) - - reference_current = None - - i = 0 - for sweep in leak_parameters_df.sweep.unique(): - for well in wells: - fname = f"{experiment_name}-{pname}-{well}-sweep{sweep}.csv" - try: - data = pd.read_csv(os.path.join(data_path, 'traces', fname)) - - except FileNotFoundError: - continue - - current = data['current'].values.flatten().astype(np.float64) - - if reference_current is None: - reference_current = current - - scaled_current = scale_to_reference(current, reference_current) - col = palette[i] - i += 1 - ax.plot(times, scaled_current, color=col, alpha=.5, label=well) - - fig_fname = f"{protocol}_overlaid_traces_scaled" - fig.suptitle(f"{protocol}: all wells") - ax.set_xlabel(r'time / ms') - ax.set_ylabel('current scaled to reference trace') - ax.legend() - fig.savefig(os.path.join(protocol_overlaid_dir, fig_fname)) - ax.cla() - - plt.close(fig) - - palette = sns.color_palette('husl', - len(leak_parameters_df.groupby(['protocol', 'sweep']))) - - fig2 = plt.figure(figsize=figsize, constrained_layout=True) - axs2 = fig2.subplots(1, 2, sharey=True) - - wells_overlaid_dir = os.path.join(output_path, 'overlaid_by_well') - if not os.path.exists(wells_overlaid_dir): - os.makedirs(wells_overlaid_dir) - - for well in passed_wells: - i = 0 - for sweep in leak_parameters_df.sweep.unique(): - for protocol in leak_parameters_df.protocol.unique(): - pname = protocol - if pname == 'staircaseramp1': - pname = 'staircaseramp' - elif pname == 'staircaseramp1_2': - pname = 'staircaseramp_2' - - times_fname = f'{experiment_name}-{pname}-times.csv' - times = np.loadtxt(os.path.join(data_path, 'traces', times_fname)) - times = times.flatten().astype(np.float64) - - fname = f"{experiment_name}-{protocol}-{well}-sweep{sweep}.csv" - try: - data = pd.read_csv(os.path.join(data_path, 'traces', fname)) - except FileNotFoundError: - continue - - current = data['current'].values.flatten().astype(np.float64) - - indices_pre_ramp = times < 3000 - - col = palette[i] - i += 1 - - label = f"{protocol}_sweep{sweep}" - - axs2[0].plot(times[indices_pre_ramp], current[indices_pre_ramp], color=col, alpha=.5, - label=label) - - indices_post_ramp = times > (times[-1] - 2000) - post_times = times[indices_post_ramp].copy() - post_times = post_times - post_times[0] + 5000 - axs2[1].plot(post_times, current[indices_post_ramp], color=col, alpha=.5, - label=label) - - axs2[0].legend() - axs2[0].set_title('before drug') - axs2[0].set_xlabel(r'time / ms') - axs2[1].set_title('after drug') - axs2[1].set_xlabel(r'time / ms') - - axs2[0].set_ylabel('current / pA') - axs2[1].set_ylabel('current / pA') - - fig2_fname = f"{well}_overlaid_traces" - fig2.suptitle(f"Leak ramp comparison: {well}") - - fig2.savefig(os.path.join(wells_overlaid_dir, fig2_fname)) - axs2[0].cla() - axs2[1].cla() - - plt.close(fig2) - - -def do_scatter_matrices(df, qc_df, output_path, reversal_potential=None): - """ - ??? - - @param df - @param qc_df - @param output_path - @reversal_potential - """ - grid = sns.pairplot(data=df, hue='passed QC', diag_kind='hist', - plot_kws={'alpha': 0.4, 'edgecolor': None}, - hue_order=[True, False]) - grid.savefig(os.path.join(output_path, 'scatter_matrix_by_QC')) - - if reversal_potential is not None: - true_reversal = reversal_potential - else: - # TODO Clarify in plot label! - true_reversal = df['E_rev'].values.mean() - - df['hue'] = df.E_rev.to_numpy() > true_reversal - grid = sns.pairplot(data=df, hue='hue', diag_kind='hist', - plot_kws={'alpha': 0.4, 'edgecolor': None}, - hue_order=[True, False]) - grid.savefig(os.path.join(output_path, 'scatter_matrix_by_reversal.png')) - - # Now do artefact parameters only - if 'drug' in qc_df: - qc_df = qc_df[qc_df.drug == 'before'] - - first_sweep = sorted(list(qc_df.sweep.unique()))[0] - qc_df = qc_df[(qc_df.protocol == 'staircaseramp1') & - (qc_df.sweep == first_sweep)] - if 'drug' in qc_df: - qc_df = qc_df[qc_df.drug == 'before'] - - qc_df = qc_df.set_index(['protocol', 'well', 'sweep']) - qc_df = qc_df[['Rseries', 'Cm', 'Rseal', 'passed QC']] - # qc_df['R_leftover'] = df['R_leftover'] - grid = sns.pairplot(data=qc_df, diag_kind='hist', plot_kws={'alpha': .4, - 'edgecolor': None}, - hue='passed QC', hue_order=[True, False]) - - grid.savefig(os.path.join(output_path, 'scatter_matrix_QC_params_by_QC')) - - def plot_reversal_spread(df, output_path, figsize=None): """ ??? @@ -860,105 +652,6 @@ def plot_histograms(df, output_path, reversal_potential=None, figsize=None): plt.close(fig) -def overlay_reversal_plots( - data_path, output_path, experiment_name, leak_parameters_df, wells, - reversal_potential=None, figsize=None): - """ - ??? - - @param data_path - @param output_path - @param experiment_name - @param leak_parameters_df - @param reversal_potential - @param figsize - """ - fig = plt.figure(figsize=figsize, constrained_layout=True) - ax = fig.subplots() - - palette = sns.color_palette('husl', len(leak_parameters_df.groupby(['protocol', 'sweep']))) - - sub_dir = os.path.join(output_path, 'overlaid_reversal_plots') - - if not os.path.exists(sub_dir): - os.makedirs(sub_dir) - - protocols_to_plot = ['staircaseramp1'] - sweeps_to_plot = [1] - - for well in wells: - # Setup figure - if False in leak_parameters_df[leak_parameters_df.well == well]['passed QC'].values: - continue - i = 0 - for protocol in protocols_to_plot: - if protocol == np.nan: - continue - - pname = protocol - if pname == 'staircaseramp1': - pname = 'staircaseramp' - elif pname == 'staircaseramp1_2': - pname = 'staircaseramp_2' - voltage_fname = os.path.join(data_path, 'traces', - f'{experiment_name}-{pname}-voltages.csv') - voltages = pd.read_csv(voltage_fname)['voltage'].values.flatten() - - times_fname = f"{experiment_name}-{pname}-times.csv" - times = np.loadtxt(os.path.join(data_path, 'traces', times_fname)) - times = times.flatten().astype(np.float64) - - # First, find the reversal ramp - json_name = os.path.join(data_path, 'traces', 'protocols', - f'{experiment_name}-{pname}.json') - with open(json_name, 'r') as f: - json_protocol = json.load(f) - v_protocol = VoltageProtocol.from_json(json_protocol) - ramps = v_protocol.get_ramps() - reversal_ramp = ramps[-1] - ramp_start, ramp_end = reversal_ramp[:2] - - # Next extract steps - istart = np.argmax(times >= ramp_start) - iend = np.argmax(times > ramp_end) - - if istart == 0 or iend == 0 or istart == iend: - raise Exception('Could not identify reversal ramp') - - for sweep in sweeps_to_plot: - fname = f"{experiment_name}-{pname}-{well}-sweep{sweep}.csv" - try: - data = pd.read_csv(os.path.join(data_path, 'traces', fname)) - except FileNotFoundError: - continue - - # Plot voltage vs current - current = data['current'].values.astype(np.float64) - - col = palette[i] - - ax.scatter(voltages[istart:iend], current[istart:iend], label=protocol, - color=col, s=1.2) - - fitted_poly = np.poly1d(np.polyfit(voltages[istart:iend], current[istart:iend], 4)) - ax.plot(voltages[istart:iend], fitted_poly(voltages[istart:iend]), color=col) - i += 1 - - if reversal_potential is not None: - ax.axvline(reversal_potential, linestyle='--', color='grey', - label='Calculated Nernst potential') - - ax.legend() - # Save figure - fig.savefig(os.path.join(sub_dir, f"overlaid_reversal_ramps_{well}")) - - # Clear figure - ax.cla() - - plt.close(fig) - return - - def scale_to_reference(trace, reference): def error2(p): return np.sum((p*trace - reference)**2) @@ -1043,5 +736,5 @@ def create_attrition_table(qc_df, subtraction_df): return res_df -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover run_from_command_line() From 407620159e642301fd1860b8893da9c811196106 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:57:42 +0000 Subject: [PATCH 15/17] No longer changing loggin level except in command line call. Closes #42. --- pcpostprocess/hergQC.py | 5 -- pcpostprocess/scripts/run_herg_qc.py | 46 ++++++------------- .../scripts/summarise_herg_export.py | 11 ----- pyproject.toml | 1 - 4 files changed, 14 insertions(+), 49 deletions(-) diff --git a/pcpostprocess/hergQC.py b/pcpostprocess/hergQC.py index 5740f3c6..b96f2608 100644 --- a/pcpostprocess/hergQC.py +++ b/pcpostprocess/hergQC.py @@ -75,13 +75,8 @@ def __init__(self, voltage, sampling_rate=5, removal_time=5, noise_len=200, self.removal_time = removal_time self.noise_len = int(noise_len) - # Passing in a plot dir enables debug mode self._plot_dir = plot_dir self.logger = logging.getLogger(__name__) - if self._plot_dir is not None: - self.logger.setLevel(logging.DEBUG) - # https://github.com/CardiacModelling/pcpostprocess/issues/42 - self._plot_dir = plot_dir # Define all thresholds diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index fd754bfe..3bb5f795 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -8,6 +8,7 @@ import logging import multiprocessing import os +import re import string import sys @@ -15,7 +16,6 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -import regex as re import scipy from syncropatch_export.trace import Trace from syncropatch_export.voltage_protocols import VoltageProtocol @@ -77,7 +77,7 @@ def run_from_command_line(): # pragma: no cover parser.add_argument('--figsize', nargs=2, type=int, default=[16, 18]) - parser.add_argument('--log_level', default='INFO') + parser.add_argument('--log_level', default='WARNING') args = parser.parse_args() @@ -104,14 +104,14 @@ def run_from_command_line(): # pragma: no cover reversal_spread_threshold=args.reversal_spread_threshold, max_processes=args.no_cpus, figure_size=args.figsize, - save_id=export_config.saveID + save_id=export_config.saveID, ) def run(data_path, output_path, qc_map, wells=None, write_traces=False, write_failed_traces=False, write_map={}, reversal_potential=-90, reversal_spread_threshold=10, - max_processes=1, figure_size=None, save_id=None): + max_processes=1, figure_size=None, save_id=None, logger=None): """ Imports traces and runs QC. @@ -174,7 +174,6 @@ def run(data_path, output_path, qc_map, wells=None, # 3. Ends with a code 00.00.00 (where 0 is any number) # # TODO: Just want to check that it ends in a 6 digit date code - # TODO: Just use ``re`` instead of ``regex`` protocols_regex = \ r'^([a-z|A-Z|_|0-9| |\-|\(|\)]+)_([0-9][0-9]\.[0-9][0-9]\.[0-9][0-9])$' protocols_regex = re.compile(protocols_regex) @@ -235,17 +234,10 @@ def run(data_path, output_path, qc_map, wells=None, if not readnames: raise ValueError('No compatible protocols specified.') - n = min(max_processes, len(readnames)) - args = zip( - readnames, - savenames, - times_list, - [output_path] * len(readnames), - [data_path] * len(readnames), - [wells] * len(readnames), - [write_traces] * len(readnames), - [save_id] * len(readnames), - ) + m = len(readnames) + n = min(max_processes, m) + args = zip(readnames, savenames, times_list, [output_path] * m, + [data_path] * m, [wells] * m, [write_traces] * m, [save_id] * m) well_selections, qc_dfs = list(zip(*starmap(n, run_qc_for_protocol, args))) qc_df = pd.concat(qc_dfs, ignore_index=True) @@ -333,21 +325,11 @@ def run(data_path, output_path, qc_map, wells=None, logging.info(f"exporting wells {wells}") - no_protocols = len(res_dict) - - n = min(max_processes, no_protocols) - args = zip( - readnames, - savenames, - times_list, - [wells_to_export] * len(savenames), - [output_path for i in readnames], - [data_path for i in readnames], - [write_traces for i in readnames], - [figure_size for i in readnames], - [reversal_potential for i in readnames], - [save_id for i in readnames], - ) + m = len(readnames) + n = min(max_processes, m) + args = zip(readnames, savenames, times_list, [wells_to_export] * m, + [output_path] * m, [data_path] * m, [write_traces] * m, + [figure_size] * m, [reversal_potential] * m, [save_id] * m) dfs = starmap(n, extract_protocol, args) if dfs: @@ -547,7 +529,7 @@ def agg_func(x): def extract_protocol(readname, savename, time_strs, selected_wells, savedir, data_path, write_traces, figure_size, reversal_potential, - save_id): + save_id, logger): # TODO: Tidy up argument order """ ??? diff --git a/pcpostprocess/scripts/summarise_herg_export.py b/pcpostprocess/scripts/summarise_herg_export.py index c9ce182c..9cbdc0ce 100644 --- a/pcpostprocess/scripts/summarise_herg_export.py +++ b/pcpostprocess/scripts/summarise_herg_export.py @@ -1,5 +1,4 @@ import argparse -import json import os import string @@ -7,10 +6,8 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd -import regex as re import scipy import seaborn as sns -from syncropatch_export.voltage_protocols import VoltageProtocol from pcpostprocess.directory_builder import setup_output_directory from pcpostprocess.scripts.run_herg_qc import create_qc_table @@ -190,7 +187,6 @@ def scatterplot_timescale_E_obs(output_path, df, passed_wells, figsize=None): plot_dfs.append(plot_df) plot_df = pd.concat(plot_dfs, ignore_index=True) - print(plot_df) plot_df['E_leak'] = (plot_df.set_index('well')['E_leak'] - plot_df.groupby('well') ['E_leak'].mean()).reset_index()['E_leak'] @@ -690,7 +686,6 @@ def create_attrition_table(qc_df, subtraction_df): agg_dict = {crit: 'min' for crit in stage_5_criteria} qc_df_sc1 = qc_df[qc_df.protocol == 'staircaseramp1'] - print(qc_df_sc1.values.shape) n_stage_1_wells = np.sum(np.all(qc_df_sc1.groupby('well') .agg(agg_dict)[original_qc_criteria].values, axis=1)) @@ -717,12 +712,6 @@ def create_attrition_table(qc_df, subtraction_df): # np.all(qc_df.groupby('well').agg(agg_dict)[stage_6_criteria].values, # axis=1)) - passed_qc_df = qc_df.groupby('well').agg(agg_dict)[stage_5_criteria] - print(passed_qc_df) - passed_wells = [well for well, row in passed_qc_df.iterrows() if np.all(row.values)] - - print(f"passed wells = {passed_wells}") - res_dict = { 'stage1': [n_stage_1_wells], 'stage2': [n_stage_2_wells], diff --git a/pyproject.toml b/pyproject.toml index 61634039..8f981439 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ dependencies = [ 'numpy>=1.21', 'matplotlib>=3.4', 'pandas>=1.3', - 'regex>=2023.12.25', 'seaborn>=0.12.2', 'jinja2>=3.1.0', 'syncropatch_export @ git+https://github.com/CardiacModelling/syncropatch_export.git' From 4e1e5156d9257c0212125cd06ad80fb2982398a3 Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Wed, 12 Nov 2025 23:59:01 +0000 Subject: [PATCH 16/17] Removed unused logger argument --- pcpostprocess/scripts/run_herg_qc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index 3bb5f795..18c87aea 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -111,7 +111,7 @@ def run_from_command_line(): # pragma: no cover def run(data_path, output_path, qc_map, wells=None, write_traces=False, write_failed_traces=False, write_map={}, reversal_potential=-90, reversal_spread_threshold=10, - max_processes=1, figure_size=None, save_id=None, logger=None): + max_processes=1, figure_size=None, save_id=None): """ Imports traces and runs QC. @@ -529,7 +529,7 @@ def agg_func(x): def extract_protocol(readname, savename, time_strs, selected_wells, savedir, data_path, write_traces, figure_size, reversal_potential, - save_id, logger): + save_id): # TODO: Tidy up argument order """ ??? From f822d6fe37440c510fce7f31e6eda4f82777668b Mon Sep 17 00:00:00 2001 From: Michael Clerx Date: Thu, 13 Nov 2025 00:02:33 +0000 Subject: [PATCH 17/17] Fixed cover pragma --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f981439..eb096bd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,6 @@ source = [ 'pcpostprocess', ] omit = [ - 'pcpostprocess/__main__.py', + 'pcpostprocess/scripts/__main__.py', ]