From 4449649bd80b24c1939736d60f283462414423c5 Mon Sep 17 00:00:00 2001 From: David Kovacs Date: Fri, 14 Feb 2020 11:20:28 +0100 Subject: [PATCH 1/4] Streamlined main setup, addition of 8 slice network Main.py: - only need to add paths, filenames and reused variables once - added option to change output to binary mask using thesholding Networkds.py - Added function unet_8_slice as an alternative architecture with 10 blocks and 8 slice input pr patient pr epoch - added dropout as vector for easier overview of the dropouts Train.py - modified to accomodate changes in networkds.py with unet_8_slice --- .DS_Store | Bin 0 -> 6148 bytes pythontoolkit/networks.py | 67 +++++++++++++++++++++++++++++++++----- pythontoolkit/train.py | 3 ++ scripts/main.py | 66 ++++++++++++++++++++++++------------- 4 files changed, 104 insertions(+), 32 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 thres] = 1 + predicted_full[predicted_full[...,0] <= thres] = 0 + + #save the data out_vol.data = predicted_full out_vol.writeFile() out_vol.closeVolume() From 224326124953a25dd7e981382ab9c0250788cfa8 Mon Sep 17 00:00:00 2001 From: David Kovacs Date: Mon, 17 Feb 2020 09:57:29 +0100 Subject: [PATCH 2/4] Update .DS_Store --- .DS_Store | Bin 6148 -> 6148 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.DS_Store b/.DS_Store index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..373b1992f2ab762d27b7b9d61a96d3fa836c7f42 100644 GIT binary patch literal 6148 zcmeHKu};H44E2>t1RucR8x2*XIGb#*Z1f1pCV?z zjHu_E8NG30`?R2(9*}R1Q!Ls!`|Ga9W}eMS9$qBRPy3YBI=Ag!?R-u-9}{E17%&EQ zodMKrmgG>-Mq|JjFa}l(@b|$(8B@g|7(X2tVhaH5!yE;3?j<tD6o5;08_;v2n)pi2t*oe JFb4jVfe)V5Ti5^q delta 70 zcmZoMXfc=|#>AjHu~2NHo+1YW5HK<@2y9MdUdFPyfSHSVGdl-A2T%b} Date: Mon, 17 Feb 2020 10:09:04 +0100 Subject: [PATCH 3/4] Update README.md test --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3717ce7..4ee2827 100644 --- a/README.md +++ b/README.md @@ -114,3 +114,5 @@ cnn.compile_network() cnn.train() ``` +### testing change on sonne + From 2db50541401000e954c893794321157f4d4f2be2 Mon Sep 17 00:00:00 2001 From: David Kovacs Date: Thu, 14 May 2020 15:12:23 +0200 Subject: [PATCH 4/4] added new files and changes 14-05-2020 --- .DS_Store | Bin .Rhistory | 0 ._CMakeLists.txt | Bin 0 -> 4096 bytes ._README.md | Bin 0 -> 4096 bytes .gitignore | 0 CMakeLists.txt | 3 ++- README.md | 7 +++++-- bin/merge_tensorboard_logs.py | 0 pythontoolkit/._CMakeLists.txt | Bin 0 -> 4096 bytes pythontoolkit/._jaj_plot.py | Bin 0 -> 4096 bytes pythontoolkit/CMakeLists.txt | 1 + pythontoolkit/jaj_plot.py | 33 ++++++++++++++++++++++++++++++++ pythontoolkit/losses.py | 15 +++++++++++++++ pythontoolkit/networks.py | 13 +++++++++++-- pythontoolkit/predict.py | 0 pythontoolkit/train.py | 8 +++----- pythontoolkit/version.py | 0 reinstall.sh | 6 ++++++ scripts/._main.py | Bin 0 -> 4096 bytes scripts/data_generator.py | 0 scripts/generate_data_pickle.py | 0 scripts/main.py | 0 toolkit-config.sh.cmake | 0 23 files changed, 76 insertions(+), 10 deletions(-) mode change 100644 => 100755 .DS_Store create mode 100755 .Rhistory create mode 100755 ._CMakeLists.txt create mode 100755 ._README.md mode change 100644 => 100755 .gitignore mode change 100644 => 100755 CMakeLists.txt mode change 100644 => 100755 README.md mode change 100644 => 100755 bin/merge_tensorboard_logs.py create mode 100755 pythontoolkit/._CMakeLists.txt create mode 100755 pythontoolkit/._jaj_plot.py mode change 100644 => 100755 pythontoolkit/CMakeLists.txt create mode 100755 pythontoolkit/jaj_plot.py mode change 100644 => 100755 pythontoolkit/losses.py mode change 100644 => 100755 pythontoolkit/networks.py mode change 100644 => 100755 pythontoolkit/predict.py mode change 100644 => 100755 pythontoolkit/train.py mode change 100644 => 100755 pythontoolkit/version.py create mode 100755 reinstall.sh create mode 100755 scripts/._main.py mode change 100644 => 100755 scripts/data_generator.py mode change 100644 => 100755 scripts/generate_data_pickle.py mode change 100644 => 100755 scripts/main.py mode change 100644 => 100755 toolkit-config.sh.cmake diff --git a/.DS_Store b/.DS_Store old mode 100644 new mode 100755 diff --git a/.Rhistory b/.Rhistory new file mode 100755 index 0000000..e69de29 diff --git a/._CMakeLists.txt b/._CMakeLists.txt new file mode 100755 index 0000000000000000000000000000000000000000..32663777d3607a82e223e8cdd7fdf1ed95d76e37 GIT binary patch literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103v;u2da z2GPN=0H|C5O$#HC4;7b6&d=3LEGWoH)yqjNE-5WeO-V^CNmULA2I&dD<{t;5C;j`a z1foa5Xb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD;0ggyXA^|MKrSRBvsj@h zwK%`DC^=OjEx#yRAv3QeHLoNyKQA#Sr&1v&HLXM;DJL;68`u|y>Kf7%s{i3$kztVg G{~rM1rYm6p literal 0 HcmV?d00001 diff --git a/._README.md b/._README.md new file mode 100755 index 0000000000000000000000000000000000000000..4e6eb4726b72deb5aaa0c54bedaa652b86dc811f GIT binary patch literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103v;{37A z3ZjEy0Z_RBnifVNA1W@DoS&VLRbWEkZB G{|5l%>M9=q literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100644 new mode 100755 index d3e4f74..abed004 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,8 +2,9 @@ project (rh-scripts) cmake_minimum_required (VERSION 2.8) +#/opt/bin was changed to /homes/kovacs/toolbox for local installation if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set (CMAKE_INSTALL_PREFIX "/opt/caai" CACHE PATH "default install path" FORCE ) + set (CMAKE_INSTALL_PREFIX "/homes/kovacs/toolbox" CACHE PATH "default install path" FORCE ) endif() # Add python-lib diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 4ee2827..63e9fca --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ If your training is interrupted, you can resume from last saved checkpoint by ju again, it will automatically resume training if you did not modify any of the parameters in the config. ## HOW TO INSTALL +Ensure CMakeLists.txt is set to copy to a folder, which you can write to. For example change '/opt/bin' to your own toolbox location, '/homes/*username*/toolbox' + ``` mkdir build cd build @@ -43,6 +45,7 @@ make install ``` ## POST INSTALLATION Add "source /opt/caai/toolkit-config.sh" to .bashrc / .bash_profile +In the above replace /opt/caai/ with the location where cnn toolbox is intalled, for instance "/toolbox/". ## KNOWN ISSUES @@ -80,7 +83,8 @@ from CAAI.losses import rmse cnn = CNN(model_name='v1', data_pickle='/users/claes/projects/LowdosePET/PETrecon/HjerteFDG_mnc/data_6fold.pickle', - data_folder='/users/claes/projects/LowdosePET/PETrecon/HjerteFDG_mnc' + data_folder='/users/claes/projects/LowdosePET/PETrecon/HjerteFDG_mnc', + network_architecture = 'custom'#DGK added ) cnn.data_loader = DataGenerator(cnn.config) @@ -114,5 +118,4 @@ cnn.compile_network() cnn.train() ``` -### testing change on sonne diff --git a/bin/merge_tensorboard_logs.py b/bin/merge_tensorboard_logs.py old mode 100644 new mode 100755 diff --git a/pythontoolkit/._CMakeLists.txt b/pythontoolkit/._CMakeLists.txt new file mode 100755 index 0000000000000000000000000000000000000000..8574e0c15f8084e108165f9e7c98361094f648ac GIT binary patch literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103v63Uz< z2-U#>R9=Cmg&AlPNSvR6K|DD>S1+-kASYEXB(#H7N@49B$lKq2LwaRWmE^!kqivx2z9xsC5b>aiB{RZEnxsZ&^VuhmA;{4L0$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103vn%cF_ z5UL{qsH+A|3mZ@jD9+H!z@U_zpR1QxP>_?V=aX2Pmz)utT2z*qoLa2sUyzy?o>`Is z(aZQ2NDDA9NFmhcBo>#H7N@49B$lKq2Lv-@rlbPR{R>p*!N9*86Gnj2c2S`?C)UJSB}DIcgln1MkYq&}r6zaS~ULNBqT zq$o3~v?R5dAteW><`7Vg6a#}0x*DJf0!c{)Ihn;J1_qZInV4Bv*}}_Gi;6Sz^Fj&| zOETht@}+qxIjKN*=apopWu_JhFvJVU={o{LBQrU%1gNlBKe#k0Co?xyAtbe;1Qbs3 z0)il$iXp-!AW>ruesLLn!yuqTfPj$^LNl;KX&9BpbjLpq#QCnc*c?PNG2C9uzg_x} zvq9g?6yC~5_a1;mNe3`bF5rOZg}VUe5}1*k6?0aHd1;=~_c*n_!~gUd&$B+A=TG|k zoeA6&v?A=3ug|#?XP7%xr%e%QYz-3f7HZ^K+Ox?ay!`m3KRsu>miIgmy8K0eTfMm` zI(q4?TTu+>KFe(965VLS8WEkZB{|5k>QN*7B literal 0 HcmV?d00001 diff --git a/pythontoolkit/CMakeLists.txt b/pythontoolkit/CMakeLists.txt old mode 100644 new mode 100755 index 4449b8e..c72a4f3 --- a/pythontoolkit/CMakeLists.txt +++ b/pythontoolkit/CMakeLists.txt @@ -11,6 +11,7 @@ INSTALL(PROGRAMS networks.py train.py predict.py + jaj_plot.py DESTINATION CAAI ) diff --git a/pythontoolkit/jaj_plot.py b/pythontoolkit/jaj_plot.py new file mode 100755 index 0000000..c8f84db --- /dev/null +++ b/pythontoolkit/jaj_plot.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +import numpy as np +import matplotlib.pyplot as plt +import os + +plt.close('all') + +def jaj_plot(im, save_plot = 1, mapc = 0, lim = 0, alph = 1, axis = 1, grid = 1, colorbar = 1, colorbar_lab = ' '): + if lim == 0: + lim = im.min(), im.max() + print(lim) + elif lim == "soft": + lim = -135, 215 + print(lim) + if mapc == 0: + mapc = "gray" + print(mapc) + fov = 500. + v = (0, round(((fov/512)*im.shape[1])), 0, round(((fov/512)*im.shape[0]))) #rescaling the axis + ax = plt.imshow(im, cmap = mapc, clim = lim, alpha = alph, extent = v) + if colorbar == 1: + cbar = plt.colorbar(ax) + cbar.set_label(colorbar_lab)#, rotation = 90) + #set axes + plt.grid(color='gray', alpha = 0.5 , linestyle='-', linewidth=1.5) + if axis == 0: + plt.axis('off') + if grid == 0: + plt.grid('off') + if save_plot == 1: + print('saving plot at ' + os.getcwd()) + plt.savefig(fname = 'test.png') + return ax diff --git a/pythontoolkit/losses.py b/pythontoolkit/losses.py old mode 100644 new mode 100755 index 6ad7174..84a73a5 --- a/pythontoolkit/losses.py +++ b/pythontoolkit/losses.py @@ -1,6 +1,21 @@ from keras import backend as K +import numpy as np def rmse(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) +def dice(y_true, y_pred): + #print(np.shape(y_pred)) + smooth = 1 + print('Calculating dice coefficient') + y_true_f = K.flatten(y_true) + y_pred_f = K.flatten(y_pred) + intersection = K.sum(y_true_f * y_pred_f) + return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth) +def dice_coef_loss(y_true, y_pred): + #print(np.shape(y_pred)) + #print(np.shape(y_true)) + print(K.int_shape(y_pred)) + print(K.int_shape(y_true)) + return -dice(y_true, y_pred) diff --git a/pythontoolkit/networks.py b/pythontoolkit/networks.py old mode 100644 new mode 100755 index 6e84e1b..c4e109f --- a/pythontoolkit/networks.py +++ b/pythontoolkit/networks.py @@ -64,7 +64,13 @@ def convt_block(layer, concat, fsize): kernel_initializer='he_normal', padding='same',strides=1, activation='relu')(block14) return output -def unet_8_slice(X, f, dims_out): +def unet_8_slice(X, config): + print('---------------------') + print('---------------------') + print('Running unet_8_slice') + print('---------------------') + print('---------------------') + f, dims_out = config['n_base_filters'],config['output_channels'] def conv_block(layer,fsize,dropout,downsample=True): for i in range(1,3): layer = Conv3D(fsize, kernel_size=3, kernel_regularizer=regularizers.l2(1e-1), @@ -106,7 +112,10 @@ def convt_block(layer, concat, fsize): block9 = convt_block(block8,block1,f) block10, _ = conv_block(block9,f,dropout[6],downsample=False) - output = Conv3D(dims_out,kernel_size=3, kernel_regularizer=regularizers.l2(1e-1), + block11 = Conv3D(dims_out,kernel_size=3, kernel_regularizer=regularizers.l2(1e-1), kernel_initializer='he_normal', padding='same',strides=1, activation='relu')(block10) + + output = Activation('sigmoid')(block11) + return output diff --git a/pythontoolkit/predict.py b/pythontoolkit/predict.py old mode 100644 new mode 100755 diff --git a/pythontoolkit/train.py b/pythontoolkit/train.py old mode 100644 new mode 100755 index 88e213b..ac0e304 --- a/pythontoolkit/train.py +++ b/pythontoolkit/train.py @@ -146,13 +146,11 @@ def build_network(self,inputs=None): if self.config['network_architecture'] == 'unet': outputs = networks.unet(inputs,f=self.config['n_base_filters'],dims_out=self.config['output_channels']) - - if self.config['network_architecture'] == 'unet_8_slice': - outputs = networks.unet_8_slice(inputs,f=self.config['n_base_filters'],dims_out=self.config['output_channels']) elif self.config['network_architecture'] == 'custom' and not self.custom_network_architecture == None: - outputs = self.custom_network_architecture(inputs,config=self.config) - + #TO_DO: generalize for all custom networks. Fix "config=self.config" + outputs = self.custom_network_architecture(inputs,config=self.config)#config=self.config was exchanged for f=self.config['n_base_filters'],dims_out=self.config['output_channels'] + #outputs = networks.unet_8_slice(inputs,f=self.config['n_base_filters'],dims_out=self.config['output_channels']) else: print("You are using a network that I dont know..") exit(-1) diff --git a/pythontoolkit/version.py b/pythontoolkit/version.py old mode 100644 new mode 100755 diff --git a/reinstall.sh b/reinstall.sh new file mode 100755 index 0000000..531cde6 --- /dev/null +++ b/reinstall.sh @@ -0,0 +1,6 @@ +#reinstall the CNN package +#by running ./reinstall.sh in command-line +mkdir build +cd build +cmake .. +make install \ No newline at end of file diff --git a/scripts/._main.py b/scripts/._main.py new file mode 100755 index 0000000000000000000000000000000000000000..9ac86ce59e6fc46fc30c69bc554c2a6660ae7a97 GIT binary patch literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103vV(~Qo z4$;A|0H|C5O$#HC4;7b6&d=3LEGWoH)yqjNE-5WeO-V^CNmULA2I)DW?i&Z9uWpjp z0MVmhGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E0qLtr!naD@PukeOGKnpcvUpO=`EQ>l=XnpUEal#`g34eSd;bq#3>)&Fp>$S}zL G{|^8)kSg8) literal 0 HcmV?d00001 diff --git a/scripts/data_generator.py b/scripts/data_generator.py old mode 100644 new mode 100755 diff --git a/scripts/generate_data_pickle.py b/scripts/generate_data_pickle.py old mode 100644 new mode 100755 diff --git a/scripts/main.py b/scripts/main.py old mode 100644 new mode 100755 diff --git a/toolkit-config.sh.cmake b/toolkit-config.sh.cmake old mode 100644 new mode 100755