From 3b6a4402a0286ff976eeb8fe7ec300a1b5efc92f Mon Sep 17 00:00:00 2001 From: Fischer Robert Date: Mon, 14 Aug 2023 10:40:33 +0200 Subject: [PATCH 1/3] restart workers --- training_functions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/training_functions.py b/training_functions.py index a0f7afb..1519b73 100644 --- a/training_functions.py +++ b/training_functions.py @@ -136,13 +136,15 @@ def training_set_per_image(label_name, trainingpath, feat_data, client, lazy = F fut = fut.result() fut = fut.compute() feat_stack = fut.data - client.restart_workers() + workers = client.cluster.workers + client.restart_workers(workers) if type(feat_stack_t_idp) is not np.ndarray: fut = client.scatter(feat_stack_t_idp) fut = fut.result() fut = fut.compute() feat_stack_t_idp = fut.data - client.restart_workers() + workers = client.cluster.workers + client.restart_workers(workers) feat_stack = np.concatenate([feat_stack, feat_stack_t_idp], axis = 2) From b3a86fa36dd81f1156974e6ef74f3f37804b8149 Mon Sep 17 00:00:00 2001 From: Fischer Robert Date: Tue, 15 Aug 2023 16:17:05 +0200 Subject: [PATCH 2/3] restarting the client appears to be a working patch --- training_functions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/training_functions.py b/training_functions.py index 1519b73..d8d77ad 100644 --- a/training_functions.py +++ b/training_functions.py @@ -136,15 +136,14 @@ def training_set_per_image(label_name, trainingpath, feat_data, client, lazy = F fut = fut.result() fut = fut.compute() feat_stack = fut.data - workers = client.cluster.workers - client.restart_workers(workers) + client.restart() if type(feat_stack_t_idp) is not np.ndarray: fut = client.scatter(feat_stack_t_idp) fut = fut.result() fut = fut.compute() feat_stack_t_idp = fut.data - workers = client.cluster.workers - client.restart_workers(workers) + workers = client.cluster.workers + client.restart() feat_stack = np.concatenate([feat_stack, feat_stack_t_idp], axis = 2) From 8d983a7b2733174c06b715910612e0179f1fb37c Mon Sep 17 00:00:00 2001 From: Fischer Robert Date: Tue, 29 Aug 2023 10:30:44 +0200 Subject: [PATCH 3/3] restart client after every calculation --- training_functions.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/training_functions.py b/training_functions.py index d8d77ad..111087e 100644 --- a/training_functions.py +++ b/training_functions.py @@ -142,7 +142,6 @@ def training_set_per_image(label_name, trainingpath, feat_data, client, lazy = F fut = fut.result() fut = fut.compute() feat_stack_t_idp = fut.data - workers = client.cluster.workers client.restart() feat_stack = np.concatenate([feat_stack, feat_stack_t_idp], axis = 2) @@ -221,21 +220,21 @@ class train_segmentation: if not c1=='time': if c2 == 'x': - im = stage1.sel( x = p2).data #feature = 'original', + im = stage1.sel( x = p2)#feature = 'original', # feat_stack = stage1feat.sel(x = p2).data imfirst = None elif c2 == 'y': - im = stage1.sel( y = p2).data + im = stage1.sel( y = p2) # feat_stack = stage1feat.sel(y = p2).data imfirst = None elif c2 == 'z': - im = stage1.sel( z = p2).data + im = stage1.sel( z = p2) # feat_stack = stage1feat.sel(z = p2).data imfirst = None elif c2 == 'time': - im = stage1.sel(time = p2).data + im = stage1.sel(time = p2) # feat_stack = stage1feat.sel(time = p2).data - imfirst = stage1.sel(time = 0).data + imfirst = stage1.sel(time = 0) # if self.lazy: # # get the reference images directly as numpy array @@ -253,11 +252,13 @@ class train_segmentation: fut = fut.result() fut = fut.compute() im = fut.data + self.client.restart() if imfirst is not None and type(imfirst) is not np.ndarray: fut = self.client.scatter(imfirst) fut = fut.result() fut = fut.compute() imfirst = fut.data + self.client.restart() im8 = im-im.min()