From b970ac1764d746c39d3faaeb286f55cd69512c72 Mon Sep 17 00:00:00 2001 From: yveskretzschmar <13993819+yveskretzschmar@users.noreply.github.com> Date: Thu, 9 Aug 2018 13:58:54 +0200 Subject: [PATCH 1/2] resume function resume function by writing current filepos of every worker to file --- blocksync.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/blocksync.py b/blocksync.py index 1654395..cd7b4e8 100644 --- a/blocksync.py +++ b/blocksync.py @@ -291,11 +291,21 @@ def sync(workerid, srcdev, dsthost, dstdev, options): t0 = time.time() t_last = t0 + + if os.path.isfile("syncpos" + f.name.replace('/','___') + "_w_" + str(workerid) + "_of_" + str(options.workers)): + syncposlog = open("syncpos" + f.name.replace('/','___') + "_w_" + str(workerid) + "_of_" + str(options.workers),"r") + startpos = int(syncposlog.readline()) - blocksize + same_blocks = ( (startpos - (chunksize * workerid)) / blocksize ) + syncposlog.close() + print("[worker %d] Start on %d of previous run" % (workerid, startpos), file = options.outfile) + f.seek(startpos) + size_blocks = ceil(chunksize / float(blocksize)) p_in.write(bytes(("%d\n%d\n" % (startpos, size_blocks)).encode("UTF-8"))) p_in.flush() print("[worker %d] Start syncing %d blocks..." % (workerid, size_blocks), file = options.outfile) + for l_block in getblocks(f, blocksize): l1_sum = hash1(l_block).digest() r1_sum = p_out.read(hash1len) @@ -319,7 +329,9 @@ def sync(workerid, srcdev, dsthost, dstdev, options): p_in.flush() p_in.write(l_block) p_in.flush() - + syncposlog = open("syncpos" + f.name.replace('/','___') + "_w_" + str(workerid) + "_of_" + str(options.workers),"w+") + syncposlog.write(str(f.tell())) + syncposlog.close() if pause_ms: time.sleep(pause_ms) From 4073d7bb8288c1f448717db48504db0c2792f84e Mon Sep 17 00:00:00 2001 From: yveskretzschmar <13993819+yveskretzschmar@users.noreply.github.com> Date: Fri, 10 Jul 2020 11:25:53 +0200 Subject: [PATCH 2/2] Update blocksync.py --- blocksync.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/blocksync.py b/blocksync.py index cd7b4e8..018c2e6 100644 --- a/blocksync.py +++ b/blocksync.py @@ -335,9 +335,6 @@ def sync(workerid, srcdev, dsthost, dstdev, options): if pause_ms: time.sleep(pause_ms) - if not interactive: - continue - t1 = float(time.time()) if (t1 - t_last) >= interval: done_blocks = same_blocks + diff_blocks @@ -347,7 +344,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): last_blocks = done_blocks t_last = t1 - if (same_blocks + diff_blocks) == size_blocks: + if (same_blocks + diff_blocks) >= size_blocks: break rate = size_blocks * blocksize / (1024.0 * 1024) / (time.time() - t0)