From 004202a2503ab9c8c8bb9ecd7e5467af319d244d Mon Sep 17 00:00:00 2001 From: froloffw7 <69122526+froloffw7@users.noreply.github.com> Date: Thu, 4 May 2023 11:03:48 +0200 Subject: [PATCH 1/2] Several changes, see commit details. Changes ======= 1. In function do_create() change 'a' open mode to 'wb', as 'a' doesn't support unbuffered operation, and "ab" doesn't allow file shrink, only enlarge. 2. Do not create remote file in dry-run mode ('--createdest' option). 3. Server return negative size if remote file doesn't exists. Worker check received value and show error message if it is negative. 4. Send file name and blocksize with separate print(), as filename could contain spaces, so split with not work correct way. 5. Enquote path. 6. Change default ciphering to 'aes128-ctr'. Probably we can just drop that option flag, if cipher is not specified. 7. Change '--compress' option to '--nocompress', as there was no way to switch compression off, which cause significant speed degradation on local network. --- blocksync.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/blocksync.py b/blocksync.py index 1e32dce..0bebb7d 100644 --- a/blocksync.py +++ b/blocksync.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 """ Synchronise block devices over the network @@ -54,14 +54,17 @@ else: USE_NOREUSE = USE_DONTNEED = False +def wrap(x): + return ("'%s'" % x) + def do_create(f, size): - f = open(f, 'a', 0) + f = open(f, 'wb', 0) f.truncate(size) f.close() -def do_open(f, mode): - f = open(f, mode) +def do_open(filename, mode): + f = open(filename, mode) if USE_NOREUSE: fadvise(f, 0, 0, POSIX_FADV_NOREUSE) f.seek(0, 2) @@ -106,8 +109,15 @@ def server(dev, deleteonexit, options): if size > 0: do_create(dev, size) - print(dev, blocksize) - f, size = do_open(dev, 'rb+') + print(dev) + print(blocksize) + try: + f, size = do_open(dev, 'rb+') + except Exception as e: + # Error accessing source device + print("-1") + return + print(size) sys.stdout.flush() @@ -237,7 +247,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): servercmd = 'tmpserver' remotescript = copy_self(workerid, cmd) - cmd += [options.interpreter, remotescript, servercmd, dstdev, '-b', str(blocksize)] + cmd += [options.interpreter, remotescript, servercmd, wrap(dstdev), '-b', str(blocksize)] cmd += ['-d', str(options.fadvise), '-1', options.hash] if options.addhash: @@ -257,16 +267,18 @@ def sync(workerid, srcdev, dsthost, dstdev, options): fadv = p_out.readline().decode('UTF-8').strip() print("[worker %d] Remote fadvise: %s" % (workerid, fadv), file = options.outfile) - p_in.write(bytes(("%d\n" % (size if options.createdest else 0)).encode("UTF-8"))) + p_in.write(bytes(("%d\n" % (size if options.createdest and not dryrun else 0)).encode("UTF-8"))) p_in.flush() - line = p_out.readline().decode('UTF-8') + # line = p_out.readline().decode('UTF-8') + a = p_out.readline().decode('UTF-8').strip() + b = p_out.readline().decode('UTF-8').strip() p.poll() if p.returncode is not None: print("[worker %d] Failed creating destination file on the remote host!" % workerid, file = options.outfile) sys.exit(1) - a, b = line.split() + # a, b = line.split() if a != dstdev: print("[worker %d] Dest device (%s) doesn't match with the remote host (%s)!" % (workerid, dstdev, a), file = options.outfile) sys.exit(1) @@ -280,6 +292,9 @@ def sync(workerid, srcdev, dsthost, dstdev, options): print("[worker %d] Error accessing device on remote host!" % workerid, file = options.outfile) sys.exit(1) remote_size = int(line) + if remote_size < 0: + print("[worker %d] Remote device doesn't exists!" % workerid, file = options.outfile) + sys.exit(1) if size > remote_size: print("[worker %d] Source device size (%d) doesn't fit into remote device size (%d)!" % (workerid, size, remote_size), file = options.outfile) sys.exit(1) @@ -355,8 +370,8 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-2", "--additionalhash", dest = "addhash", help = "second hash used for extra comparison (default is none)") parser.add_option("-d", "--fadvise", dest = "fadvise", type = "int", help = "lower cache pressure by using posix_fadivse (requires Python 3 or python-fadvise; 0 = off, 1 = local on, 2 = remote on, 3 = both on; defaults to 3)", default = 3) parser.add_option("-p", "--pause", dest = "pause", type="int", help = "pause between processing blocks, reduces system load (ms, defaults to 0)", default = 0) - parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to blowfish)", default = "blowfish") - parser.add_option("-C", "--compress", dest = "compress", action = "store_true", help = "enable compression over SSH (defaults to on)", default = True) + parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to aes128-ctr)", default = "aes128-ctr") + parser.add_option("-N", "--nocompress", dest = "compress", action = "store_false", help = "enable compression over SSH (defaults to on)", default = True) parser.add_option("-i", "--id", dest = "keyfile", help = "SSH public key file") parser.add_option("-P", "--pass", dest = "passenv", help = "environment variable containing SSH password (requires sshpass)") parser.add_option("-s", "--sudo", dest = "sudo", action = "store_true", help = "use sudo on the remote end (defaults to off)", default = False) @@ -364,7 +379,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-n", "--dryrun", dest = "dryrun", action = "store_true", help = "do a dry run (don't write anything, just report differences)", default = False) parser.add_option("-T", "--createdest", dest = "createdest", action = "store_true", help = "create destination file using truncate(2). Should be safe for subsequent syncs as truncate only modifies the file when the size differs", default = False) parser.add_option("-S", "--script", dest = "script", help = "location of script on remote host (otherwise current script is sent over)") - parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python2)", default = "python2") + parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python3)", default = "python3") parser.add_option("-t", "--interval", dest = "interval", type = "int", help = "interval between stats output (seconds, defaults to 1)", default = 1) parser.add_option("-o", "--output", dest = "outfile", help = "send output to file instead of console") parser.add_option("-f", "--force", dest = "force", action= "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists") From cb46fc6524e6f2528e7816a0874dea919d8038a2 Mon Sep 17 00:00:00 2001 From: froloffw7 <69122526+froloffw7@users.noreply.github.com> Date: Thu, 4 May 2023 11:22:54 +0200 Subject: [PATCH 2/2] Reverse mode https://github.com/theraser/blocksync/issues/18 --- blocksync.py | 54 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/blocksync.py b/blocksync.py index 0bebb7d..6258c6a 100644 --- a/blocksync.py +++ b/blocksync.py @@ -57,14 +57,14 @@ def wrap(x): return ("'%s'" % x) -def do_create(f, size): - f = open(f, 'wb', 0) +def do_create(dev, size): + f = open(dev, 'wb', 0) f.truncate(size) f.close() -def do_open(filename, mode): - f = open(filename, mode) +def do_open(dev, mode): + f = open(dev, mode) if USE_NOREUSE: fadvise(f, 0, 0, POSIX_FADV_NOREUSE) f.seek(0, 2) @@ -112,9 +112,13 @@ def server(dev, deleteonexit, options): print(dev) print(blocksize) try: - f, size = do_open(dev, 'rb+') + if options.reverse: + f, size = do_open(dev, 'rb') + else: + f, size = do_open(dev, 'rb+') + except Exception as e: - # Error accessing source device + # Error accessing file print("-1") return @@ -140,12 +144,16 @@ def server(dev, deleteonexit, options): stdout.flush() res = stdin.read(COMPLEN) if res == DIFF: - newblock = stdin.read(blocksize) - newblocklen = len(newblock) - f.seek(-newblocklen, 1) - f.write(newblock) - if USE_DONTNEED: - fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) + if options.reverse: + stdout.write(block) + stdout.flush() + else: + newblock = stdin.read(blocksize) + newblocklen = len(newblock) + f.seek(-newblocklen, 1) + f.write(newblock) + if USE_DONTNEED: + fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) if i == maxblock: break @@ -195,7 +203,10 @@ def sync(workerid, srcdev, dsthost, dstdev, options): print("[worker %d] Local fadvise: %s" % (workerid, fadv), file = options.outfile) try: - f, size = do_open(srcdev, 'rb') + if options.reverse: + f, size = do_open(srcdev, 'rb+') + else: + f, size = do_open(srcdev, 'rb') except Exception as e: print("[worker %d] Error accessing source device! %s" % (workerid, e), file = options.outfile) sys.exit(1) @@ -253,6 +264,9 @@ def sync(workerid, srcdev, dsthost, dstdev, options): if options.addhash: cmd += ['-2', options.addhash] + if options.reverse: + cmd += ['-R'] + print("[worker %d] Running: %s" % (workerid, " ".join(cmd[2 if options.passenv and (dsthost != 'localhost') else 0:])), file = options.outfile) p = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) @@ -332,9 +346,16 @@ def sync(workerid, srcdev, dsthost, dstdev, options): else: p_in.write(DIFF) p_in.flush() - p_in.write(l_block) - p_in.flush() - + if options.reverse: + newblock = p_out.read(blocksize) + newblocklen = len(newblock) + f.seek(-newblocklen, 1) + f.write(newblock) + if USE_DONTNEED: + fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) + else: + p_in.write(l_block) + p_in.flush() if pause_ms: time.sleep(pause_ms) @@ -372,6 +393,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-p", "--pause", dest = "pause", type="int", help = "pause between processing blocks, reduces system load (ms, defaults to 0)", default = 0) parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to aes128-ctr)", default = "aes128-ctr") parser.add_option("-N", "--nocompress", dest = "compress", action = "store_false", help = "enable compression over SSH (defaults to on)", default = True) + parser.add_option("-R", "--reverse", dest = "reverse", action = "store_true", help = "Remote location to local", default = False) parser.add_option("-i", "--id", dest = "keyfile", help = "SSH public key file") parser.add_option("-P", "--pass", dest = "passenv", help = "environment variable containing SSH password (requires sshpass)") parser.add_option("-s", "--sudo", dest = "sudo", action = "store_true", help = "use sudo on the remote end (defaults to off)", default = False)