Skip to content

Commit 672675a

Browse files
alainabbasalainabbas
committed
sftp
1 parent 54ebc86 commit 672675a

17 files changed

+220
-135
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
.env
22
/logs/*
33
/test
4-
/rootfs/config
4+
/rootfs/config.conf
55
/.idea
66
/rootfs/cn_s3_api/__pycache__
77
.DS_Store

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ ENV TIMEZONE=Europe/Paris \
1010
RUN apt-get clean -yq
1111
RUN apt-get update -yq
1212
RUN apt-get upgrade -yq
13-
RUN apt-get install --no-install-recommends -yq ca-certificates python3 python3-pyinotify python3-boto3 gettext
13+
RUN apt-get install --no-install-recommends -yq ca-certificates python3 python3-pyinotify python3-boto3 gettext python3-paramiko
1414

1515
COPY ./rootfs /
1616

‎env.example‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1+
# For S3
12
S3_ENDPOINT=https://s3.rbx.io.cloud.ovh.net/
23
S3_KEY_ID=xx
34
S3_KEY_SECRET=xx
45
S3_REGION=RBX
56
S3_BUCKET=My-Bucket
7+
# For Sftp
68
DIR_SOURCE=/data
7-
S3_DIR_DEST=/data-sd1
8-
9+
DIR_DEST=/data-sd1
10+
TYPE=s3|sftp
11+
SFTP_HOST=sftp.exemple.com
12+
SFTP_USER=sftp
13+
SFTP_PRIVATE_KEY_FILE=./.ssh/id_key

‎rootfs/checkdir.py‎

Lines changed: 37 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,77 +1,48 @@
1-
import os.path
2-
from cn_s3_api.api import CNS3Api
1+
import sys
2+
import os
3+
sys.path.insert(1, os.getcwd())
4+
35
import argparse
4-
import configparser
56
import logging
6-
7-
8-
__CONFIG__=configparser.RawConfigParser()
9-
__SECTION__=''
10-
def read_config(file):
11-
with open(file) as f:
12-
file_content = f.read()
13-
__CONFIG__.read_string(file_content)
14-
return __CONFIG__
15-
16-
def config(section,key,default=''):
17-
c=__CONFIG__[section]
18-
return c.get(key,default)
19-
20-
def get_config():
21-
items=__CONFIG__.items('config')
22-
data = {}
23-
for k, v in items:
24-
data[k] = v
25-
return data
26-
27-
def connect_s3():
28-
s3_api = CNS3Api(dict(
29-
aws_access_key_id=config(__SECTION__, 'access_key'),
30-
aws_secret_access_key=config(__SECTION__, 'secret_key'),
31-
endpoint_url=config(__SECTION__, 'end_point'),
32-
region_name=config(__SECTION__, 'region'),
33-
))
34-
return s3_api
35-
def copy_to_s3(file):
36-
s3_api = connect_s3()
37-
filename=os.path.basename(file)
38-
s3path=config(__SECTION__,'dir_dest') + "/" + filename
39-
try:
40-
s3_api.upload_file(config(__SECTION__,'bucket'), file, s3path)
41-
except Error:
42-
print("Erreur")
43-
44-
def delete_to_s3(file):
45-
s3_api = connect_s3()
46-
filename = os.path.basename(file)
47-
s3path = config(__SECTION__, 'dir_dest') + "/" + filename
48-
try:
49-
s3_api.remove(config(__SECTION__,'bucket'),s3path)
50-
except Error:
51-
print("Erreur")
52-
53-
def list_s3(dir):
54-
s3_api = connect_s3()
55-
return s3_api.list(config(__SECTION__,'bucket'),dir)
56-
def search_list(key,s3dir_list):
57-
dirname=config(__SECTION__, 'dir_dest')
58-
for i in s3dir_list:
7+
from utils import config
8+
from utils import s3_utils
9+
from utils import sftp_utils
10+
def search_list(key,dir_list):
11+
dirname=config.get_key(config.get_section(), 'dir_dest')
12+
for i in sdir_list:
5913
if (i['Key'] == dirname + "/" + key):
6014
return i
6115
return False
16+
def copy_file(file):
17+
type=config.get_key(config.get_section(),'type','config')
18+
if type == 's3':
19+
s3_utils.copy_to_s3(file)
20+
elif type == 'sftp':
21+
sftp_utils.copy_to_sftp(file)
22+
23+
def list_dest():
24+
dest=config.get_key(config.get_section(), 'dir_dest')
25+
type = config.get_key(config.get_section(), 'type',"config" )
26+
listdest=[]
27+
if type == "s3":
28+
listdest = s3_utils.list_s3(dest)
29+
elif type == "sftp":
30+
list=sftp_utils.list(dest)
31+
for f in list:
32+
listdest.append({'Key': f.filename,'Size': f.st_size})
33+
return listdest
6234
def main():
63-
global __SECTION__
6435
parser = argparse.ArgumentParser()
65-
parser.add_argument('--config', help='configFile',default='config')
66-
parser.add_argument('--section', help='a section in the configfile', default='s3')
36+
parser.add_argument('--config', help='configFile',default='./config.conf')
37+
parser.add_argument('--section', help='a section in the configfile', default='config')
6738
args = parser.parse_args()
68-
__SECTION__=args.section
69-
read_config(args.config)
39+
config.set_section(args.section)
40+
config.read_config(args.config)
7041
format= '%(asctime)s %(message)s'
71-
logging.basicConfig(filename=config(args.section,'logfile'), level=logging.INFO, format=format)
72-
path = config(args.section, 'dir_source')
42+
logging.basicConfig(filename=config.get_key(args.section,'logfile'), level=logging.INFO, format=format)
43+
path = config.get_key(args.section, 'dir_source')
7344
#Load s3 list
74-
listdest=list_s3(config(__SECTION__, 'dir_dest') + "/")
45+
listdest=list_dest()
7546
#list source directory
7647
for f in os.listdir(path):
7748
s=search_list(f,listdest)
@@ -82,12 +53,12 @@ def main():
8253
print(f + " OK")
8354
else:
8455
print(f + " KO BAD SIZE" + "s3-size=" + str(s['Size']) + " /size=" + str(size))
85-
copy_to_s3(path +"/" + f)
56+
copy_file(path +"/" + f)
8657
print(f + " COPIED")
8758
else:
8859
print(f + " KO NOT FOUND")
8960
#copy
90-
copy_to_s3(path +"/" + f)
61+
copy_file(path +"/" + f)
9162
print(f + " COPIED")
9263

9364

‎rootfs/docker-entrypoint.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
11
#!/bin/bash
2-
cat /template/config.conf|envsubst >/config.conf
3-
./sync.py --config /config.conf
2+
if [ "$TYPE" = "sftp" ];then
3+
echo "generation config.conf for sftp"
4+
cat /template/config_sftp.conf|envsubst >/config.conf
5+
else
6+
echo "generation config.conf for S3"
7+
cat /template/config_s3.conf|envsubst >/config.conf
8+
fi
9+
./sync.py --config /config.conf

‎rootfs/sync.py‎

Lines changed: 30 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,79 +1,51 @@
11
#!/usr/bin/python3
2-
import os.path
3-
from cn_s3_api.api import CNS3Api
2+
import sys
3+
import os
4+
sys.path.insert(1, os.getcwd())
5+
46
import argparse
5-
import configparser
7+
68
import pyinotify
79
import logging
810

9-
__CONFIG__=configparser.RawConfigParser()
10-
__SECTION__=''
11+
from utils import config
12+
from utils import s3_utils
13+
from utils import sftp_utils
14+
1115
class EventHandler(pyinotify.ProcessEvent):
1216
def process_IN_CLOSE_WRITE(self, event):
1317
logger=logging.getLogger();
1418
logger.info(f"IN_CLOSE_WRITE event detected on: {event.pathname}")
15-
copy_to_s3(event.pathname)
19+
copy_file(event.pathname)
1620
logger.info(f"IN_CLOSE_WRITE end of copy: {event.pathname}")
17-
1821
def process_IN_DELETE(self,event):
1922
logger=logging.getLogger();
2023
logger.info(f"IN_DELETE event detected on: {event.pathname}")
21-
delete_to_s3(event.pathname)
22-
def read_config(file):
23-
with open(file) as f:
24-
file_content = f.read()
25-
__CONFIG__.read_string(file_content)
26-
return __CONFIG__
27-
28-
def config(section,key,default=''):
29-
c=__CONFIG__[section]
30-
return c.get(key,default)
31-
32-
def get_config():
33-
items=__CONFIG__.items('config')
34-
data = {}
35-
for k, v in items:
36-
data[k] = v
37-
return data
38-
39-
def connect_s3():
40-
s3_api = CNS3Api(dict(
41-
aws_access_key_id=config(__SECTION__, 'access_key'),
42-
aws_secret_access_key=config(__SECTION__, 'secret_key'),
43-
endpoint_url=config(__SECTION__, 'end_point'),
44-
region_name=config(__SECTION__, 'region'),
45-
))
46-
return s3_api
47-
def copy_to_s3(file):
48-
s3_api = connect_s3()
49-
filename=os.path.basename(file)
50-
s3path=config(__SECTION__,'dir_dest') + "/" + filename
51-
try:
52-
s3_api.upload_file(config(__SECTION__,'bucket'), file, s3path)
53-
except Error:
54-
print("Erreur")
55-
56-
def delete_to_s3(file):
57-
s3_api = connect_s3()
58-
filename = os.path.basename(file)
59-
s3path = config(__SECTION__, 'dir_dest') + "/" + filename
60-
try:
61-
s3_api.remove(config(__SECTION__,'bucket'),s3path)
62-
except Error:
63-
print("Erreur")
64-
24+
delete_file(event.pathname)
25+
def copy_file(file):
26+
type=config.get_key(config.get_section(),'type','config')
27+
if type == 's3':
28+
s3_utils.copy_to_s3(file)
29+
elif type == 'sftp':
30+
sftp_utils.copy_to_sftp(file)
31+
def delete_file(file):
32+
type = config.get_key(config.get_section(), 'type', 'config')
33+
if type == 's3':
34+
s3_utils.delete_to_s3(file)
35+
elif type == 'sftp':
36+
sftp_utils.delete(file)
6537
def main():
66-
global __SECTION__
6738
parser = argparse.ArgumentParser()
6839
parser.add_argument('--config', help='configFile',default='config')
69-
parser.add_argument('--section', help='a section in the configfile', default='s3')
70-
40+
parser.add_argument('--section', help='a section in the configfile', default='config')
7141
args = parser.parse_args()
72-
__SECTION__=args.section
73-
read_config(args.config)
42+
config.set_section(args.section)
43+
config.read_config(args.config)
7444
format= '%(asctime)s %(message)s'
75-
logging.basicConfig(filename=config(args.section,'logfile'), level=logging.INFO, format=format)
76-
path = config(args.section, 'dir_source')
45+
logging.basicConfig(filename=config.get_key(args.section,'logfile'), level=logging.INFO, format=format)
46+
logger=logging.getLogger()
47+
logger.info("Start sync")
48+
path = config.get_key(args.section, 'dir_source')
7749
wm = pyinotify.WatchManager()
7850
mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_DELETE
7951
notifier = pyinotify.Notifier(wm, EventHandler())

rootfs/template/config.conf renamed to rootfs/template/config_s3.conf

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
[s3]
2-
type=s3
1+
[config]
2+
type=sftp
33
end_point=${S3_ENDPOINT}
44
access_key=${S3_KEY_ID}
55
secret_key=${S3_KEY_SECRET}
66
region= ${S3_REGION}
77
bucket=${S3_BUCKET}
88
dir_source=${DIR_SOURCE}
9-
dir_dest=${S3_DIR_DEST}
9+
dir_dest=${DIR_DEST}
1010
logfile=/var/log/s3sync/journal.log

‎rootfs/template/config_sftp.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[config]
2+
type=sftp
3+
host= ${SFTP_HOST}
4+
user= ${SFTP_USER}
5+
private_key_file=${SFTP_PRIVATE_KEY_FILE}
6+
dir_source=${DIR_SOURCE}
7+
dir_dest=${DIR_DEST}
8+
logfile=/var/log/s3sync/journal.log

‎rootfs/utils/__init__.py‎

Whitespace-only changes.

‎rootfs/utils/__pycache__/__init__.cpython-312.pyc‎

145 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)