From 3f513e9fb5b9db409c91a97d19c6760b83752d42 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Mon, 23 Nov 2015 08:00:31 +0100 Subject: [PATCH 01/57] fix sudo --- plugins/__helper.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/plugins/__helper.py b/plugins/__helper.py index 904462f..752bd06 100644 --- a/plugins/__helper.py +++ b/plugins/__helper.py @@ -254,7 +254,8 @@ def install_package(packages, update=True): envars['DEBIAN_FRONTEND'] = 'noninteractive' if update: - run_command(['apt-get', '-y', '-qq', 'update']) + run_command(['apt-get', '-y', '-qq', 'update'], runas='root') + command = ['apt-get', '-o', 'Dpkg::Options::=--force-confold', @@ -267,7 +268,8 @@ def install_package(packages, update=True): elif distribution.lower() in ['centos', 'redhat', 'fedora', 'amazon']: if update: - run_command(['yum', '-y', 'clean', 'all']) + run_command(['yum', '-y', 'clean', 'all'], runas='root') + command = ['yum', '-y', '--nogpgcheck', @@ -283,7 +285,7 @@ def install_package(packages, update=True): elif distribution.lower() in ['arch']: if update: - run_command(['pacman', '-S', '--noconfirm', 'pacman']) + run_command(['pacman', '-S', '--noconfirm', 'pacman'], runas='root') command = ['pacman', '-S', '--noconfirm', @@ -292,7 +294,7 @@ def install_package(packages, update=True): else: return 1, '', "Distribution not supported: %s" % distribution - return run_command(command, envars=envars) + return run_command(command, envars=envars, runas='root') except Exception as e: return 1, '', "Error installing packages %s" % e @@ -677,14 +679,9 @@ def command(self, command, args=None, std_input=None, run_as=None, working_dir=N if run_as and not is_win(): if not check_sudo(): return 255, '', 'Sudo is not available:' + # don't use su - xxx or env variables will not be available - if run_as == _ROOT: - command = [which('sudo'), ' '.join(map(str, command))] - - else: - command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))] - - # :TODO: Run_as for windows :S + command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))] try: p = Popen( From 03518a613bf9883fced2ac8ae3a272a4dc784204 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 24 Nov 2015 11:28:02 +0100 Subject: [PATCH 02/57] Fix valid data on next execution --- plugins/plugin_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/plugin_monitor.py b/plugins/plugin_monitor.py index bdde257..def627d 100644 --- a/plugins/plugin_monitor.py +++ b/plugins/plugin_monitor.py @@ -144,7 +144,7 @@ def cmd_monitor_get(self, *argv, **kwargs): retval.append(self._parse_script_name(script) + GLUE + str(interval) + GLUE + from_cache) # Execute script if cache wont be valid on next command_get execution - if not self._cache_read(script, interval - COMMAND_INTERVAL + CACHE_SOFT_TIME): + if not self._cache_read(script, interval - COMMAND_INTERVAL - CACHE_SOFT_TIME): to_execute.append({'script': script, 'runas': runas}) for data in to_execute: From 07752ae6b141e9224f77442c4943386a398f5736 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 24 Nov 2015 12:39:03 +0100 Subject: [PATCH 03/57] Cron check from 5 minutes to 2 minutes --- cron.d/ecmanaged-ecagent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron.d/ecmanaged-ecagent b/cron.d/ecmanaged-ecagent index e24a3cd..b9df2e3 100644 --- a/cron.d/ecmanaged-ecagent +++ b/cron.d/ecmanaged-ecagent @@ -1,4 +1,4 @@ SHELL=/bin/bash HOME=/opt/ecmanaged/ecagent/ -*/5 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 +*/2 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 From a413597a7a61c20fc8d2ec2bb7441fe78fd257a2 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 24 Nov 2015 12:39:27 +0100 Subject: [PATCH 04/57] Delete ecmanaged-ecagent-init --- cron.d/ecmanaged-ecagent-init | 1 - 1 file changed, 1 deletion(-) delete mode 100755 cron.d/ecmanaged-ecagent-init diff --git a/cron.d/ecmanaged-ecagent-init b/cron.d/ecmanaged-ecagent-init deleted file mode 100755 index f123cff..0000000 --- a/cron.d/ecmanaged-ecagent-init +++ /dev/null @@ -1 +0,0 @@ -*/5 * * * * root /etc/init.d/ecagentd check > /dev/null 2>&1 From 05a203a11ee3828dc0b6d09cc8b9ac49127881ee Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 24 Nov 2015 12:39:33 +0100 Subject: [PATCH 05/57] Delete ecmanaged-ecagent-systemd --- cron.d/ecmanaged-ecagent-systemd | 1 - 1 file changed, 1 deletion(-) delete mode 100644 cron.d/ecmanaged-ecagent-systemd diff --git a/cron.d/ecmanaged-ecagent-systemd b/cron.d/ecmanaged-ecagent-systemd deleted file mode 100644 index 66f2950..0000000 --- a/cron.d/ecmanaged-ecagent-systemd +++ /dev/null @@ -1 +0,0 @@ -*/5 * * * root service ecagentd status || service ecagentd start > /dev/null 2>&1 From 06da64dd4a75c2b19f8f9cf5ad8a92a8ed6487ef Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 24 Nov 2015 13:02:23 +0100 Subject: [PATCH 06/57] Update setup.py --- setup.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/setup.py b/setup.py index b712bec..64457fe 100644 --- a/setup.py +++ b/setup.py @@ -16,28 +16,6 @@ from distutils.core import setup - -def _create_data_files(): - data_files=[('config', ['config/ecagent.init.cfg', 'config/xmpp_cert.pub']), - ('monitor/mplugin/__base__', ['monitor/mplugin/__base__/data.json']), - ('/etc/sudoers.d', ['sudoers.d/ecmanaged']), - ('',['configure.py','ecagent.bat', 'ecagent.sh', 'ecagentd.tac']) - ] - - from commands import getstatusoutput - retcode, systemd_system_unit_dir = getstatusoutput('pkg-config systemd --variable=systemdsystemunitdir') - - if retcode == 0: - # systemd - data_files.append((systemd_system_unit_dir, ['ecagentd.service'])) - data_files.append(('/etc/cron.d',['cron.d/ecmanaged-ecagent-systemd'])) - else: - data_files.append(('/etc/init.d',['ecagentd'])) - data_files.append(('/etc/cron.d',['cron.d/ecmanaged-ecagent-init'])) - - return data_files - - setup(name='ecmanaged-ecagent', version='3.0', license='Apache v2', @@ -50,9 +28,8 @@ def _create_data_files(): maintainer = 'Arindam Choudhury', maintainer_email = 'arindam@live.com', - url='www.ecmanaged.com', - + platforms=['All'], packages=['ecagent', 'plugins','monitor.mplugin.__base__'], From a4b5135d6d0c79889158eedf76428243d16f8033 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Mon, 30 Nov 2015 17:46:21 +0100 Subject: [PATCH 07/57] new debian build --- build/debian/changelog | 5 ++ build/debian/conffiles | 1 - build/debian/ecmanaged-ecagent.dirs | 1 - build/debian/ecmanaged-ecagent.init | 124 ---------------------------- build/debian/postinst | 5 ++ build/debian/prerm | 9 +- build/debian/rules | 4 - config/ecagent.init.cfg | 23 ------ configure.py | 5 -- ecagentd.tac | 5 -- setup.py | 2 +- 11 files changed, 16 insertions(+), 168 deletions(-) delete mode 100755 build/debian/ecmanaged-ecagent.init delete mode 100644 config/ecagent.init.cfg diff --git a/build/debian/changelog b/build/debian/changelog index e69de29..eac9407 100644 --- a/build/debian/changelog +++ b/build/debian/changelog @@ -0,0 +1,5 @@ +ecmanaged-ecagent (3.0.0-1) stable; urgency=low + + * version 3.0 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 diff --git a/build/debian/conffiles b/build/debian/conffiles index b1edf78..e69de29 100644 --- a/build/debian/conffiles +++ b/build/debian/conffiles @@ -1 +0,0 @@ -/opt/ecmanaged/ecagent/config/ecagent.cfg diff --git a/build/debian/ecmanaged-ecagent.dirs b/build/debian/ecmanaged-ecagent.dirs index eca14de..eb6ebf4 100644 --- a/build/debian/ecmanaged-ecagent.dirs +++ b/build/debian/ecmanaged-ecagent.dirs @@ -1,2 +1 @@ -etc/apt/sources.list.d opt/ecmanaged/ecagent diff --git a/build/debian/ecmanaged-ecagent.init b/build/debian/ecmanaged-ecagent.init deleted file mode 100755 index 53341fc..0000000 --- a/build/debian/ecmanaged-ecagent.init +++ /dev/null @@ -1,124 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: ecagentd -# Required-Start: $remote_fs $network -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start ECM Agent daemon -### END INIT INFO - -set +e - -NAME=ecagentd -TWISTD=/usr/bin/twistd -TAC=$NAME.tac -DESC="ECM Agent" - -test -x $TWISTD || exit 0 - -DIR=/opt/ecmanaged/ecagent -PID_FILE=$DIR/twistd.pid - -# Check working dir -if test ! -d "$DIR"; then - echo "Unable to access work dir: $DIR" - exit 1; -fi - -export LANG="C" -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -# clean APT related environment when started from cloud-init -# bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=439763 - -unset PERL_DL_NONLAZY -unset DEBCONF_REDIR -unset DEBIAN_FRONTEND -unset DEBIAN_HAS_FRONTEND -unset DPKG_NO_TSTP - -# define LSB log_* functions. -. /lib/lsb/init-functions - -getpid() { - echo `cat $PID_FILE 2>/dev/null` -} - -killagent() { - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) kill -9 $PID >/dev/null 2>&1; pkill -f ${TAC} >/dev/null 2>&1 ;; - *) pkill -f ${TAC} >/dev/null 2>&1 ;; - esac -} - -zombies() { - PID=$(getpid) - for i in $(ps axo ppid,state,etimes|grep -w Z|awk '{if ($3 >= 300) print $1 }'); do - if [ $i -eq ${PID} ]; then - $0 restart - fi - done -} - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - killagent - /bin/rm -f $PID_FILE > /dev/null 2>&1 - start-stop-daemon --oknodo --start --pidfile $PID_FILE \ - --background --chdir $DIR --exec $TWISTD -- -ny $TAC >/dev/null 2>&1 - case "$?" in - 0) log_end_msg 0 ;; - *) log_end_msg 1; exit 1 ;; - esac - ;; - - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - start-stop-daemon --retry 5 --oknodo --stop --signal 9 --quiet --pidfile $PID_FILE 2>/dev/null - case "$?" in - 0) log_end_msg 0; killagent ;; - *) log_end_msg 1; killagent ; exit 1 ;; - esac - ;; - - status) - log_daemon_msg "Checking status of $DESC" "$NAME" - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) log_end_msg 0 ;; - *) log_end_msg 1; exit 1 ;; - esac - ;; - - check) - log_daemon_msg "Checking status of $DESC" "$NAME" - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) zombies; log_end_msg 0 ;; - *) $0 restart ;; - esac - ;; - - kill) - killagent - ;; - - restart) - $0 stop - $0 start - ;; - - *) - echo "Usage: /etc/init.d/$NAME {start|stop|restart|status|check}" >&2 - exit 1 - ;; - -esac - -exit 0 diff --git a/build/debian/postinst b/build/debian/postinst index 6ef5e00..54c2e8e 100644 --- a/build/debian/postinst +++ b/build/debian/postinst @@ -3,3 +3,8 @@ # # see: dh_installdeb(1) +# Stop ecagent +INIT=/opt/ecmanaged/ecagent/init +if [ -x ${INIT} ]; then + ${INIT} start +fi diff --git a/build/debian/prerm b/build/debian/prerm index c8bfa2f..49cbde6 100644 --- a/build/debian/prerm +++ b/build/debian/prerm @@ -5,13 +5,14 @@ set -e -CONF=/opt/ecmanaged/ecagent/config/ecagent.cfg +# Stop ecagent +INIT=/opt/ecmanaged/ecagent/init +if [ -x ${INIT} ]; then + ${INIT} stop +fi case "$1" in remove) - if test ! -f "$CONF"; then - rm -f $CONF - fi ;; upgrade|deconfigure) ;; diff --git a/build/debian/rules b/build/debian/rules index 188aeb6..4bf4b42 100755 --- a/build/debian/rules +++ b/build/debian/rules @@ -6,12 +6,8 @@ export DH_ALWAYS_EXCLUDE=GIT:ecagent/.git %: dh $@ -override_dh_installinit: - dh_installinit --init-script=ecagentd -- "start 90 2 3 4 5 . stop 10 0 1 6 ." - override_dh_fixperms: dh_fixperms chmod 0700 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config chmod 0400 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config/ecagent.cfg chmod 0755 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent - diff --git a/config/ecagent.init.cfg b/config/ecagent.init.cfg deleted file mode 100644 index 955f461..0000000 --- a/config/ecagent.init.cfg +++ /dev/null @@ -1,23 +0,0 @@ -#XMPP config options - -[XMPP] -retry_max_delay = 60 - -#Logging options (critical, error, warning, info, debug) -[Log] -log_to_file = True -log_path = ./log/ecagentd.log -log_level = info -max_log_file_size = 1024 -max_log_files = 10 - -[Plugins] -timeout = 300 -python_interpreter_windows = ../python27/pythonw.exe -python_interpreter_linux = /usr/bin/python -tools_path_linux = ./tools/linux -tools_path_windows = .\tools\win32 - -[ssl] -public_key = ./config/xmpp_cert.pub -private_key = ./config/private.key diff --git a/configure.py b/configure.py index 1ed0562..cf8aae7 100755 --- a/configure.py +++ b/configure.py @@ -56,11 +56,6 @@ # Parse config file or end execution config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') -config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.init.cfg') - -# Is initial config (move init to cfg) -if not os.path.exists(config_file) and os.path.exists(config_file_init): - os.rename(config_file_init, config_file) # manipulate configuration file if not os.path.isfile(config_file): diff --git a/ecagentd.tac b/ecagentd.tac index bb0ef0f..e1076bd 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -59,11 +59,6 @@ open(pid_file, 'w').write(str(os.getpid())) # Start agent and setup logging config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') -config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.init.cfg') - -# Is initial config (move init to cfg) -if not os.path.exists(config_file) and os.path.exists(config_file_init): - os.rename(config_file_init, config_file) os.chmod(config_file, stat.S_IRWXU) diff --git a/setup.py b/setup.py index 64457fe..4db463e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ packages=['ecagent', 'plugins','monitor.mplugin.__base__'], - data_files=[('config', ['config/ecagent.init.cfg', 'config/xmpp_cert.pub']), + data_files=[('config', ['config/ecagent.cfg', 'config/xmpp_cert.pub']), ('monitor/mplugin/__base__', ['monitor/mplugin/__base__/data.json']), ('/etc/sudoers.d', ['sudoers.d/ecmanaged']), ('/etc/cron.d', ['cron.d/ecmanaged-ecagent']), From 021623234f178aab28315065f9386c99005b6829 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Mon, 30 Nov 2015 17:56:53 +0100 Subject: [PATCH 08/57] new debian build --- build/debian.old/changelog | 5 ++ build/debian.old/compat | 1 + build/{debian => debian.old}/conffiles | 0 build/debian.old/control | 13 +++++ build/debian.old/copyright | 18 ++++++ build/{debian => debian.old}/cron.d | 0 build/{debian => debian.old}/dirs | 0 .../ecmanaged-ecagent.dirs | 0 build/{debian => debian.old}/install | 0 build/debian.old/postinst | 10 ++++ build/{debian => debian.old}/prerm | 0 build/debian.old/rules | 13 +++++ build/{debian => debian.old}/templates | 0 build/debian/compat | 2 +- build/debian/control | 12 ++-- build/debian/copyright | 55 ++++++++++++++----- build/debian/postinst | 22 +++++--- {dist_build => build}/debian/preinst | 0 build/debian/rules | 20 ++++--- cron.d/ecmanaged-ecagent | 6 +- dist_build/debian/changelog | 17 ------ dist_build/debian/compat | 1 - dist_build/debian/control | 13 ----- dist_build/debian/copyright | 45 --------------- dist_build/debian/postinst | 16 ------ dist_build/debian/rules | 15 ----- ecagentd.service | 22 -------- setup.cfg | 7 +-- 28 files changed, 139 insertions(+), 174 deletions(-) create mode 100644 build/debian.old/changelog create mode 100644 build/debian.old/compat rename build/{debian => debian.old}/conffiles (100%) create mode 100644 build/debian.old/control create mode 100644 build/debian.old/copyright rename build/{debian => debian.old}/cron.d (100%) rename build/{debian => debian.old}/dirs (100%) rename build/{debian => debian.old}/ecmanaged-ecagent.dirs (100%) rename build/{debian => debian.old}/install (100%) create mode 100644 build/debian.old/postinst rename build/{debian => debian.old}/prerm (100%) create mode 100755 build/debian.old/rules rename build/{debian => debian.old}/templates (100%) rename {dist_build => build}/debian/preinst (100%) delete mode 100644 dist_build/debian/changelog delete mode 100644 dist_build/debian/compat delete mode 100644 dist_build/debian/control delete mode 100644 dist_build/debian/copyright delete mode 100644 dist_build/debian/postinst delete mode 100644 dist_build/debian/rules delete mode 100644 ecagentd.service diff --git a/build/debian.old/changelog b/build/debian.old/changelog new file mode 100644 index 0000000..eac9407 --- /dev/null +++ b/build/debian.old/changelog @@ -0,0 +1,5 @@ +ecmanaged-ecagent (3.0.0-1) stable; urgency=low + + * version 3.0 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 diff --git a/build/debian.old/compat b/build/debian.old/compat new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/build/debian.old/compat @@ -0,0 +1 @@ +5 diff --git a/build/debian/conffiles b/build/debian.old/conffiles similarity index 100% rename from build/debian/conffiles rename to build/debian.old/conffiles diff --git a/build/debian.old/control b/build/debian.old/control new file mode 100644 index 0000000..5e40af8 --- /dev/null +++ b/build/debian.old/control @@ -0,0 +1,13 @@ +Source: ecmanaged-ecagent +Section: admin +Priority: optional +Maintainer: Juan Carlos Moreno +Build-Depends: cdbs, debhelper (>= 7.0.50) +Standards-Version: 3.9.2 +Homepage: http://www.ecmanaged.com + +Package: ecmanaged-ecagent +Architecture: all +Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-apt, python-httplib2 +Description: ecmanaged-ecagent + ECM Agent - ECMANAGED Monitor and deploy agent diff --git a/build/debian.old/copyright b/build/debian.old/copyright new file mode 100644 index 0000000..bc8f769 --- /dev/null +++ b/build/debian.old/copyright @@ -0,0 +1,18 @@ +Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=166 +Upstream-Name: ecmanaged-ecagent + +Files: * +Copyright: 2012 Juan Carlos Moreno +License: Apache License, Version 2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. diff --git a/build/debian/cron.d b/build/debian.old/cron.d similarity index 100% rename from build/debian/cron.d rename to build/debian.old/cron.d diff --git a/build/debian/dirs b/build/debian.old/dirs similarity index 100% rename from build/debian/dirs rename to build/debian.old/dirs diff --git a/build/debian/ecmanaged-ecagent.dirs b/build/debian.old/ecmanaged-ecagent.dirs similarity index 100% rename from build/debian/ecmanaged-ecagent.dirs rename to build/debian.old/ecmanaged-ecagent.dirs diff --git a/build/debian/install b/build/debian.old/install similarity index 100% rename from build/debian/install rename to build/debian.old/install diff --git a/build/debian.old/postinst b/build/debian.old/postinst new file mode 100644 index 0000000..54c2e8e --- /dev/null +++ b/build/debian.old/postinst @@ -0,0 +1,10 @@ +#! /bin/bash +# postinst script for ecmanaged-ecagent +# +# see: dh_installdeb(1) + +# Stop ecagent +INIT=/opt/ecmanaged/ecagent/init +if [ -x ${INIT} ]; then + ${INIT} start +fi diff --git a/build/debian/prerm b/build/debian.old/prerm similarity index 100% rename from build/debian/prerm rename to build/debian.old/prerm diff --git a/build/debian.old/rules b/build/debian.old/rules new file mode 100755 index 0000000..4bf4b42 --- /dev/null +++ b/build/debian.old/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f + +# Exclude git directory +export DH_ALWAYS_EXCLUDE=GIT:ecagent/.git + +%: + dh $@ + +override_dh_fixperms: + dh_fixperms + chmod 0700 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config + chmod 0400 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config/ecagent.cfg + chmod 0755 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent diff --git a/build/debian/templates b/build/debian.old/templates similarity index 100% rename from build/debian/templates rename to build/debian.old/templates diff --git a/build/debian/compat b/build/debian/compat index 7ed6ff8..f11c82a 100644 --- a/build/debian/compat +++ b/build/debian/compat @@ -1 +1 @@ -5 +9 \ No newline at end of file diff --git a/build/debian/control b/build/debian/control index 5e40af8..b6aeec7 100644 --- a/build/debian/control +++ b/build/debian/control @@ -1,13 +1,13 @@ Source: ecmanaged-ecagent +Maintainer: Arindam Choudhury Section: admin Priority: optional -Maintainer: Juan Carlos Moreno -Build-Depends: cdbs, debhelper (>= 7.0.50) -Standards-Version: 3.9.2 +Build-Depends: dh-python, python (>= 2.6.6-3), debhelper (>= 9) +Standards-Version: 3.9.6 Homepage: http://www.ecmanaged.com Package: ecmanaged-ecagent Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-apt, python-httplib2 -Description: ecmanaged-ecagent - ECM Agent - ECMANAGED Monitor and deploy agent +Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo, python-dbus +Description: ECM Agent - ECMANAGED Monitor and deploy agent + Deploy and monitor agent for ECmanaged cloud management tool, based on XMPP protocol. diff --git a/build/debian/copyright b/build/debian/copyright index bc8f769..70738bb 100644 --- a/build/debian/copyright +++ b/build/debian/copyright @@ -1,18 +1,45 @@ -Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=166 +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: ecmanaged-ecagent +Upstream-Contact: Juan Carlos Moreno +Source: http://www.ecmanaged.com Files: * -Copyright: 2012 Juan Carlos Moreno -License: Apache License, Version 2.0 +Copyright: 2012, ACKSTORM, S.L. All rights reserved. +License: GPL-3+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at + your option) any later version. + . + In addition, as a special exception, the copyright holders give + permission to link the code of portions of this program with the + OpenSSL library under certain conditions as described in each + individual source file, and distribute linked combinations including + the two. + . + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the + file(s), but you are not obligated to do so. If you do not wish to do + so, delete this exception statement from your version. If you delete + this exception statement from all source files in the program, then + also delete it here. + . + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License (GPL) version 3 can be found at + /usr/share/common-licenses/GPL-3. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. +Files: debian/* +Copyright: 2010-2012, ACK STORM, S.L. +License: GPL-3+ + On Debian GNU/Linux systems, the complete text of the GNU General + Public License (GPL) version 3 can be found at + /usr/share/common-licenses/GPL-3 \ No newline at end of file diff --git a/build/debian/postinst b/build/debian/postinst index 54c2e8e..14f2c79 100644 --- a/build/debian/postinst +++ b/build/debian/postinst @@ -1,10 +1,16 @@ -#! /bin/bash -# postinst script for ecmanaged-ecagent -# -# see: dh_installdeb(1) +#!/bin/sh -# Stop ecagent -INIT=/opt/ecmanaged/ecagent/init -if [ -x ${INIT} ]; then - ${INIT} start +set -e + +if getent passwd ecmanaged >/dev/null 2>&1; then + chown -R ecmanaged:ecmanaged /opt/ecmanaged + mkdir -p /etc/ecmanaged + chown -R ecmanaged:ecmanaged /etc/ecmanaged fi + +case "$1" in + configure) + if [ -f /opt/ecmanaged/ecagent/init ]; then + /opt/ecmanaged/ecagent/init start > /dev/null 2>&1 + fi +esac \ No newline at end of file diff --git a/dist_build/debian/preinst b/build/debian/preinst similarity index 100% rename from dist_build/debian/preinst rename to build/debian/preinst diff --git a/build/debian/rules b/build/debian/rules index 4bf4b42..3b90b0d 100755 --- a/build/debian/rules +++ b/build/debian/rules @@ -1,13 +1,15 @@ #!/usr/bin/make -f +DH_VERBOSE = 1 +export DH_OPTIONS=-v +# This file was automatically generated by stdeb 0.8.2 at +# Thu, 30 Jul 2015 08:59:04 +0000 +export PYBUILD_NAME=ecmanaged-ecagent +%: + dh $@ --with python2 --buildsystem=pybuild -# Exclude git directory -export DH_ALWAYS_EXCLUDE=GIT:ecagent/.git +override_dh_auto_install: + python setup.py install --root=debian/ecmanaged-ecagent --install-layout=deb --no-compile + +override_dh_auto_build: -%: - dh $@ -override_dh_fixperms: - dh_fixperms - chmod 0700 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config - chmod 0400 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config/ecagent.cfg - chmod 0755 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent diff --git a/cron.d/ecmanaged-ecagent b/cron.d/ecmanaged-ecagent index b9df2e3..4706770 100644 --- a/cron.d/ecmanaged-ecagent +++ b/cron.d/ecmanaged-ecagent @@ -1,4 +1,8 @@ SHELL=/bin/bash HOME=/opt/ecmanaged/ecagent/ -*/2 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 +# Start agent on reboot +@reboot root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 + +# Check agent every two minutes +*/2 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 diff --git a/dist_build/debian/changelog b/dist_build/debian/changelog deleted file mode 100644 index a47addb..0000000 --- a/dist_build/debian/changelog +++ /dev/null @@ -1,17 +0,0 @@ -ecmanaged-ecagent (3.0-1) UNRELEASED; urgency=medium - - * better system integration - - -- Arindam Choudhury Thu, 30 Jul 2015 09:06:14 +0000 - -ecmanaged-ecagent (2.2-2) unstable; urgency=low - - * Support for systemd - - -- Arindam Choudhury Thu, 30 Jul 2015 09:06:14 +0000 - -ecmanaged-ecagent (2.2-1) unstable; urgency=low - - * source package automatically created by stdeb 0.8.2 - - -- Arindam Choudhury Thu, 30 Jul 2015 08:59:04 +0000 \ No newline at end of file diff --git a/dist_build/debian/compat b/dist_build/debian/compat deleted file mode 100644 index f11c82a..0000000 --- a/dist_build/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 \ No newline at end of file diff --git a/dist_build/debian/control b/dist_build/debian/control deleted file mode 100644 index b6aeec7..0000000 --- a/dist_build/debian/control +++ /dev/null @@ -1,13 +0,0 @@ -Source: ecmanaged-ecagent -Maintainer: Arindam Choudhury -Section: admin -Priority: optional -Build-Depends: dh-python, python (>= 2.6.6-3), debhelper (>= 9) -Standards-Version: 3.9.6 -Homepage: http://www.ecmanaged.com - -Package: ecmanaged-ecagent -Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo, python-dbus -Description: ECM Agent - ECMANAGED Monitor and deploy agent - Deploy and monitor agent for ECmanaged cloud management tool, based on XMPP protocol. diff --git a/dist_build/debian/copyright b/dist_build/debian/copyright deleted file mode 100644 index 70738bb..0000000 --- a/dist_build/debian/copyright +++ /dev/null @@ -1,45 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: ecmanaged-ecagent -Upstream-Contact: Juan Carlos Moreno -Source: http://www.ecmanaged.com - -Files: * -Copyright: 2012, ACKSTORM, S.L. All rights reserved. -License: GPL-3+ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or (at - your option) any later version. - . - In addition, as a special exception, the copyright holders give - permission to link the code of portions of this program with the - OpenSSL library under certain conditions as described in each - individual source file, and distribute linked combinations including - the two. - . - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the - file(s), but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. If you delete - this exception statement from all source files in the program, then - also delete it here. - . - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - . - You should have received a copy of the GNU General Public License - along with this program. If not, see . - . - On Debian GNU/Linux systems, the complete text of the GNU General - Public License (GPL) version 3 can be found at - /usr/share/common-licenses/GPL-3. - -Files: debian/* -Copyright: 2010-2012, ACK STORM, S.L. -License: GPL-3+ - On Debian GNU/Linux systems, the complete text of the GNU General - Public License (GPL) version 3 can be found at - /usr/share/common-licenses/GPL-3 \ No newline at end of file diff --git a/dist_build/debian/postinst b/dist_build/debian/postinst deleted file mode 100644 index 14f2c79..0000000 --- a/dist_build/debian/postinst +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -set -e - -if getent passwd ecmanaged >/dev/null 2>&1; then - chown -R ecmanaged:ecmanaged /opt/ecmanaged - mkdir -p /etc/ecmanaged - chown -R ecmanaged:ecmanaged /etc/ecmanaged -fi - -case "$1" in - configure) - if [ -f /opt/ecmanaged/ecagent/init ]; then - /opt/ecmanaged/ecagent/init start > /dev/null 2>&1 - fi -esac \ No newline at end of file diff --git a/dist_build/debian/rules b/dist_build/debian/rules deleted file mode 100644 index 3b90b0d..0000000 --- a/dist_build/debian/rules +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/make -f -DH_VERBOSE = 1 -export DH_OPTIONS=-v -# This file was automatically generated by stdeb 0.8.2 at -# Thu, 30 Jul 2015 08:59:04 +0000 -export PYBUILD_NAME=ecmanaged-ecagent -%: - dh $@ --with python2 --buildsystem=pybuild - -override_dh_auto_install: - python setup.py install --root=debian/ecmanaged-ecagent --install-layout=deb --no-compile - -override_dh_auto_build: - - diff --git a/ecagentd.service b/ecagentd.service deleted file mode 100644 index ed21258..0000000 --- a/ecagentd.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=ECManaged Agent for monitoring and deployment - -[Service] -Type=simple -User=ecmanaged -Group=ecmanaged -PIDFile=/opt/ecmanaged/ecagent/twistd.pid -#ExecStart=/usr/bin/twistd -y /opt/ecmanaged/ecagent/ecagentd.tac -ExecStart=\ - /usr/bin/twistd \ - --nodaemon \ - --pidfile=/opt/ecmanaged/ecagent/twistd.pid \ - --no_save \ - --python=/opt/ecmanaged/ecagent/ecagentd.tac - -WorkingDirectory=/opt/ecmanaged/ecagent -Restart=always -RestartSec=300 - -[Install] -WantedBy=multi-user.target diff --git a/setup.cfg b/setup.cfg index 586e0cd..a7a39d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ provides = ecmanaged-ecagent build_requires = systemd requires = python2 - python-devel python-twisted python-protocols python-configobj @@ -27,8 +26,4 @@ requires = python2 python-crypto python-httplib2 shadow-utils - python-pip - pygobject3 - PolicyKit - PackageKit - dbus-python \ No newline at end of file + dbus-python From 5d3d24be2d1d09bf072ccc9ea4193a693ff8e853 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 30 Nov 2015 18:20:33 +0100 Subject: [PATCH 09/57] add conf --- .gitignore | 1 - config/ecagent.cfg | 23 +++++++++++++++++++++++ setup.py | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 config/ecagent.cfg diff --git a/.gitignore b/.gitignore index 5a101d5..661dc4e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ log monitor/mplugin/__base__/data.json */*.pyc -config/ecagent.cfg monitor/.cache/* monitor/mplugin/__base__/.counter.dat monitor/mplugin/__base__/.touch diff --git a/config/ecagent.cfg b/config/ecagent.cfg new file mode 100644 index 0000000..955f461 --- /dev/null +++ b/config/ecagent.cfg @@ -0,0 +1,23 @@ +#XMPP config options + +[XMPP] +retry_max_delay = 60 + +#Logging options (critical, error, warning, info, debug) +[Log] +log_to_file = True +log_path = ./log/ecagentd.log +log_level = info +max_log_file_size = 1024 +max_log_files = 10 + +[Plugins] +timeout = 300 +python_interpreter_windows = ../python27/pythonw.exe +python_interpreter_linux = /usr/bin/python +tools_path_linux = ./tools/linux +tools_path_windows = .\tools\win32 + +[ssl] +public_key = ./config/xmpp_cert.pub +private_key = ./config/private.key diff --git a/setup.py b/setup.py index 4db463e..b49daf4 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ from distutils.core import setup setup(name='ecmanaged-ecagent', - version='3.0', + version='3.0.0', license='Apache v2', description='ECManaged Agent - Monitoring and deployment agent', long_description='ECManaged Agent - Monitoring and deployment agent', From 01b4584c3af6612531394a075b523337b2d7f204 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 30 Nov 2015 18:21:41 +0100 Subject: [PATCH 10/57] changed version --- build/debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/debian/changelog b/build/debian/changelog index eac9407..2035bbe 100644 --- a/build/debian/changelog +++ b/build/debian/changelog @@ -1,4 +1,4 @@ -ecmanaged-ecagent (3.0.0-1) stable; urgency=low +ecmanaged-ecagent (3.0-1) stable; urgency=low * version 3.0 From 9640af40e36cb6d06b03b335f2b45092b1fc366d Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 30 Nov 2015 18:22:37 +0100 Subject: [PATCH 11/57] service can be hosted on different server so dont check using which --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b49daf4..4db463e 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ from distutils.core import setup setup(name='ecmanaged-ecagent', - version='3.0.0', + version='3.0', license='Apache v2', description='ECManaged Agent - Monitoring and deployment agent', long_description='ECManaged Agent - Monitoring and deployment agent', From 804d8f602dbfc8a816f709d16beeab8b9bb45f8b Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 4 Dec 2015 16:35:34 +0100 Subject: [PATCH 12/57] logging --- ecagent/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecagent/client.py b/ecagent/client.py index 9bc71d8..92199c7 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging as log - +import twlogging as log # Twisted imports from twisted.internet.defer import DeferredSemaphore from twisted.words.xish.domish import Element From 567605372fc659a6e2c76fd188bb5b60a1f055b0 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 7 Dec 2015 17:30:34 +0100 Subject: [PATCH 13/57] send roster request --- ecagent/core.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/ecagent/core.py b/ecagent/core.py index 352ed5f..65adc78 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -147,8 +147,11 @@ def _authd(self, xml_stream): #Keepalive: Send a newline every 60 seconds #to avoid server disconnect - self._keep_alive_lc = LoopingCall(self._xs.send, '\n') - self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) + self._keep_alive_lc = LoopingCall(self.loopcall) + d = self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) + d.addCallback(self.periodic_task_good) + d.addErrback(self.periodic_task_crashed) + self._xs.addObserver(STREAM_END_EVENT, lambda _: self._keep_alive_lc.stop()) @@ -161,18 +164,24 @@ def _authd(self, xml_stream): def _newid(self): return str(int(random() * (10 ** 31))) - def send(self, elem): - mem_clean('core.send [start]') + def loopcall(self): + log.info("sending roster request") + request = Element((None, 'iq'), attribs={'type': 'get', 'id': self._newid() }) + query = request.addElement('query') + query.attributes['xmlns'] = 'jabber:iq:roster' + self._xs.send(request) + + def periodic_task_good(self, reason): + log.info("going good") + def periodic_task_crashed(self, reason): + log.info("periodic_task broken") + + def send(self, elem): if not elem.getAttribute('id'): log.debug('No message ID in message, creating one') elem['id'] = self._newid() self._xs.send(elem.toXml()) - #Reset keepalive looping call timer - if self._keep_alive_lc.running: - self._keep_alive_lc.stop() - self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) - - mem_clean('core.send [stop]') + From ed40d650080666c1324c46e8b9e73d1cf707c612 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Fri, 11 Dec 2015 18:52:57 +0100 Subject: [PATCH 14/57] keepalived refactor --- ecagent/core.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/ecagent/core.py b/ecagent/core.py index 65adc78..1d914f4 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -84,8 +84,6 @@ def __init__(self, user, password, host, observers, self.failed_count = 0 self._xs = None - self._keep_alive_lc = None - self._user = user self._password = password self._host = host @@ -145,16 +143,6 @@ def _authd(self, xml_stream): """ log.info("XMPPClient authenticated") - #Keepalive: Send a newline every 60 seconds - #to avoid server disconnect - self._keep_alive_lc = LoopingCall(self.loopcall) - d = self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) - d.addCallback(self.periodic_task_good) - d.addErrback(self.periodic_task_crashed) - - self._xs.addObserver(STREAM_END_EVENT, - lambda _: self._keep_alive_lc.stop()) - for message, callable in self._observers: self._xs.addObserver(message, callable) @@ -164,24 +152,9 @@ def _authd(self, xml_stream): def _newid(self): return str(int(random() * (10 ** 31))) - def loopcall(self): - log.info("sending roster request") - request = Element((None, 'iq'), attribs={'type': 'get', 'id': self._newid() }) - query = request.addElement('query') - query.attributes['xmlns'] = 'jabber:iq:roster' - self._xs.send(request) - - def periodic_task_good(self, reason): - log.info("going good") - - def periodic_task_crashed(self, reason): - log.info("periodic_task broken") - def send(self, elem): if not elem.getAttribute('id'): log.debug('No message ID in message, creating one') elem['id'] = self._newid() self._xs.send(elem.toXml()) - - From 7765443ed0b29cd88a8a537cf90b27103e753ee0 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Fri, 11 Dec 2015 18:55:24 +0100 Subject: [PATCH 15/57] keepalived refactor --- ecagent/agent.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 2e355ed..ec3ff1c 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -38,6 +38,8 @@ _CHECK_RAM_MAX_RSS_MB = 125 _CHECK_RAM_INTERVAL = 60 +KEEPALIVED_TIMEOUT = 120 + class SMAgent: def __init__(self, config): reactor.callWhenRunning(self._check_config) @@ -81,6 +83,8 @@ def __init__(self, config): self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) + + self.keepalive = LoopingCall(self._stop) log.debug("Loading XMPP...") Client.__init__( @@ -88,6 +92,10 @@ def __init__(self, config): config['XMPP'], [('/iq', self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) + + def _stop(self): + log.info("No data received!!! michael naig") + reactor.stop() def __onIq(self, msg): """ @@ -100,6 +108,13 @@ def __onIq(self, msg): log.debug("Message type: %s" % msg['type']) if msg['type'] == 'set': + if self.keepalive.running: + log.info("Stop keepalived") + self.keepalive.stop() + + log.info("Starting keepalived") + self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) + message = IqMessage(msg) recv_command = message.command.replace('.', '_') @@ -210,4 +225,4 @@ def _check_memory(self, num_running_commands): if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) - reactor.stop() \ No newline at end of file + reactor.stop() From 7c74fbac6f7b994fbee6f6fa6132acf37ca9d25a Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Fri, 11 Dec 2015 18:55:57 +0100 Subject: [PATCH 16/57] keepalived refactor --- ecagent/core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ecagent/core.py b/ecagent/core.py index 1d914f4..89b4ad0 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -KEEPALIVED_TIMEOUT = 60 MAX_FAILED_LOGINS = 5 XMPP_PORT = 5222 @@ -24,7 +23,6 @@ from twisted.words.protocols.jabber import client, jid, xmlstream from twisted.words.xish.domish import Element from twisted.internet import reactor -from twisted.internet.task import LoopingCall from twisted.words.protocols.jabber.xmlstream import STREAM_END_EVENT from twisted.words.protocols.jabber.client import IQ From 485846f6eb6a75410af7a7f429a8471ad7ed990c Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Fri, 11 Dec 2015 18:57:05 +0100 Subject: [PATCH 17/57] minor change --- ecagent/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index ec3ff1c..31d518c 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -109,10 +109,10 @@ def __onIq(self, msg): if msg['type'] == 'set': if self.keepalive.running: - log.info("Stop keepalived") + log.debug("Stop keepalived") self.keepalive.stop() - log.info("Starting keepalived") + log.debug("Starting keepalived") self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) message = IqMessage(msg) From 58f2603008e5869ff6d539e494db61cc7c886ec0 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 11 Dec 2015 19:04:11 +0100 Subject: [PATCH 18/57] refactor observers --- ecagent/agent.py | 2 +- ecagent/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 31d518c..8a0e1c8 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -90,7 +90,7 @@ def __init__(self, config): Client.__init__( self, config['XMPP'], - [('/iq', self.__onIq), ], + [("/iq[@type='set']", self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) def _stop(self): diff --git a/ecagent/client.py b/ecagent/client.py index 92199c7..0b2504c 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -36,7 +36,7 @@ def __init__(self, config, observers, resource='XMPPClient'): """ my_observers = [ ('/presence', self._onPresence), - ('/iq', self._onPossibleErrorIq), + ("/iq[@type='error']", self._onPossibleErrorIq), ] my_observers.extend(observers) From 3e91cb750b5a1cbab8dc67bf7634290e9f25ff1f Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Sun, 13 Dec 2015 09:03:39 +0100 Subject: [PATCH 19/57] Clean plugins --- ecagent/agent.py | 71 +++-- ecagent/client.py | 6 +- plugins/__ecmhaproxy.py | 229 ---------------- plugins/__haproxy.py | 490 ----------------------------------- plugins/plugin_configfile.py | 70 ----- plugins/plugin_haproxy.py | 114 -------- plugins/plugin_puppet.py | 232 ----------------- plugins/plugin_saltstack.py | 178 ------------- plugins/plugin_source.py | 488 ---------------------------------- 9 files changed, 39 insertions(+), 1839 deletions(-) delete mode 100644 plugins/__ecmhaproxy.py delete mode 100644 plugins/__haproxy.py delete mode 100644 plugins/plugin_configfile.py delete mode 100644 plugins/plugin_haproxy.py delete mode 100644 plugins/plugin_puppet.py delete mode 100644 plugins/plugin_saltstack.py delete mode 100644 plugins/plugin_source.py diff --git a/ecagent/agent.py b/ecagent/agent.py index 8a0e1c8..9cc35f6 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -84,7 +84,7 @@ def __init__(self, config): self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) - self.keepalive = LoopingCall(self._stop) + self.keepalive = LoopingCall(self._reconnect) log.debug("Loading XMPP...") Client.__init__( @@ -93,9 +93,12 @@ def __init__(self, config): [("/iq[@type='set']", self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) - def _stop(self): - log.info("No data received!!! michael naig") - reactor.stop() + def _reconnect(self): + """ + Disconnect the current reactor to try to connect again + """ + log.info("No data received in %ss: Trying to reconnect" % KEEPALIVED_TIMEOUT) + reactor.disconnectAll() def __onIq(self, msg): """ @@ -107,48 +110,44 @@ def __onIq(self, msg): log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) - if msg['type'] == 'set': - if self.keepalive.running: - log.debug("Stop keepalived") - self.keepalive.stop() - - log.debug("Starting keepalived") - self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) - - message = IqMessage(msg) - recv_command = message.command.replace('.', '_') + if self.keepalive.running: + log.debug("Stop keepalived") + self.keepalive.stop() + + log.debug("Starting keepalived") + self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) - if recv_command in self.running_commands: - if time() > self.running_commands[recv_command]: - del self.running_commands[recv_command] - self.num_running_commands -= 1 - log.debug("Deleted %s from running_commands dict as should have been completed" % (recv_command)) + message = IqMessage(msg) + recv_command = message.command.replace('.', '_') - if recv_command not in self.running_commands: - log.debug('recieved new command: %s with message: %s' % (message.command, message)) + if recv_command in self.running_commands: + if time() > self.running_commands[recv_command]: + del self.running_commands[recv_command] + self.num_running_commands -= 1 + log.debug("Deleted %s from running_commands dict as should have been completed" % (recv_command)) - if hasattr(message, 'command') and hasattr(message, 'from_'): - log.debug('online contacts: %s' % self._online_contacts) + if recv_command not in self.running_commands: + log.debug('recieved new command: %s with message: %s' % (message.command, message)) - if message.from_ not in self._online_contacts: - log.warn('IQ sender not in roster (%s), dropping message' % message.from_) - else: - self.running_commands[recv_command] = time() + int(message.command_args['timeout']) - self.num_running_commands += 1 - log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) - self._processCommand(message) + if hasattr(message, 'command') and hasattr(message, 'from_'): + log.debug('online contacts: %s' % self._online_contacts) + if message.from_ not in self._online_contacts: + log.warn('IQ sender not in roster (%s), dropping message' % message.from_) else: - log.warn('Unknown ecm_message received: Full XML:\n%s' % (msg.toXml())) + self.running_commands[recv_command] = time() + int(message.command_args['timeout']) + self.num_running_commands += 1 + log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) + self._processCommand(message) - del message else: - log.debug("already running given command %s" %recv_command) - result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) - self._send(result, message) + log.warn('Unknown ecm_message received: Full XML:\n%s' % (msg.toXml())) + del message else: - log.warn('Unknown IQ type received: Full XML:\n%s' % (msg.toXml())) + log.debug("already running given command %s" %recv_command) + result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) + self._send(result, message) del msg mem_clean('__onIq [end]') diff --git a/ecagent/client.py b/ecagent/client.py index 0b2504c..9edf575 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -14,7 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import twlogging as log +import logging as log + # Twisted imports from twisted.internet.defer import DeferredSemaphore from twisted.words.xish.domish import Element @@ -36,7 +37,7 @@ def __init__(self, config, observers, resource='XMPPClient'): """ my_observers = [ ('/presence', self._onPresence), - ("/iq[@type='error']", self._onPossibleErrorIq), + ('/iq', self._onPossibleErrorIq), ] my_observers.extend(observers) @@ -83,6 +84,7 @@ def _onPresence(self, elem): log.debug("%s is now available" % presence.sender) #Store full jid. self._online_contacts.add(presence.sender) + else: log.debug("%s is not available anymore" % presence.sender) if presence.jid in self._online_contacts: diff --git a/plugins/__ecmhaproxy.py b/plugins/__ecmhaproxy.py deleted file mode 100644 index 9c5b80d..0000000 --- a/plugins/__ecmhaproxy.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Copyright (C) 2013 - Aybuke Ozdemir - -this file is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -this file is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see - -""" -# -*- coding: utf-8 -*- - -import warnings -from os import path -from tempfile import mktemp -import socket -import csv - -import simplejson as json -from __haproxy import HAProxyConfig, Option - -NAME = 'haproxy' -RECV_SIZE = 1024 -DEFAULT_SOCKET = '/tmp/haproxy' -HAPROXY_BIN = '/usr/sbin/haproxy' - -TEMPLATE_FILE = '/etc/haproxy/haproxy.tpl' -TEMPLATE_CONFIG = """ -global - maxconn 4096 - user haproxy - group haproxy - daemon - -defaults - retries 3 - option redispatch - option http-server-close - option forwardfor if-none - timeout connect 5000 - timeout client 50000 - timeout server 50000 - timeout check 5000 - balance leastconn -""" - - -class ECMHAConfig: - def __init__(self, ecm_hash, template=TEMPLATE_FILE, as_json=False): - if as_json: - ecm_hash = json.loads(ecm_hash) - - if not self._check(ecm_hash): - raise Exception('Invalid hash received') - - if not path.isfile(template): - warnings.warn("Template file not found, using default config") - template = mktemp() - f = open(template, 'w') - f.write(TEMPLATE_CONFIG) - f.close() - - self.template = template - self.ecm_hash = ecm_hash - self.config = HAProxyConfig(template) - self._loads() - - def show(self, as_json=False): - config_to_show = self.config.createHash() - if as_json: - config_to_show = json.dumps(config_to_show) - print config_to_show - - def read(self, read_file, as_json=True): - f = open(read_file, 'r') - first_line = f.readline() - f.close() - - if first_line.startswith('#'): - json_config = first_line.split('#')[1] - - if not as_json: - json_config = json.loads(json_config) - return json_config - - return False - - def valid(self): - config = self.config.getConfig() - return is_valid(config) - - def write(self, write_file): - if write_file == self.template: - raise Exception("Can't use template as final file") - - config = self.config.getConfig() - f = open(write_file, 'w') - f.write("#" + json.dumps(self.ecm_hash) + "\n") - f.write(config) - f.close() - - def _check(self, ecm_hash): - if ecm_hash.get('health', None) is None: - return False - if ecm_hash.get('listeners', None) is None: - return False - if ecm_hash.get('backends', None) is None: - return False - - return True - - # private functions - - def _loads(self): - health_timeout = self.ecm_hash['health']['timeout'] - health_threshold = self.ecm_hash['health']['threshold'] - health_interval = self.ecm_hash['health']['threshold'] - health_check = self.ecm_hash['health']['check'] - - listeners = False - - # Get check data - (hproto, hport, hpath) = health_check.split('::') - - for listener in self.ecm_hash['listeners']: - listeners = True - (fproto, fport, bproto, bport) = listener.split('::') - self.config.addListen(name="front-%s" % fport, port=fport, mode=self._protocol_dic(fproto)) - listen = self.config.getListen("front-%s" % fport) - - # Add check timeout configuration - listen.addOption(Option('timeout', ('check %i' % (health_timeout*1000),))) - - # Add cookie only if there are backend (config crashes otherwise) - if len(self.ecm_hash['backends']) and self._protocol_dic(fproto) == 'http': - listen.addOption(Option('cookie', ('ECM-HA-ID insert indirect nocache maxidle 30m maxlife 8h',))) - - if hpath: - check = 'httpchk HEAD %s' % hpath - listen.addOption(Option('option', (check,))) - - for backend in self.ecm_hash['backends']: - uuid = backend.keys()[0] - server = '%s %s:%s check inter %i rise %i fall %i' % (uuid, backend[uuid], fport, health_interval*1000, - health_threshold, health_threshold) - if self._protocol_dic(fproto) == 'http': - server += ' cookie %s' % uuid - listen.addOption(Option('server', (server,))) - - if not listeners: - # Add default listener - self.config.addListen(name="empty", port=80, mode='http') - - @staticmethod - def _protocol_dic(proto): - retval = 'tcp' - if proto.lower() == 'http': - retval = 'http' - - return retval - - -class ECMHASocket(object): - def __init__(self, socket_file=DEFAULT_SOCKET): - self.socket_file = socket_file - - def connect(self): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.connect(self.socket_file) - return s - - def communicate(self, command): - """ Send a single command to the socket and return a single response (raw string) """ - s = self.connect() - if not command.endswith('\n'): command += '\n' - s.send(command) - result = '' - buf = s.recv(RECV_SIZE) - while buf: - result += buf - buf = s.recv(RECV_SIZE) - s.close() - return result - - def get_server_info(self): - result = {} - output = self.communicate('show info') - for line in output.splitlines(): - try: - key, val = line.split(':') - except ValueError, e: - continue - result[key.strip()] = val.strip() - return result - - def get_server_stats(self): - output = self.communicate('show stat -1 4 -1') - #sanitize and make a list of lines - output = output.lstrip('# ').strip() - output = [l.strip(',') for l in output.splitlines()] - csvreader = csv.DictReader(output) - result = [d.copy() for d in csvreader] - return result - - -def is_valid(config): - import subprocess - - tmp_file = mktemp() - f = open(tmp_file, 'w') - f.write(config) - f.close() - - p = subprocess.Popen([HAPROXY_BIN, "-c", "-f", tmp_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if p.returncode == 0: - return True - - warnings.warn("Invalid configuration file: %s" % err) - return False - diff --git a/plugins/__haproxy.py b/plugins/__haproxy.py deleted file mode 100644 index 6c2b014..0000000 --- a/plugins/__haproxy.py +++ /dev/null @@ -1,490 +0,0 @@ -""" -Copyright (C) 2013 - Aybuke Ozdemir - -This file is part of python-haproxy-tools - -python-haproxy-tools is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -python-haproxy-tools is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see - -""" -# -*- coding: utf-8 -*- - -SECTIONS = ['global', 'defaults', 'listen', 'frontend', 'backend'] - - -class HAProxyConfig(): - def __init__(self, config_path): - self.config_path = config_path - self.config = self.__readConfig() - - # Set globals section. - gs = self.__getSection('global') - if gs: - self.globalh = Global(gs) - else: - self.globalh = None - - # Set defaults section. - ds = self.__getSection('defaults') - if ds: - self.defaults = Defaults(ds) - else: - self.defaults = None - - #Set Listen. - self.listens = [] - for name in self.__getSectionNames('listen'): - l = Listen(self.__getSectionWithName('listen', name)) - self.listens.append(l) - - # Set Frontends. - self.frontends = [] - for name in self.__getSectionNames('frontend'): - f = Frontend(self.__getSectionWithName('frontend', name)) - self.frontends.append(f) - - #Set Backend. - self.backends = [] - for name in self.__getSectionNames('backend'): - b = Backend(self.__getSectionWithName('backend', name)) - self.backends.append(b) - - def getConfig(self): - out = "# This file created by python-haproxy-tools\n" - if self.globalh: - out += self.globalh.getConfig() - if self.defaults: - out += self.defaults.getConfig() - for l in self.listens: - out += l.getConfig() - for f in self.frontends: - out += f.getConfig() - for b in self.backends: - out += b.getConfig() - return out - - def createHash(self): - hash_config = {'global': [], 'defaults': [], 'backend': {}, 'frontend': {}, 'listen': {}} - - b = self.getGlobal() - for a in b.options: - hash_config['global'].append({a.name: ' '.join(a.params)}) - - b = self.getDefaults() - for a in b.options: - hash_config['defaults'].append({a.name: ' '.join(a.params)}) - - for b in self.getBackends(): - hash_config['backend'][b.name] = [] - for a in b.options: - hash_config['backend'][b.name].append({a.name: ' '.join(a.params)}) - - for b in self.getFrontends(): - hash_config['frontend'][b.name] = [] - for a in b.options: - hash_config['frontend'][b.name].append({a.name: ' '.join(a.params)}) - - for b in self.getListens(): - hash_config['listen'][b.name] = [] - for a in b.options: - hash_config['listen'][b.name].append({a.name: ' '.join(a.params)}) - - return hash_config - - def writeConfig(self,ha_file): - hash_config = self.createHash() - f = open(ha_file, 'w') - for key in SECTIONS: - if not hash_config.get(key): - continue - - if key in ['global','defaults']: - f.write("%s\n" %key) - - for param in hash_config[key]: - if isinstance(param, dict): - f.write(" %s %s\n" % (param.keys()[0], param[param.keys()[0]])) - - else: - f.write("%s %s\n" % (key,param)) - for value in hash_config[key][param]: - f.write(" %s %s\n" % (value.keys()[0], value[value.keys()[0]])) - f.write("\n") - - f.write("\n") - f.close() - - def getGlobal(self): - return self.globalh - - def getDefaults(self): - return self.defaults - - def getFrontend(self, name): - name = name.strip() - for fe in self.frontends: - if fe.description.name == name: - return fe - - def getBackend(self, name): - name = name.strip() - for be in self.backends: - if be.description.name == name: - return be - - def getListen(self, name): - name = name.strip() - for l in self.listens: - if l.description.name == name: - return l - - def getFnames(self): - fa = [] - for f in self.frontends: - fa.append(f.name) - return fa - - def getLnames(self): - la = [] - for l in self.listens: - la.append(l.name) - return la - - def getBnames(self): - ba = [] - for b in self.backends: - ba.append(b.name) - return ba - - def addListen(self, name, ip=None, port=None, mode=None): - ca = [] - ca.append('listen %s' %name) - coption = "" - if ip: - cip = ip - else: - cip = "*" - if port: - cport = port - else: - cport = "80" - coption = "bind %s:%s" %(cip,cport) - ca.append(coption) - if mode: - ca.append('mode %s' %mode) - - listen = Listen(ca) - self.listens.append(listen) - - def addFrontend(self, name, ip=None, port=None, mode=None): - ca = [] - ca.append('frontend %s' %name) - if ip: - cip = ip - else: - cip = ip - if port: - cport = port - else: - cport = "80" - coption = "bind %s:%s" %(cip, cport) - ca.append(coption) - if mode: - ca.append('mode:%s' %mode) - - frontend = Frontend(ca) - self.frontends.append(frontend) - - def addBackend(self, name, balance): - ca = [] - ca.append('backend %s' %name) - coption = "balance %s" %balance - ca.append(coption) - backend = Backend(ca) - self.backends.append(backend) - - def addGlobal(self, option, value): - goption = Option(option, value) - self.globalh.addOption(goption) - - def addDefaults(self, option, value): - doption = Option(option, value) - self.defaults.addOption(doption) - - def delBackend(self, name): - for be in self.backends: - if be.description.name == name: - self.backends.remove(be) - - def delFrontend(self, name): - for fe in self.frontends: - if fe.description.name == name: - self.frontends.remove(fe) - - def delListen(self, name): - for l in self.listens: - if l.description.name == name: - self.listens.remove(l) - - def getFrontends(self): - return self.frontends - - def getBackends(self): - return self.backends - - def getListens(self): - return self.listens - - def __getSectionNames(self, title): - l_names = [] - for row in self.config: - row = row.strip() - if row.startswith(title): - name = row.split()[1] - l_names.append(name) - return l_names - - def __getSectionWithName(self, title, name): - config_array = [] - title = title.strip() - start_flag = False - - for line in self.config: - line = line.strip() - - if line == '': - continue - - line_array = line.split() - ltitle = line_array[0].strip() - - if len(line_array) > 1: - lname = line_array[1].strip() - - if ltitle == title.strip(): - if name == lname: - config_array.append(line) - start_flag = True - continue - - if ltitle in SECTIONS: - start_flag = False - - if start_flag: - config_array.append(line) - return config_array - - def __getSection(self, title): - config_array = [] - title = title.strip() - start_flag = False - - for line in self.config: - line = line.strip() - - if line == '': - continue - - ltitle = line.split()[0].strip() - if ltitle == title: - config_array.append(line) - start_flag = True - continue - - if ltitle in SECTIONS: - start_flag = False - - if start_flag: - config_array.append(line) - if config_array: - return config_array - else: - return None - - def __readConfig(self): - config_file = open(self.config_path) - config = config_file.readlines() - config_file.close() - return config - - -class Option(): - def __init__(self, param_name, params): - self.name = param_name - self.params = params - - def getRow(self): - return self.__repr__() - - def getParamName(self): - return self.name - - def getParams(self): - return self.params - - def getConfig(self): - return "%s %s" %(self.name, " ".join(self.params)) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.params))) - - -class Section(): - def __init__(self, config_array): - self.config_array = config_array - self.options = [] - self.description = None - - is_first_row = True - for row in self.config_array: - if is_first_row: - is_first_row = False - items = row.split() - title = items[0] - - if len(items) > 1: - name = items[1] - else: - name = None - if len(items) > 2: - params = items[2:] - else: - params = None - - self.description = Description(title, name) - - if params: - for p in params: - o = Option('bind', tuple([p])) - self.options.append(o) - - continue - - param_name = self.getParamName(row).strip() - params = self.getParams(row) - option = Option(param_name, params) - self.options.append(option) - - def getParamName(self, row): - return row.split()[0] - - def getParams(self, row): - return tuple(row.split()[1:]) - - def getConfig(self): - opts = self.options - des = self.description - config_output = "" - config_output += des.getConfig() + '\n' - for opt in opts: - config_output += ' ' + opt.getConfig() + '\n' - config_output += '\n' - return config_output - - def addOption(self, option): - self.options.append(option) - return True - - def delOption(self, option): - for opt in self.options: - if opt == option.name: - self.options.remove(opt) - return True - - def setOption(self, option): - for opt in self.options: - if opt.name == option.name: - opt.params = option.params - return self.options - - -class Description(): - def __init__(self, title, name=None): - self.title = title - self.name = name - - def getConfig(self): - out = [] - out.append(self.title) - if self.name: - out.append(self.name) - return " ".join(out) - - -class Global(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.options))) - - -class Defaults(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.options))) - - -class Listen(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - -class Frontend(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - -class Backend(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - diff --git a/plugins/plugin_configfile.py b/plugins/plugin_configfile.py deleted file mode 100644 index 8d5bb78..0000000 --- a/plugins/plugin_configfile.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding:utf-8 -*- - -RUN_AS_ROOT = False - -import os - -from base64 import b64decode -from shutil import move - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMConfigfile(ECMPlugin): - def cmd_configfile_run(self, *argv, **kwargs): - """ - Deploy a file - Syntax: configfile.run[configfile,file,chown_user,chown_group,chmod,rotate,command,runas] - """ - code_base64 = kwargs.get('configfile', None) - filename = kwargs.get('path', None) - chown_user = kwargs.get('chown_user', None) - chown_group = kwargs.get('chown_group', None) - chmod = kwargs.get('chmod', None) - rotate = kwargs.get('rotate', False) - - command = kwargs.get('command', None) - runas = kwargs.get('command_runas', None) - - if not code_base64 or not filename: - raise ecm.InvalidParameters(self.cmd_configfile_run.__doc__) - - ret = {'out': 0, 'stdout': '', 'stderr': ''} - try: - if rotate and os.path.isfile(filename): - new_file = filename + '_rotated_' + ecm.utime() - move(filename, new_file) - ret['stdout'] = ecm.output("Old configfile moved to '%s'" % new_file) - - # Write down file - ecm.file_write(filename, b64decode(code_base64)) - ret['stdout'] += ecm.output("Configfile created successfully at '%s'" % filename) - - except Exception as e: - raise Exception("Unable to write configfile: %s" % e) - - try: - # Chown to specified user/group - if chown_user and chown_group and os.path.isfile(filename): - ecm.chown(filename, chown_user, chown_group) - ret['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - # Chown to specified user/group - if chmod and os.path.isfile(filename): - ecm.chmod(filename, chmod) - ret['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - except Exception as e: - raise Exception("Unable to change permissions for configfile: %s" % e) - - if command: - working_dir = os.path.dirname(filename) - out, stdout, stderr = ecm.run_command(command, runas=runas, workdir=working_dir) - ret = ecm.format_output(out, stdout, stderr) - - return ret - - -ECMConfigfile().run() diff --git a/plugins/plugin_haproxy.py b/plugins/plugin_haproxy.py deleted file mode 100644 index 6ce5695..0000000 --- a/plugins/plugin_haproxy.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -RUN_AS_ROOT = True - -import simplejson as json -from base64 import b64decode -from __ecmhaproxy import ECMHAConfig, ECMHASocket - -HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg' -HAPROXY_INIT = '/etc/init.d/haproxy' - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMHaproxy(ECMPlugin): - def cmd_haproxy_config_get(self, *argv, **kwargs): - """ - haproxy.config_get[] - """ - f = open(HAPROXY_CONFIG, 'r') - first_line = f.readline() - f.close() - - if first_line.startswith('#'): - config = first_line.split('#')[1].rstrip('\n') - status = self._status() - return { - 'config': json.loads(config), - 'status': status - } - - raise Exception("Unable to get config") - - def cmd_haproxy_config_set(self, *argv, **kwargs): - """ - haproxy.config_set[config=base64_balancer_config] - """ - config = kwargs.get('config', None) - if not config: - raise ecm.InvalidParameters(self.cmd_haproxy_config_set.__doc__) - - try: - config = b64decode(config) - except Exception: - raise ecm.InvalidParameters('Invalid base64 configuration') - - ha_config = ECMHAConfig(config, as_json=True) - if ha_config.valid(): - ha_config.write(HAPROXY_CONFIG) - self._restart() - - return self.cmd_haproxy_config_get() - - else: - raise Exception('Invalid configuration') - - def cmd_haproxy_stats(self, *argv, **kwargs): - """ - haproxy.stats[] - """ - try: - ha = ECMHASocket() - return { - 'stats': ha.get_server_stats(), - 'info': ha.get_server_info() - } - - except: - raise Exception("Unable to get config") - - - @staticmethod - def _status(): - try: - status_array = {} - ha = ECMHASocket() - for line in ha.get_server_stats(): - status_array.setdefault(line['pxname'], {})[line['svname']] = line['status'] - return status_array - - except: - raise Exception("Unable to get config") - - @staticmethod - def _restart(): - import subprocess - p = subprocess.Popen([HAPROXY_INIT, 'reload'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if p.returncode == 0: - return True - - raise Exception("Error restarting service: %s" % err) - - -ECMHaproxy().run() - diff --git a/plugins/plugin_puppet.py b/plugins/plugin_puppet.py deleted file mode 100644 index d1624d2..0000000 --- a/plugins/plugin_puppet.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -from base64 import b64decode -from tempfile import mkdtemp -from shutil import rmtree -import tarfile - -# Local -from __plugin import ECMPlugin -import __helper as ecm - -MODULES_PATH = '/etc/puppet/modules' -MODULES_PATH_WINDOWS = 'c:\ECM\puppet\modules' - -BOOTSTRAP = 'http://bootstrap.ecmanaged.com/puppet/linux/' -BOOTSTRAP_ALT = 'http://bootstrap-devel.ecmanaged.com/puppet/linux/' - -BOOTSTRAP_WINDOWS = 'http://bootstrap.ecmanaged.com/puppet/windows/' -BOOTSTRAP_WINDOWS_ALT = 'http://bootstrap-devel.ecmanaged.com/puppet/windows/' - -class ECMPuppet(ECMPlugin): - def cmd_puppet_available(self, *argv, **kwargs): - """ Checks if puppet commands are available - """ - return bool(self._is_available()) - - def cmd_puppet_install(self, *argv, **kwargs): - """ Installs saltstack using bootstrap scripts - """ - if self._is_available(): - return True - - bootstrap = BOOTSTRAP - bootstrap_file = 'bootstrap.sh' - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS - bootstrap_file = 'bootstrap.ps1' - - if not self._install(bootstrap, bootstrap_file): - # Try alternative bootstrap - bootstrap = BOOTSTRAP_ALT - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS_ALT - - if not self._install(bootstrap,bootstrap_file): - raise Exception("Unable to install puppet") - - return True - - def cmd_puppet_apply(self, *argv, **kwargs): - """ - Syntax: puppet.appy[recipe_code,evars,facts] - """ - recipe_base64 = kwargs.get('recipe_code', None) - metadata_platform = kwargs.get('metadata_platform', None) - metadata_server = kwargs.get('metadata_server', None) - - if not recipe_base64: - raise ecm.InvalidParameters(self.cmd_puppet_apply.__doc__) - - # Set module path - module_path = kwargs.get('module_path', None) - if module_path is None: - module_path = MODULES_PATH - if ecm.is_win(): - module_path = MODULES_PATH_WINDOWS - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata_server) - - # Update metadata - ecm.write_metadata_platform(metadata_b64=metadata_platform) - ecm.write_metadata(metadata_b64=metadata_server) - - try: - catalog = b64decode(recipe_base64) - except: - raise ecm.InvalidParameters("Unable to decode recipe") - - try: - command = ['puppet', - 'apply', - '--modulepath', - module_path, - '--detailed-exitcodes', - '--debug'] - - out, stdout, stderr = ecm.run_command(command, stdin=catalog, envars=envars) - ret = ecm.format_output(out, stdout, stderr) - - # exit code of '2' means there were changes - if ret['out'] == 2: ret['out'] = 0 - if "\nError: " in ret['stderr']: ret['out'] = 4 - - return ret - - except Exception as e: - raise Exception("Error running puppet apply: %s" % e) - - def cmd_puppet_apply_file(self, *argv, **kwargs): - """ - Syntax: puppet.apply_file[recipe_url,envars,facts] - """ - recipe_url = kwargs.get('recipe_url', None) - metadata = kwargs.get('metadata', None) - - if not recipe_url: - raise ecm.InvalidParameters(self.cmd_puppet_apply.__doc__) - - recipe_file = None - recipe_path = None - module_path = MODULES_PATH - if ecm.is_win(): - module_path = MODULES_PATH_WINDOWS - module_path = kwargs.get('module_path', module_path) - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - try: - # Download recipe url - recipe_path = mkdtemp() - tmp_file = recipe_path + '/recipe.tar.gz' - - if ecm.download_file(recipe_url, tmp_file): - if tarfile.is_tarfile(tmp_file): - tar = tarfile.open(tmp_file) - tar.extractall(path=recipe_path) - - for file_name in tar.getnames(): - if file_name.endswith('.catalog.pson'): - recipe_file = file_name - tar.close() - - # Apply puppet catalog - return self._run_catalog(recipe_file, recipe_path, module_path=module_path, envars=envars) - else: - raise Exception("Invalid recipe tgz file") - else: - raise Exception("Unable to download file") - - except: - raise Exception("Unable to get recipe") - - finally: - rmtree(recipe_path, ignore_errors=True) - - def _is_available(self): - """ which puppet - """ - if ecm.is_win(): - return ecm.which('puppet.exe') - - return ecm.which('puppet') - - def _run_catalog(self, recipe_file, recipe_path, module_path, envars=None): - """ Execute catalog file - """ - retval = self._run_puppet(recipe_file, recipe_path, module_path, 'catalog', envars) - - # Try old way - if 'invalid option' in retval.get('stdout', ''): - retval = self._run_puppet(recipe_file, recipe_path, module_path, 'apply', envars) - - return retval - - def _run_puppet(self, recipe_file, recipe_path, module_path, catalog_cmd='catalog', envars=None): - """ Real puppet execution - """ - puppet_cmd = self._is_available() - if not puppet_cmd: - raise Exception("Puppet is not available") - - command = [puppet_cmd, 'apply', '--detailed-exitcodes', '--modulepath', module_path, '--debug', - '--' + catalog_cmd, recipe_file] - - out, std_out, std_err = ecm.run_command(command, workdir=recipe_path, envars=envars) - ret = ecm.format_output(out, std_out, std_err) - - # --detailed-exitcodes - # Provide transaction information via exit codes. If this is enabled, - # an exit code of '2' means there were changes, - # an exit code of '4' means there were failures during the transaction, - # and an exit code of '6' means there were both changes and failures. - - # bug in exitcodes in some version even with errors return 0 - # http://projects.puppetlabs.com/issues/6322 - - if ret['out'] == 2: - ret['out'] = 0 - if "\nError: " in ret['stderr']: - ret['out'] = 4 - - return ret - - def _install(self, bootstrap_url, bootstrap_file = 'bootstrap.sh'): - """ Installs puppet using bootstrap url - """ - tmp_dir = mkdtemp() - bootstrap_file = tmp_dir + '/' + bootstrap_file - ecm.download_file(bootstrap_url, bootstrap_file) - - # wget -O - http://bootstrap.ecmanaged.com/puppet/linux/ | sudo sh - - if ecm.file_read(bootstrap_file): - envars = {'DEBIAN_FRONTEND': 'noninteractive'} - ecm.run_file(bootstrap_file, envars=envars) - - rmtree(tmp_dir) - return bool(self._is_available()) - - -ECMPuppet().run() diff --git a/plugins/plugin_saltstack.py b/plugins/plugin_saltstack.py deleted file mode 100644 index 243b5bd..0000000 --- a/plugins/plugin_saltstack.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -from tempfile import mkdtemp -from shutil import rmtree -from base64 import b64decode - -# Local -from __plugin import ECMPlugin -import __helper as ecm - -DEFAULT_SALT_PATH = '/srv/salt' -DEFAULT_SALT_PATH_WINDOWS = 'C:\ECM\SALTSTACK\salt' - -DEFAULT_PILLAR_PATH = '/srv/pillar' -DEFAULT_PILLAR_PATH_WINDOWS = 'C:\ECM\SALTSTACK\pillar' - -BOOTSTRAP = 'http://bootstrap.ecmanaged.com/saltstack/linux' -BOOTSTRAP_ALT = 'http://bootstrap.saltstack.org' - -BOOTSTRAP_WINDOWS = 'http://bootstrap.ecmanaged.com/saltstack/windows' -BOOTSTRAP_WINDOWS_ALT = 'http://bootstrap.saltstack.org' - -TOP_CONTENT = """base: - '*': - - ecmanaged -""" - -class ECMSaltstack(ECMPlugin): - def cmd_saltstack_available(self, *argv, **kwargs): - """ Checks if saltstack commands are available - """ - return bool(self._is_available()) - - def cmd_saltstack_install(self, *argv, **kwargs): - """ Installs saltstack using bootstrap scripts - """ - if self._is_available(): return True - - bootstrap = BOOTSTRAP - bootstrap_file = 'bootstrap.sh' - - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS - bootstrap_file = 'bootstrap.ps1' - - if not self._install(bootstrap,bootstrap_file): - # Try alternative bootstrap - bootstrap = BOOTSTRAP_ALT - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS_ALT - - if not self._install(bootstrap,bootstrap_file): - raise Exception("Unable to install saltstack") - - return True - - def cmd_saltstack_apply(self, *argv, **kwargs): - """ - Apply a saltstack manifest - Syntax: saltstack.apply[recipe_code,pillar_code,envars,facts] - """ - recipe_b64 = kwargs.get('recipe_code', None) - pillar_b64 = kwargs.get('pillar_code', None) - metadata = kwargs.get('metadata', None) - - if not recipe_b64: - raise ecm.InvalidParameters(self.cmd_saltstack_apply.__doc__) - - saltstack_cmd = self._is_available() - if not saltstack_cmd: - raise Exception('Saltstack no available') - - # Get default paths - default_path = DEFAULT_SALT_PATH - if ecm.is_win(): - default_path = DEFAULT_SALT_PATH_WINDOWS - module_path = kwargs.get('module_path', default_path) - - default_pillar_path = DEFAULT_PILLAR_PATH - if ecm.is_win(): - default_pillar_path = DEFAULT_PILLAR_PATH_WINDOWS - pillar_path = kwargs.get('pillar_path', default_pillar_path) - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - try: - # Create top file - self._create_top_file(module_path) - - recipe_file = module_path + '/ecmanaged.sls' - ecm.file_write(recipe_file, b64decode(recipe_b64)) - - if pillar_b64: - self._create_top_file(pillar_path) - pillar_file = pillar_path + '/ecmanaged.sls' - ecm.file_write(pillar_file, b64decode(pillar_b64)) - - except: - raise Exception("Unable to write recipe") - - try: - # salt-call state.highstate - command = [saltstack_cmd, 'state.highstate', '--local', '--no-color', '-l debug'] - - out, stdout, stderr = ecm.run_command(command, envars=envars, workdir=module_path) - return ecm.format_output(out, stdout, stderr) - - except Exception as e: - raise Exception("Error running saltstack state.highstate: %s" % e) - - def _create_top_file(self, path): - top_file = path + '/top.sls' - ecm.file_write(top_file, TOP_CONTENT) - - def _is_available(self): - """ it's salt-call on path? - """ - if ecm.is_win(): - return ecm.which('salt-call.exe') - return ecm.which('salt-call') - - def _install(self, bootstrap_url, bootstrap_file = 'bootstrap.sh'): - """ Installs saltstack using bootstrap url - """ - - tmp_dir = mkdtemp() - bootstrap_file = tmp_dir + '/' + bootstrap_file - ecm.download_file(bootstrap_url, bootstrap_file) - - # wget -O - http://bootstrap.saltstack.org | sudo sh - - # Options: - #-h Display this message - #-v Display script version - #-n No colours. - #-D Show debug output. - #-c Temporary configuration directory - #-g Salt repository URL. (default: git://github.com/saltstack/salt.git) - #-k Temporary directory holding the minion keys which will pre-seed the master. - #-M Also install salt-master - #-S Also install salt-syndic - #-N Do not install salt-minion - #-X Do not start daemons after installation - #-C Only run the configuration function. This option automatically bypasses any installation. - #-P Allow pip based installations. On some distributions the required salt packages or its dependencies are not available as a package for that distribution. Using this flag allows the script to use pip as a last resort method. NOTE: This works for functions which actually implement pip based installations. - #-F Allow copied files to overwrite existing(config, init.d, etc) - #-U If set, fully upgrade the system prior to bootstrapping salt - #-K If set, keep the temporary files in the temporary directories specified with -c and -k. - - if ecm.file_read(bootstrap_file): - envars = {'DEBIAN_FRONTEND': 'noninteractive'} - ecm.run_file(bootstrap_file, args=['-n', '-P', '-X'], envars=envars) - - rmtree(tmp_dir) - return bool(self._is_available()) - - -ECMSaltstack().run() diff --git a/plugins/plugin_source.py b/plugins/plugin_source.py deleted file mode 100644 index 40adcac..0000000 --- a/plugins/plugin_source.py +++ /dev/null @@ -1,488 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = False - -import os - -from tempfile import mkdtemp, NamedTemporaryFile -from urlparse import urlparse -from shutil import move, rmtree -from base64 import b64decode - -try: - import tarfile - import zipfile - import gzip - import bz2 - -except ImportError: - pass - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMSource(ECMPlugin): - def cmd_source_run(self, *argv, **kwargs): - """ - Syntax: source.run[path,source,branch,envars,facts,username,password,private_key,chown_user,chown_group,rotate,type] - """ - path = kwargs.get('path', None) - url = kwargs.get('source', None) - branch = kwargs.get('branch', None) - user = kwargs.get('username', None) - passwd = kwargs.get('password', None) - private_key = kwargs.get('private_key', None) - chown_user = kwargs.get('chown_user', None) - chown_group = kwargs.get('chown_group', None) - rotate = kwargs.get('rotate', True) - extract = kwargs.get('extract', True) - - stype = kwargs.get('type', None) - metadata = kwargs.get('metadata', None) - - if not path or not url or not stype: - raise ecm.InvalidParameters(self.cmd_source_run.__doc__) - - if private_key: - try: - private_key = b64decode(private_key) - except: - raise ecm.InvalidParameters("Invalid private key format") - - if stype.upper() in ('URL', 'FILE'): - source = FILE(path, rotate, extract) - - elif stype.upper() == 'GIT': - source = GIT(path, rotate) - - elif stype.upper() == 'SVN': - source = SVN(path, rotate) - - else: - raise ecm.InvalidParameters("Unknown source") - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - retval = source.clone(url=url, - branch=branch, - envars=envars, - username=user, - password=passwd, - private_key=private_key) - - # Chown to specified user/group - if chown_user and chown_group and os.path.isdir(path): - ecm.chown(path, chown_user, chown_group, recursive=True) - retval['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - return self._return(retval) - - def _return(self, ret): - output = { - 'out': ret.get('out', 1), - 'stderr': ret.get('stderr', ''), - 'stdout': ret.get('stdout', '') - } - return output - - -class GIT: - def __init__(self, working_dir, rotate=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - # Get git path - self.git_cmd = self._is_available() - - if not self.git_cmd: - if not self._install(): - raise Exception('Unable to find or install git') - self.git_cmd = self._is_available() - - def clone(self, url, branch, envars, username, password, private_key): - """ runs git clone URL - """ - command = self._get_command(url,branch) - - # Create git command with user and password - if username and password: - parsed = urlparse(url) - if parsed.scheme in ('http', 'https'): - command = command.replace('://', '://' + username + ':' + password + '@') - - elif parsed.scheme == 'ssh': - command = command.replace('://', '://' + username + '@') - - elif private_key: - helper, indetity = self._certificate_helper(private_key) - envars['GIT_SSH'] = helper - - out, stdout, stderr = ecm.run_command(command=command, workdir=self.working_dir, envars=envars) - result_exec = ecm.format_output(out, stdout, stderr) - - if not result_exec['out']: - extra_msg = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - if self.old_dir: - extra_msg += ecm.output("Old source files moved to '%s'" % self.old_dir) - result_exec['stdout'] += extra_msg - - if private_key: - try: - os.unlink(helper) - os.unlink(indetity) - except: - pass - - return result_exec - - def _get_command(self, url, branch): - param = '' - if branch and branch != 'master': - param = " -b " + str(branch) - - command = self.git_cmd + " clone" + param + " --quiet --verbose '" + url + "' ." - command_pull = self.git_cmd + " pull --quiet --verbose" - - if os.path.isdir(self.working_dir + '/.git'): - # GIT clone on already .git repo, do a pull and hope... - command = command_pull - - return command - - def _certificate_helper(self, private_key=None): - """ - Returns the path to a helper script which can be used in the GIT_SSH env - var to use a custom private key file. - """ - opts = { - 'StrictHostKeyChecking': 'no', - 'PasswordAuthentication': 'no', - 'KbdInteractiveAuthentication': 'no', - 'ChallengeResponseAuthentication': 'no', - } - - # Create identity file - identity = NamedTemporaryFile(delete=False) - ecm.chmod(identity.name, 0600) - identity.writelines([private_key]) - identity.close() - - # Create helper script - helper = NamedTemporaryFile(delete=False) - helper.writelines([ - '#!/bin/sh\n', - 'exec ssh ' + - ' '.join('-o%s=%s' % (key, value) for key, value in opts.items()) + - ' -i ' + identity.name + - ' $*\n' - ]) - - helper.close() - ecm.chmod(helper.name, 0750) - - return helper.name, identity.name - - def _is_available(self): - """ checks if git is on path - """ - if ecm.is_win(): - return ecm.which('git.exe') - - return ecm.which('git') - - def _install(self): - """ Try to install git - """ - ecm.install_package('git') - return bool(self._is_available()) - - -class SVN: - def __init__(self, working_dir, rotate=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - # Get git path - self.svn_cmd = self._is_available() - - if not self.svn_cmd: - if not self._install(): - raise Exception('Unable to find or install subversion') - self.svn_cmd = self._is_available() - - def clone(self, url, branch, envars, username, password, private_key): - """ svn co URL - """ - # Add username and password to url - if username and password: - url = url.replace('://', '://' + username + ':' + password + '@') - - command = self.svn_cmd + " co '" + url + "' ." - - out, stdout, stderr = ecm.run_command(command=command, workdir=self.working_dir, envars=envars) - result_exec = ecm.format_output(out, stdout, stderr) - - if not result_exec['out']: - extra_msg = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - if self.old_dir: - extra_msg += ecm.output("Old source files moved to '%s'" % self.old_dir) - - if result_exec['stdout']: - result_exec['stdout'] = extra_msg + result_exec['stdout'] - - else: - result_exec['stdout'] = extra_msg - - return result_exec - - def _is_available(self): - """ is svn on path - """ - if ecm.is_win(): - return ecm.which('svn.cmd') - return ecm.which('svn') - - def _install(self): - """ try to install subversion - """ - ecm.install_package('subversion') - return self._is_available() - - -class FILE: - def __init__(self, working_dir, rotate=False, extract=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - self.extract = extract - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - def clone(self, branch, envars, url, username, password, private_key): - """ Downloads a file from a remote url and decompress it - """ - file_downloaded = ecm.download_file( - url=url, - user=username, - passwd=password - ) - - tmp_dir = os.path.dirname(file_downloaded) - - if file_downloaded: - if self.extract: - extract = self._extract(file_downloaded) - - if extract: - extract['head'] = '' - if extract.get('stdout', None): - extract['head'] = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - - if extract.get('stdout', None) and self.old_dir: - extract['head'] += ecm.output("Old source files moved to '%s'" % self.old_dir) - else: - move(file_downloaded, self.working_dir) - extract = { - 'stdout': ecm.output("Source deployed successfully to '%s'" % self.working_dir), - 'stderr': '', - 'out': 0 - } - - else: - rmtree(tmp_dir, ignore_errors=True) - raise Exception("Unable to download file") - - # Clean and output - rmtree(tmp_dir, ignore_errors=True) - ret = { - 'stdout': extract.get('head', '') + extract.get('stdout', ''), - 'stderr': extract.get('stderr', 'Unable to download file'), - 'out': extract.get('out', 1) - } - - return ret - - def _extract(self, filename): - """ extractor helper - """ - try: - file_type = self._get_file_type(filename) - opener = mode = None - - if file_type == 'zip': - opener, mode = zipfile.ZipFile, 'r' - - elif file_type == 'gz': - if tarfile.is_tarfile(filename): - opener, mode = tarfile.open, 'r:gz' - - elif file_type == 'bz2': - if tarfile.is_tarfile(filename): - opener, mode = tarfile.open, 'r:bz2' - - if not opener: - raise Exception("Unsupported file compression") - - cfile = opener(filename, mode) - - # if first member is dir, skip 1st container path - if file_type == 'zip': - members = cfile.namelist() - else: - members = cfile.getmembers() - - stdout = '' - for member in members: - if file_type == 'zip': - member_name = member - else: - member_name = member.name - - stdout += "Extracted " + member_name + "\n" - cfile.extractall(self.working_dir) - cfile.close() - - except Exception as e: - try: - return self._extract_alternative(filename) - except: - raise Exception("Could not extract file: %s" % e) - - ret = {'out': 0, 'stderr': '', 'stdout': stdout} - return ret - - def _extract_alternative(self, filename): - """ extractor helper: Try to extract file using system commands - """ - from shutil import move - from os import path - - file_type = self._get_file_type(filename) - - # Move file before decompress - move(filename, self.working_dir) - filename = path.join(self.working_dir, path.basename(filename)) - - if file_type == 'zip': - package = 'unzip' - command = 'unzip' - args = [filename] - - elif file_type == 'gz': - package = 'gzip' - command = 'gunzip' - args = [filename] - - elif file_type == 'bz2': - package = 'bzip2' - command = 'bzip2' - args = ['-d', filename] - - else: - raise Exception("Unsupported file compression") - - exists = ecm.which(command) - if not exists: - # Try to install package - ecm.install_package(package) - exists = ecm.which(command) - - if exists and command: - # Decompress - out, stdout, stderr = ecm.run_command(command, args, workdir=self.working_dir) - ret = {'out': out, 'stderr': stderr, 'stdout': stdout} - return ret - - raise Exception("Could not extract file") - - def _get_file_type(self, filename): - """ get compressed file type based on marks - """ - magic_dict = { - "\x1f\x8b\x08": "gz", - "\x42\x5a\x68": "bz2", - "\x50\x4b\x03\x04": "zip" - } - - max_len = max(len(x) for x in magic_dict) - with open(filename) as f: - file_start = f.read(max_len) - for magic, filetype in magic_dict.items(): - if file_start.startswith(magic): - return filetype - - return False - - -class Deploy: - def __init__(self, working_dir, rotate): - self.working_dir = os.path.abspath(working_dir) - self.rotate = rotate - - def prepare(self): - """ Common function to create and rotate - """ - to_dir = None - if self.rotate and os.path.isdir(self.working_dir): - drive, path = os.path.splitdrive(self.working_dir) - split_path = ecm.split_path(path) - - try: - a = split_path[1] - except IndexError: - # Unsafe rotate - self.rotate = False - - if self.rotate: - to_dir = self.working_dir + '_rotated_' + ecm.utime() - move(self.working_dir, to_dir) - - # create working dir - if not os.path.isdir(self.working_dir): - ecm.mkdir_p(self.working_dir) - - return to_dir - - def rollback(self, path): - return - - -ECMSource().run() From 4de18817cc09580a1ea2ab8cae3729e3da23637d Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 10:41:11 +0100 Subject: [PATCH 20/57] timeout refactor --- ecagent/agent.py | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 9cc35f6..0d78531 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -80,11 +80,14 @@ def __init__(self, config): self.running_commands = {} self.num_running_commands = 0 + self.check_timeout = False self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) - self.keepalive = LoopingCall(self._reconnect) + # self.keepalive = LoopingCall(self._reconnect) + + self.timestamp = 0 log.debug("Loading XMPP...") Client.__init__( @@ -93,12 +96,12 @@ def __init__(self, config): [("/iq[@type='set']", self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) - def _reconnect(self): - """ - Disconnect the current reactor to try to connect again - """ - log.info("No data received in %ss: Trying to reconnect" % KEEPALIVED_TIMEOUT) - reactor.disconnectAll() + # def _reconnect(self): + # """ + # Disconnect the current reactor to try to connect again + # """ + # log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) + # reactor.disconnectAll() def __onIq(self, msg): """ @@ -110,18 +113,21 @@ def __onIq(self, msg): log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) - if self.keepalive.running: - log.debug("Stop keepalived") - self.keepalive.stop() - - log.debug("Starting keepalived") - self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) + self.check_timeout = False + self.timestamp = time() + + # if self.keepalive.running: + # log.debug("Stop keepalived") + # self.keepalive.stop() + # + # log.debug("Starting keepalived") + # self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) message = IqMessage(msg) recv_command = message.command.replace('.', '_') if recv_command in self.running_commands: - if time() > self.running_commands[recv_command]: + if self.timestamp > self.running_commands[recv_command]: del self.running_commands[recv_command] self.num_running_commands -= 1 log.debug("Deleted %s from running_commands dict as should have been completed" % (recv_command)) @@ -135,14 +141,10 @@ def __onIq(self, msg): if message.from_ not in self._online_contacts: log.warn('IQ sender not in roster (%s), dropping message' % message.from_) else: - self.running_commands[recv_command] = time() + int(message.command_args['timeout']) + self.running_commands[recv_command] = self.timestamp + int(message.command_args['timeout']) self.num_running_commands += 1 log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) self._processCommand(message) - - else: - log.warn('Unknown ecm_message received: Full XML:\n%s' % (msg.toXml())) - del message else: log.debug("already running given command %s" %recv_command) @@ -219,9 +221,16 @@ def _send(self, result, message): self.send(message.toEtree()) def _check_memory(self, num_running_commands): + + if self.check_timeout: + if time() >= self.timestamp + KEEPALIVED_TIMEOUT: + log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) + reactor.disconnectAll() + rss, vms = mem_usage() log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) reactor.stop() + self.check_timeout = True From e27738faaceff4f98b2fb6f67e928842c5352756 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 10:50:35 +0100 Subject: [PATCH 21/57] type error --- ecagent/client.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/ecagent/client.py b/ecagent/client.py index 9edf575..a60b33a 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -37,7 +37,7 @@ def __init__(self, config, observers, resource='XMPPClient'): """ my_observers = [ ('/presence', self._onPresence), - ('/iq', self._onPossibleErrorIq), + ("/iq[@type='error']", self._onPossibleErrorIq), ] my_observers.extend(observers) @@ -63,15 +63,14 @@ def __init__(self, config, observers, resource='XMPPClient'): ) def _onPossibleErrorIq(self, elem): - if elem['type'] == "error": - sender = elem['from'] - for el in elem.elements(): - if el.name == 'error' and el['code'] == '404': - log.warn('Received a 404 code from the server, setting the target user as offline') - if sender in self._online_contacts: - self._online_contacts.remove(sender) - else: - log.debug('Received a 404 from %s which not (anymore?) in the online contacts list.') + sender = elem['from'] + for el in elem.elements(): + if el.name == 'error' and el['code'] == '404': + log.warn('Received a 404 code from the server, setting the target user as offline') + if sender in self._online_contacts: + self._online_contacts.remove(sender) + else: + log.debug('Received a 404 from %s which not (anymore?) in the online contacts list.') def _onPresence(self, elem): """ From 33263d1d9b5ca214f17d4aac2405fc31c538d485 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 11:24:43 +0100 Subject: [PATCH 22/57] enable logging --- ecagent/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecagent/client.py b/ecagent/client.py index a60b33a..a7b9868 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging as log - +import ecagent.twlogging as log # Twisted imports from twisted.internet.defer import DeferredSemaphore from twisted.words.xish.domish import Element From b5c3ad10d5a52758e4f55d65b722008ce1429b01 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 11:39:26 +0100 Subject: [PATCH 23/57] timeout without using time() --- ecagent/agent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 0d78531..6501046 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -81,6 +81,7 @@ def __init__(self, config): self.running_commands = {} self.num_running_commands = 0 self.check_timeout = False + self.counter = 0 self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) @@ -113,7 +114,7 @@ def __onIq(self, msg): log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) - self.check_timeout = False + self.counter = 0 self.timestamp = time() # if self.keepalive.running: @@ -221,11 +222,10 @@ def _send(self, result, message): self.send(message.toEtree()) def _check_memory(self, num_running_commands): - - if self.check_timeout: - if time() >= self.timestamp + KEEPALIVED_TIMEOUT: - log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) - reactor.disconnectAll() + log.info("Value of counter : %s" % self.counter) + if (_CHECK_RAM_INTERVAL * self.counter) >= KEEPALIVED_TIMEOUT: + log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) + reactor.disconnectAll() rss, vms = mem_usage() log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) @@ -233,4 +233,4 @@ def _check_memory(self, num_running_commands): log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) reactor.stop() - self.check_timeout = True + self.counter += 1 From 597ca2cc1dbc7849fae5ac643c67605200ec2e9e Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 12:39:50 +0100 Subject: [PATCH 24/57] clean memory only in _check_memory --- ecagent/agent.py | 12 +----------- ecagent/core.py | 1 - ecagent/message.py | 2 -- ecagent/runner.py | 7 ------- 4 files changed, 1 insertion(+), 21 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 6501046..5ddb990 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -109,8 +109,6 @@ def __onIq(self, msg): A new IQ message has been received and we should process it. """ log.debug('__onIq') - mem_clean('__onIq [start]') - log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) @@ -153,7 +151,6 @@ def __onIq(self, msg): self._send(result, message) del msg - mem_clean('__onIq [end]') def _processCommand(self, message): if not self.verify.signature(message): @@ -164,7 +161,6 @@ def _processCommand(self, message): flush_callback = self._flush message.command_name = message.command.replace('.', '_') - mem_clean('run_command [start]') d = self.command_runner.run_command(message, flush_callback) # Clean message @@ -176,7 +172,6 @@ def _processCommand(self, message): errbackKeywords={'message': message}, ) del message - mem_clean('run_command [end]') return d else: @@ -185,11 +180,9 @@ def _processCommand(self, message): self._onCallFinished(result, message) del message - mem_clean('run_command [end]') return def _onCallFinished(self, result, message): - mem_clean('agent._onCallFinished') log.debug('Call Finished') self._send(result, message) del self.running_commands[message.command_name] @@ -213,20 +206,17 @@ def _flush(self, result, message): def _send(self, result, message): log.debug('Send Response') - mem_clean('agent._send') message.toResult(*result) del result - mem_clean('agent._send') self.send(message.toEtree()) def _check_memory(self, num_running_commands): - log.info("Value of counter : %s" % self.counter) if (_CHECK_RAM_INTERVAL * self.counter) >= KEEPALIVED_TIMEOUT: log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) reactor.disconnectAll() - + mem_clean('periodic memory clean') rss, vms = mem_usage() log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: diff --git a/ecagent/core.py b/ecagent/core.py index 89b4ad0..ee6feee 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -28,7 +28,6 @@ # Local import twlogging as log -from functions import mem_clean # Add registerAccount to XMPPAuthenticator diff --git a/ecagent/message.py b/ecagent/message.py index a578cb1..99e3aaa 100644 --- a/ecagent/message.py +++ b/ecagent/message.py @@ -23,7 +23,6 @@ # Local import ecagent.twlogging as log -from ecagent.functions import mem_clean AGENT_VERSION_CORE = 3 AGENT_VERSION_PROTOCOL = 1 @@ -121,7 +120,6 @@ def toEtree(self): return msg def toXml(self): - mem_clean('toXml [start]') return self.toXml() def toResult(self, retvalue, stdout, stderr, timed_out, partial=0): diff --git a/ecagent/runner.py b/ecagent/runner.py index 3d25c50..5af41f8 100644 --- a/ecagent/runner.py +++ b/ecagent/runner.py @@ -33,7 +33,6 @@ from twisted.internet.error import ProcessTerminated, ProcessDone import ecagent.twlogging as log -from ecagent.functions import mem_clean class CommandRunner(): @@ -66,10 +65,6 @@ def __init__(self, config): self._commands = {} reactor.callWhenRunning(self._load_commands) - def __del__(self): - log.debug("__del__ CommandRunner()") - mem_clean("__del__: CommandRunner") - def _load_commands(self): for path in self.command_paths: log.debug("Processing dir: %s" % path) @@ -134,7 +129,6 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None else: log.info("[INIT] Loading commands from %s" % filename) - mem_clean('spawnProcess [start]') crp = CommandRunnerProcess(cmd_timeout, command_args, flush_callback, message) d = crp.getDeferredResult() reactor.spawnProcess(crp, command, args, env=self.env) @@ -142,7 +136,6 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None del cmd_timeout, filename, command_name, command_args del flush_callback, message, args - mem_clean('spawnProcess [end]') return d From 83de55b1d28e696ca1675e4632d1d517a5d00fc5 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 14 Dec 2015 17:00:21 +0100 Subject: [PATCH 25/57] mem clean refactored --- ecagent/agent.py | 7 +++---- ecagent/functions.py | 12 +++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 5ddb990..b93cca6 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -144,13 +144,13 @@ def __onIq(self, msg): self.num_running_commands += 1 log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) self._processCommand(message) - del message else: log.debug("already running given command %s" %recv_command) result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) self._send(result, message) del msg + del message def _processCommand(self, message): if not self.verify.signature(message): @@ -216,11 +216,10 @@ def _check_memory(self, num_running_commands): if (_CHECK_RAM_INTERVAL * self.counter) >= KEEPALIVED_TIMEOUT: log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) reactor.disconnectAll() - mem_clean('periodic memory clean') - rss, vms = mem_usage() - log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) + rss = mem_clean('periodic memory clean') if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) reactor.stop() + del rss self.counter += 1 diff --git a/ecagent/functions.py b/ecagent/functions.py index c401d01..91f238f 100644 --- a/ecagent/functions.py +++ b/ecagent/functions.py @@ -36,20 +36,18 @@ def mem_usage(): vms /= 1000000.0 except: pass + log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) return rss, vms -def mem_clean(where='', dolog=False): +def mem_clean(where=''): _collect = collect() rss, vms = mem_usage() string = "_mem_clean: %s collected %d objects. (current mem: rss=%sMB | vms=%sMB)" % (where, _collect, rss, vms) - if dolog: - log.info(string) + log.debug(string) - else: - log.debug(string) - - del _collect, where, dolog, rss, vms, string + del _collect, where, vms, string + return rss \ No newline at end of file From 117e50c467ecbf07ad9ec97948d17867f266e4dc Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Tue, 15 Dec 2015 10:23:06 +0100 Subject: [PATCH 26/57] AGENT GROUPS CONFIG (configure.py --groups) --- config/ecagent.cfg | 2 +- configure.py | 17 ++++-- ecagent/config.py | 7 +++ plugins/plugin_reboot_require.py | 100 ------------------------------- 4 files changed, 20 insertions(+), 106 deletions(-) delete mode 100644 plugins/plugin_reboot_require.py diff --git a/config/ecagent.cfg b/config/ecagent.cfg index 955f461..d7f4fde 100644 --- a/config/ecagent.cfg +++ b/config/ecagent.cfg @@ -20,4 +20,4 @@ tools_path_windows = .\tools\win32 [ssl] public_key = ./config/xmpp_cert.pub -private_key = ./config/private.key +private_key = ./config/private.key \ No newline at end of file diff --git a/configure.py b/configure.py index cf8aae7..e815711 100755 --- a/configure.py +++ b/configure.py @@ -32,12 +32,13 @@ sys.path.append(".") configure_account = None +configure_server_groups = None configure_groups = None try: - optlist, args = getopt.getopt(sys.argv[1:], 'a:s:', ["account=", "server-groups="]) + optlist, args = getopt.getopt(sys.argv[1:], 'a:s:g:', ["account=", "server-groups=", "groups="]) except getopt.GetoptError: - print 'Please configure agent with ./configure --account=XXXXX' + print 'Please configure agent with ./configure.py --account=XXXXX' sys.exit(-1) for option, value in optlist: @@ -45,10 +46,13 @@ configure_account = value elif option in ("-s", "--server-groups"): + configure_server_groups = value + + elif option in ("-g", "--groups"): configure_groups = value -if not configure_account and not configure_groups: - print 'Please configure agent with ./configure --account=XXXXX' +if not configure_account and not configure_server_groups: + print 'Please configure agent with ./configure.py --account=XXXXX' sys.exit(-1) root_dir = os.path.dirname(os.path.realpath(__file__)) @@ -68,8 +72,11 @@ if configure_account: config['XMPP']['account'] = configure_account +if configure_server_groups: + config['XMPP']['groups'] = configure_server_groups + if configure_groups: - config['XMPP']['groups'] = configure_groups + config['XMPP']['agent_groups'] = configure_groups config.write() print 'Manual configuration override succeeded.' diff --git a/ecagent/config.py b/ecagent/config.py index 2051c0c..d202662 100644 --- a/ecagent/config.py +++ b/ecagent/config.py @@ -110,6 +110,7 @@ def _get_config2(self, unique_id): address = self._get_ip() uuid = self._get_stored_uuid() account = self.get_stored_account() + agent_groups = self.get_stored_agent_groups params = "?uuid=%s&ipaddress=%s&hostname=%s&unique_id=%s&account=%s" \ % (uuid, address, hostname, unique_id, account) @@ -148,6 +149,10 @@ def _get_stored_unique_id(self): def get_stored_account(self): return self['XMPP'].get('account', '') + + def get_stored_agent_groups(self): + return self['XMPP'].get('agent_groups', '') + def parse_meta_data(self, json_data): meta_data = None try: @@ -217,3 +222,5 @@ def _get_unique_id(): unique_id = 'mac::' + ':'.join(findall('..', '%012x' % getnode())) return unique_id + + diff --git a/plugins/plugin_reboot_require.py b/plugins/plugin_reboot_require.py deleted file mode 100644 index 2454a9b..0000000 --- a/plugins/plugin_reboot_require.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -import os -from commands import getstatusoutput -from pkg_resources import parse_version - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -pkg_kit = True - -try: - from gi.repository import PackageKitGlib -except ImportError: - pkg_kit = False - - -class ECMRebootRequirePackageKit(ECMPlugin): - def cmd_reboot_require(self, *argv, **kwargs): - from platform import release, machine - from gi.repository import PackageKitGlib - - working_kernel = release() - - client = PackageKitGlib.Client() - client.refresh_cache(False, None, lambda p, t, d: True, None) - res = client.resolve(PackageKitGlib.FilterEnum.INSTALLED, ['kernel'], None, lambda p, t, d: True, None) - - if res.get_exit_code() != PackageKitGlib.ExitEnum.SUCCESS: - return False - - package_ids = res.get_package_array() - - if len(package_ids) == 0: - return False - - installed_kernel = None - - for pkg in package_ids: - if pkg.get_arch() == machine(): - if installed_kernel is None: - installed_kernel = pkg - else: - if parse_version(pkg.get_version()) > parse_version(installed_kernel.get_version()): - installed_kernel = pkg - - installed_kernel = installed_kernel.get_version() + '.' +installed_kernel.get_arch() - - return parse_version(installed_kernel) > parse_version(working_kernel) - -class ECMRebootRequire(ECMPlugin): - def cmd_reboot_require(self, *argv, **kwargs): - distribution, _version = ecm.get_distribution() - if distribution.lower() in ['debian', 'ubuntu']: - if os.path.exists('/var/run/reboot-required.pkgs') or os.path.exists('/var/run/reboot-required'): - return True - else: - retval, current_kernel = getstatusoutput('uname -r') - if retval != 0: - return False - retval, latest_kernel = getstatusoutput("dpkg --list | grep linux-image | head -n1 | cut -d ' ' -f3 | perl -pe 's/^linux-image-(\S+).*/$1/'") - if retval != 0: - return False - return parse_version(latest_kernel) > parse_version(current_kernel) - return False - - elif distribution.lower() in ['centos', 'redhat', 'fedora', 'amazon']: - retval, current_kernel = getstatusoutput('uname -r') - if retval != 0: - return False - retval, latest_kernel = getstatusoutput("rpm -q --last kernel | perl -pe 's/^kernel-(\S+).*/$1/' | head -1") - if retval != 0: - return False - return parse_version(latest_kernel) > parse_version(current_kernel) - return False - - - -if pkg_kit: - ECMRebootRequirePackageKit().run() -else: - ECMRebootRequire().run() \ No newline at end of file From ec158c92f170970c21bba337369c244b672032d1 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Tue, 15 Dec 2015 10:33:34 +0100 Subject: [PATCH 27/57] the error message in stream end in log make it look like something went wrong --- ecagent/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecagent/core.py b/ecagent/core.py index ee6feee..6d3969b 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -126,7 +126,7 @@ def _failed_auth(self, error): def _stream_end(self, error): """ overwrite in derivated class """ - log.info("XMPPClient stream end: %s" % error) + log.info("XMPPClient stream end") def _connected(self, xml_stream): log.info("XMPPClient connected") From 52fd837c3c2ba5a99f335337af6a7c5d0fe1b907 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 16 Dec 2015 11:51:58 +0100 Subject: [PATCH 28/57] refactor timeout --- ecagent/agent.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index b93cca6..4fb1f17 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- + # -*- coding:utf-8 -*- # Copyright (C) 2012 Juan Carlos Moreno # @@ -28,7 +28,7 @@ import ecagent.twlogging as log from ecagent.verify import ECVerify -from ecagent.functions import mem_clean, mem_usage +from ecagent.functions import mem_clean from message import AGENT_VERSION_PROTOCOL @@ -38,7 +38,6 @@ _CHECK_RAM_MAX_RSS_MB = 125 _CHECK_RAM_INTERVAL = 60 -KEEPALIVED_TIMEOUT = 120 class SMAgent: def __init__(self, config): @@ -108,6 +107,9 @@ def __onIq(self, msg): """ A new IQ message has been received and we should process it. """ + if self.memory_checker.running: + self.memory_checker.stop() + log.debug('__onIq') log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) @@ -188,6 +190,9 @@ def _onCallFinished(self, result, message): del self.running_commands[message.command_name] self.num_running_commands -= 1 log.debug('command finished %s' %message.command_name) + + if self.num_running_commands == 0 and not self.memory_checker.running: + self.memory_checker.start(_CHECK_RAM_INTERVAL) def _onCallFailed(self, failure, *argv, **kwargs): log.error("onCallFailed") @@ -199,6 +204,9 @@ def _onCallFailed(self, failure, *argv, **kwargs): del self.running_commands[message.command_name] self.num_running_commands -= 1 self._onCallFinished(result, message) + + if self.num_running_commands == 0 and not self.memory_checker.running: + self.memory_checker.start(_CHECK_RAM_INTERVAL) def _flush(self, result, message): log.debug('Flush Message') @@ -213,8 +221,8 @@ def _send(self, result, message): self.send(message.toEtree()) def _check_memory(self, num_running_commands): - if (_CHECK_RAM_INTERVAL * self.counter) >= KEEPALIVED_TIMEOUT: - log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) + if self.counter == 2: + log.info("No data received in %s sec: Trying to reconnect" % (time() - self.timestamp)) reactor.disconnectAll() rss = mem_clean('periodic memory clean') if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: From 2b96df4d24fd76bc38ba832706c8d337bd535fb7 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 16 Dec 2015 13:10:14 +0100 Subject: [PATCH 29/57] fallback --- ecagent/agent.py | 52 ++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 4fb1f17..2ecb614 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -1,4 +1,4 @@ - # -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- # Copyright (C) 2012 Juan Carlos Moreno # @@ -38,6 +38,7 @@ _CHECK_RAM_MAX_RSS_MB = 125 _CHECK_RAM_INTERVAL = 60 +KEEPALIVED_TIMEOUT = 120 class SMAgent: def __init__(self, config): @@ -79,15 +80,12 @@ def __init__(self, config): self.running_commands = {} self.num_running_commands = 0 - self.check_timeout = False - self.counter = 0 + self.timestamp = 0 self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) - # self.keepalive = LoopingCall(self._reconnect) - - self.timestamp = 0 + self.keepalive = LoopingCall(self._reconnect) log.debug("Loading XMPP...") Client.__init__( @@ -96,33 +94,30 @@ def __init__(self, config): [("/iq[@type='set']", self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) - # def _reconnect(self): - # """ - # Disconnect the current reactor to try to connect again - # """ - # log.info("No data received in %s sec: Trying to reconnect" % KEEPALIVED_TIMEOUT) - # reactor.disconnectAll() + def _reconnect(self): + """ + Disconnect the current reactor to try to connect again + """ + log.info("No data received in %ss: Trying to reconnect" % KEEPALIVED_TIMEOUT) + reactor.disconnectAll() def __onIq(self, msg): """ A new IQ message has been received and we should process it. """ - if self.memory_checker.running: - self.memory_checker.stop() - log.debug('__onIq') + log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) - self.counter = 0 - self.timestamp = time() + if self.keepalive.running: + log.debug("Stop keepalived") + self.keepalive.stop() + + log.debug("Starting keepalived") + self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) - # if self.keepalive.running: - # log.debug("Stop keepalived") - # self.keepalive.stop() - # - # log.debug("Starting keepalived") - # self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) + self.timestamp = time() message = IqMessage(msg) recv_command = message.command.replace('.', '_') @@ -146,6 +141,7 @@ def __onIq(self, msg): self.num_running_commands += 1 log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) self._processCommand(message) + else: log.debug("already running given command %s" %recv_command) result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) @@ -190,9 +186,6 @@ def _onCallFinished(self, result, message): del self.running_commands[message.command_name] self.num_running_commands -= 1 log.debug('command finished %s' %message.command_name) - - if self.num_running_commands == 0 and not self.memory_checker.running: - self.memory_checker.start(_CHECK_RAM_INTERVAL) def _onCallFailed(self, failure, *argv, **kwargs): log.error("onCallFailed") @@ -204,9 +197,6 @@ def _onCallFailed(self, failure, *argv, **kwargs): del self.running_commands[message.command_name] self.num_running_commands -= 1 self._onCallFinished(result, message) - - if self.num_running_commands == 0 and not self.memory_checker.running: - self.memory_checker.start(_CHECK_RAM_INTERVAL) def _flush(self, result, message): log.debug('Flush Message') @@ -221,13 +211,9 @@ def _send(self, result, message): self.send(message.toEtree()) def _check_memory(self, num_running_commands): - if self.counter == 2: - log.info("No data received in %s sec: Trying to reconnect" % (time() - self.timestamp)) - reactor.disconnectAll() rss = mem_clean('periodic memory clean') if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) reactor.stop() del rss - self.counter += 1 From e9264f5abcf56c742b07492baca217cb2c4e1695 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 4 Jan 2016 17:54:52 +0100 Subject: [PATCH 30/57] changes for windows --- ecagent/agent.py | 1 - ecagent/runner.py | 3 ++- ecagentd.tac | 8 +++++++- monitor/mplugin/__base__/__base__.py | 19 +++++++++++-------- plugins/__mplugin.py | 5 +++-- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 2ecb614..8cf534b 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import resource from time import time # Twisted imports diff --git a/ecagent/runner.py b/ecagent/runner.py index 5af41f8..514e6d4 100644 --- a/ecagent/runner.py +++ b/ecagent/runner.py @@ -73,7 +73,8 @@ def _load_commands(self): for filename in os.listdir(path): if not filename.startswith('plugin_'): continue - if os.path.splitext(filename)[1] != '.py': + + if os.path.splitext(filename)[1] not in ['.py','.exe']: continue log.debug(" Queuing plugin %s for process." % filename) diff --git a/ecagentd.tac b/ecagentd.tac index e1076bd..3362ae4 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -24,7 +24,13 @@ import random # Twisted from twisted.application.service import Application -root_dir = os.path.dirname(os.path.realpath(__file__)) + +try: + root_dir = os.path.dirname(os.path.abspath(__file__)) +except NameError: # We are the main py2exe script, not a module + import sys + root_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + os.chdir(root_dir) if root_dir not in sys.path: diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index 9617f42..bbb1424 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -420,14 +420,17 @@ def _boot_time_windows(): except: return - @staticmethod - def _get_load(): - m1, m5, m15 = getloadavg() - return { - '1m': m1, - '5m': m5, - '15m': m15 - } + def _get_load(self): + if not self.is_win(): + from os import getloadavg + m1, m5, m15 = getloadavg() + return { + '1m': m1, + '5m': m5, + '15m': m15 + } + + return 0 @staticmethod def _to_dict(obj): diff --git a/plugins/__mplugin.py b/plugins/__mplugin.py index f5c4606..39ed566 100644 --- a/plugins/__mplugin.py +++ b/plugins/__mplugin.py @@ -53,8 +53,9 @@ class MPlugin: def __init__(self, plugin_path=None): # set alarm for timeout - signal.signal(signal.SIGALRM, _timeout) - signal.alarm(CHECK_TIMEOUT) + if not self.is_win(): + signal.signal(signal.SIGALRM, _timeout) + signal.alarm(CHECK_TIMEOUT) if plugin_path: self.path = abspath(plugin_path) From efc6d83272121db4f0efb49f026183c5968d695d Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 11 Jan 2016 09:58:48 +0100 Subject: [PATCH 31/57] fix for windows --- ecagent/runner.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ecagent/runner.py b/ecagent/runner.py index 514e6d4..b289e24 100644 --- a/ecagent/runner.py +++ b/ecagent/runner.py @@ -108,10 +108,11 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None need_sudo = ['plugin_pip.py', 'plugin_service.py', 'plugin_update.py', 'plugin_haproxy.py', 'plugin_monitor.py', 'plugin_pip_extra.py', 'plugin_puppet.py', 'plugin_saltstack.py', 'plugin_proc.py'] ext = os.path.splitext(filename)[1] if ext == '.py': - if os.path.split(filename)[1] in need_sudo: - command = 'sudo' - # -u: sets unbuffered output - args = [command, self._python_runner, '-u', '-W ignore::DeprecationWarning', filename, command_name] + from sys import platform + if platform.startswith("win32") or os.path.split(filename)[1] not in need_sudo: + command = self._python_runner + args = [command, '-u', '-W ignore::DeprecationWarning', filename, command_name] + else: command = self._python_runner # -u: sets unbuffered output From cdad12fa9289302e5f3467fa08c772011c0f96e7 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 13 Jan 2016 12:57:27 +0100 Subject: [PATCH 32/57] get inode uses from df -i --- monitor/mplugin/__base__/__base__.py | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index bbb1424..98a7892 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -230,6 +230,46 @@ def _get_network(self): pass return retval + + def _get_partition_inodes(self): + """ + runs df -i + Filesystem Inodes IUsed IFree IUse% Mounted on + /dev/mapper/fedora-root 2436448 335632 2100816 14% / + /dev/mapper/fedora-home 1310720 49938 1260782 4% /home + + and returns output as list of dictionaries + + [ { 'Filesystem': '/dev/mapper/fedora-root', + 'IFree': '2100817', + 'IUse%': '14%', + 'IUsed': '335631', + 'Inodes': '2436448', + 'Mounted on': '/'}, + { 'Filesystem': '/dev/mapper/fedora-home', + 'IFree': '1260404', + 'IUse%': '4%', + 'IUsed': '50316', + 'Inodes': '1310720', + 'Mounted on': '/home'}] + """ + + from commands import getstatusoutput + + retcode, inodes = getstatusoutput('df -i') + + inodes = inodes.split() + inode = inodes[7:] + + inode_list = [] + i = 0 + + while i Date: Wed, 13 Jan 2016 18:47:22 +0100 Subject: [PATCH 33/57] inodes done --- monitor/mplugin/__base__/__base__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index 98a7892..593d989 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -50,6 +50,7 @@ def run(self): 'net': self._get_network(), 'netstat': self._get_netstat(), 'disk_io': self._get_disk_io(), + 'inodes' : self._get_inodes(), 'cputimes': self._get_cpu_times(), 'process': self._get_processes(), 'swap': self._get_swap(), @@ -231,7 +232,7 @@ def _get_network(self): return retval - def _get_partition_inodes(self): + def _get_inodes(self): """ runs df -i Filesystem Inodes IUsed IFree IUse% Mounted on @@ -253,6 +254,8 @@ def _get_partition_inodes(self): 'Inodes': '1310720', 'Mounted on': '/home'}] """ + if self.is_win(): + return [] from commands import getstatusoutput @@ -268,7 +271,14 @@ def _get_partition_inodes(self): inode_list.append({'Filesystem': inode[i], 'Inodes': inode[i + 1], 'IUsed': inode[i + 2], 'IFree': inode[i + 3], 'IUse%': inode[i + 4], 'Mounted on': inode[i + 5]}) i += 6 - return inode_list + ret_inodes = [] + + for part in psutil.disk_partitions(all=False): + for inode in inode_list: + if part.mountpoint == inode['Mounted on']: + ret_inodes.append(inode) + + return ret_inodes def _get_netstat(self): From aef63d9444643e8e74cb3c75071ffaee01c88bf0 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Thu, 14 Jan 2016 09:25:51 +0100 Subject: [PATCH 34/57] inodes code refactored --- monitor/mplugin/__base__/__base__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index 593d989..899202b 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -264,21 +264,21 @@ def _get_inodes(self): inodes = inodes.split() inode = inodes[7:] + partitions_list = [] + + for part in psutil.disk_partitions(all=False): + partitions_list.append(part.mountpoint) + inode_list = [] i = 0 while i Date: Mon, 7 Mar 2016 11:19:59 +0100 Subject: [PATCH 35/57] the percentage is -5% than what shown by df due to reserved blocks --- monitor/mplugin/__base__/__base__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index 899202b..e04ac4c 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -131,7 +131,7 @@ def _get_disk(self): if hasattr(usage, 'free'): tmp['free'] = self.to_gb(usage.free) if hasattr(usage, 'percent'): - tmp['percent'] = usage.percent + tmp['percent'] = (float(usage.total - usage.free) / float(usage.total)) * 100 retval[part.mountpoint] = tmp From fbff42bffff16eb66e19b531af9f5af7763bb8de Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Wed, 9 Mar 2016 09:31:55 +0100 Subject: [PATCH 36/57] Remove python-dbus dependency --- build/debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/debian/control b/build/debian/control index b6aeec7..5d4b641 100644 --- a/build/debian/control +++ b/build/debian/control @@ -8,6 +8,6 @@ Homepage: http://www.ecmanaged.com Package: ecmanaged-ecagent Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo, python-dbus +Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo Description: ECM Agent - ECMANAGED Monitor and deploy agent Deploy and monitor agent for ECmanaged cloud management tool, based on XMPP protocol. From e5112c2216c54a83b649da0b224515ec776998c3 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 9 Mar 2016 10:01:38 +0100 Subject: [PATCH 37/57] remove python-dbus dependency from fedora rpm --- build/redhat/ecmanaged-ecagent.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/redhat/ecmanaged-ecagent.spec b/build/redhat/ecmanaged-ecagent.spec index 9024a3a..74d0644 100644 --- a/build/redhat/ecmanaged-ecagent.spec +++ b/build/redhat/ecmanaged-ecagent.spec @@ -18,7 +18,7 @@ Vendor: Juan Carlos Moreno Packager: Arindam Choudhury Provides: ecmanaged-ecagent #fedora -Requires: python2 python-devel python-twisted python-protocols python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils dbus-python sudo +Requires: python2 python-devel python-twisted python-protocols python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils sudo #centos Requires: python2 python-devel python-twisted-core python-twisted-web python-twisted-words python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils dbus-python sudo Url: www.ecmanaged.com From 5fa65906ec8e740bdf6549d442c97c5055740bce Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 9 Mar 2016 10:53:49 +0100 Subject: [PATCH 38/57] retain config file while updating --- config/{ecagent.cfg => ecagent.cfg.init} | 0 ecagentd.tac | 12 +++++++++++- setup.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) rename config/{ecagent.cfg => ecagent.cfg.init} (100%) diff --git a/config/ecagent.cfg b/config/ecagent.cfg.init similarity index 100% rename from config/ecagent.cfg rename to config/ecagent.cfg.init diff --git a/ecagentd.tac b/ecagentd.tac index 3362ae4..2d97cfb 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -19,7 +19,6 @@ import os import gc import sys import stat -import random # Twisted from twisted.application.service import Application @@ -65,6 +64,17 @@ open(pid_file, 'w').write(str(os.getpid())) # Start agent and setup logging config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') +config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init') + +# rename config/ecagent.cfg.init to config/ecagent.cfg for fresh install + +if os.path.exists(config_file_init): + os.rename(config_file_init, config_file) + + +if not os.path.isfile (config_file): + print "Failed to find config file." + sys.exit(-1) os.chmod(config_file, stat.S_IRWXU) diff --git a/setup.py b/setup.py index 4db463e..989517d 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ packages=['ecagent', 'plugins','monitor.mplugin.__base__'], - data_files=[('config', ['config/ecagent.cfg', 'config/xmpp_cert.pub']), + data_files=[('config', ['config/ecagent.cfg.init', 'config/xmpp_cert.pub']), ('monitor/mplugin/__base__', ['monitor/mplugin/__base__/data.json']), ('/etc/sudoers.d', ['sudoers.d/ecmanaged']), ('/etc/cron.d', ['cron.d/ecmanaged-ecagent']), From a47d74eeeecd3f4cc52fb9a4f97269d92497a528 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 9 Mar 2016 11:00:32 +0100 Subject: [PATCH 39/57] retain config file while updating --- ecagentd.tac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecagentd.tac b/ecagentd.tac index 2d97cfb..7a830ad 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -68,7 +68,9 @@ config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.in # rename config/ecagent.cfg.init to config/ecagent.cfg for fresh install -if os.path.exists(config_file_init): +if os.path.exists(config_file) and os.path.exists(config_file_init): + os.remove(config_file_init) +else: os.rename(config_file_init, config_file) From 031abc8e49d73117d57de45ca6b2fe3080400cc4 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 9 Mar 2016 11:31:44 +0100 Subject: [PATCH 40/57] retain config file while updating --- ecagentd.tac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecagentd.tac b/ecagentd.tac index 7a830ad..73f1095 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -70,9 +70,9 @@ config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.in if os.path.exists(config_file) and os.path.exists(config_file_init): os.remove(config_file_init) -else: - os.rename(config_file_init, config_file) +if os.path.exists(config_file_init) and not os.path.exists(config_file): + os.rename(config_file_init, config_file) if not os.path.isfile (config_file): print "Failed to find config file." From f16fca4f5303d78fdc811cf7614fed30c06dff21 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Thu, 10 Mar 2016 12:04:07 +0100 Subject: [PATCH 41/57] Update agent.py --- ecagent/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecagent/agent.py b/ecagent/agent.py index 8cf534b..5a0b3bf 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -37,7 +37,7 @@ _CHECK_RAM_MAX_RSS_MB = 125 _CHECK_RAM_INTERVAL = 60 -KEEPALIVED_TIMEOUT = 120 +KEEPALIVED_TIMEOUT = 180 class SMAgent: def __init__(self, config): From f908d26f5edd2e2865fc3ee98fcf961238ec0e99 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Sat, 12 Mar 2016 08:30:26 +0100 Subject: [PATCH 42/57] add support for configu_file_init --- configure.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.py b/configure.py index e815711..a552467 100755 --- a/configure.py +++ b/configure.py @@ -60,12 +60,15 @@ # Parse config file or end execution config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') +config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init # manipulate configuration file if not os.path.isfile(config_file): - print 'Unable to read the config file at %s' % config_file - print 'Agent will now quit' - sys.exit(-1) + config_file = config_file_init + if not os.path.isfile(config_file): + print 'Unable to read the config file at %s' % config_file + print 'Agent will now quit' + sys.exit(-1) config = SMConfigObj(config_file) From 6c2738501e287e449aa79a12978c38fa552b3de3 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Sat, 12 Mar 2016 09:20:06 +0000 Subject: [PATCH 43/57] configure.py changes --- config/ecagent.cfg.init | 2 +- configure.py | 33 ++++++++++++++------------------- setup.py | 2 +- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/config/ecagent.cfg.init b/config/ecagent.cfg.init index d7f4fde..955f461 100644 --- a/config/ecagent.cfg.init +++ b/config/ecagent.cfg.init @@ -20,4 +20,4 @@ tools_path_windows = .\tools\win32 [ssl] public_key = ./config/xmpp_cert.pub -private_key = ./config/private.key \ No newline at end of file +private_key = ./config/private.key diff --git a/configure.py b/configure.py index a552467..0bfad14 100755 --- a/configure.py +++ b/configure.py @@ -36,23 +36,20 @@ configure_groups = None try: - optlist, args = getopt.getopt(sys.argv[1:], 'a:s:g:', ["account=", "server-groups=", "groups="]) + optlist, args = getopt.getopt(sys.argv[1:], 'a:t:', ["account=", "tags="]) except getopt.GetoptError: - print 'Please configure agent with ./configure.py --account=XXXXX' + print 'Please configure agent with ./configure.py --account=XXXXX --tags=XXX,XXX' sys.exit(-1) for option, value in optlist: if option in ("-a", "--account"): configure_account = value - elif option in ("-s", "--server-groups"): - configure_server_groups = value + elif option in ("-t", "--tags"): + configure_tags = value - elif option in ("-g", "--groups"): - configure_groups = value - -if not configure_account and not configure_server_groups: - print 'Please configure agent with ./configure.py --account=XXXXX' +if not configure_account and not configure_tags: + print 'Please configure agent with ./configure.py --account=XXXXX --tags=XXX' sys.exit(-1) root_dir = os.path.dirname(os.path.realpath(__file__)) @@ -60,26 +57,24 @@ # Parse config file or end execution config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') -config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init +config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init') # manipulate configuration file if not os.path.isfile(config_file): config_file = config_file_init - if not os.path.isfile(config_file): - print 'Unable to read the config file at %s' % config_file - print 'Agent will now quit' - sys.exit(-1) + +if not os.path.isfile(config_file): + print 'Unable to read the config file at %s' % config_file + print 'Agent will now quit' + sys.exit(-1) config = SMConfigObj(config_file) if configure_account: config['XMPP']['account'] = configure_account -if configure_server_groups: - config['XMPP']['groups'] = configure_server_groups - -if configure_groups: - config['XMPP']['agent_groups'] = configure_groups +if configure_tags: + config['XMPP']['tags'] = configure_tags config.write() print 'Manual configuration override succeeded.' diff --git a/setup.py b/setup.py index 989517d..462adb0 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ from distutils.core import setup setup(name='ecmanaged-ecagent', - version='3.0', + version='3.1', license='Apache v2', description='ECManaged Agent - Monitoring and deployment agent', long_description='ECManaged Agent - Monitoring and deployment agent', From 31a597e83604d27c527ee077a6478ed390769548 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Sat, 12 Mar 2016 09:31:38 +0000 Subject: [PATCH 44/57] configure.py changes --- build/debian/changelog | 10 ++++++++-- configure.py | 3 +-- setup.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/build/debian/changelog b/build/debian/changelog index 2035bbe..decd586 100644 --- a/build/debian/changelog +++ b/build/debian/changelog @@ -1,5 +1,11 @@ -ecmanaged-ecagent (3.0-1) stable; urgency=low +ecmanaged-ecagent (3.2-2) stable; urgency=low - * version 3.0 + * version 3.2 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 + +ecmanaged-ecagent (3.2-1) stable; urgency=low + + * version 3.2 -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 diff --git a/configure.py b/configure.py index 0bfad14..f105fc7 100755 --- a/configure.py +++ b/configure.py @@ -32,8 +32,7 @@ sys.path.append(".") configure_account = None -configure_server_groups = None -configure_groups = None +configure_tags = None try: optlist, args = getopt.getopt(sys.argv[1:], 'a:t:', ["account=", "tags="]) diff --git a/setup.py b/setup.py index 462adb0..1161d28 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ from distutils.core import setup setup(name='ecmanaged-ecagent', - version='3.1', + version='3.2', license='Apache v2', description='ECManaged Agent - Monitoring and deployment agent', long_description='ECManaged Agent - Monitoring and deployment agent', From c892cc479d214e0ebfbafd0d20dad54bdf638ca3 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 23 Mar 2016 13:10:54 +0100 Subject: [PATCH 45/57] run as sudo --- ecagent/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecagent/runner.py b/ecagent/runner.py index b289e24..13c92f6 100644 --- a/ecagent/runner.py +++ b/ecagent/runner.py @@ -114,9 +114,9 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None args = [command, '-u', '-W ignore::DeprecationWarning', filename, command_name] else: - command = self._python_runner + command = 'sudo' # -u: sets unbuffered output - args = [command, '-u', '-W ignore::DeprecationWarning', filename, command_name] + args = [command, self._python_runner, '-u', '-W ignore::DeprecationWarning', filename, command_name] else: command = filename From 0aa041b28235d929b2d82baf49467a3381ab31e9 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Thu, 28 Jul 2016 17:41:50 +0200 Subject: [PATCH 46/57] Update __base__.py exclude bindfs from _get_inodes() --- monitor/mplugin/__base__/__base__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index e04ac4c..b5c9ea2 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -269,13 +269,20 @@ def _get_inodes(self): for part in psutil.disk_partitions(all=False): partitions_list.append(part.mountpoint) - inode_list = [] + inode_list = [] i = 0 - while i Date: Wed, 7 Sep 2016 17:34:23 +0200 Subject: [PATCH 47/57] Update plugin_monitor.py python multiprocessing for windows --- plugins/plugin_monitor.py | 107 +++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/plugins/plugin_monitor.py b/plugins/plugin_monitor.py index def627d..0b34591 100644 --- a/plugins/plugin_monitor.py +++ b/plugins/plugin_monitor.py @@ -19,6 +19,9 @@ import os import sys +from multiprocessing import Process +from multiprocessing import Pool + import simplejson as json from base64 import b64decode from time import time @@ -53,20 +56,22 @@ class ECMMonitor(ECMPlugin): - def cmd_monitor_get(self, *argv, **kwargs): """ Runs monitor commands from monitor path """ - + config = None b64_config = kwargs.get('config', None) - + + thread_pool = Pool(20) + try: - config = json.loads(b64decode(b64_config)) + if isinstance(b64_config, str): + config = json.loads(b64decode(b64_config)) except: pass - + retval = [] to_execute = [] @@ -74,18 +79,18 @@ def cmd_monitor_get(self, *argv, **kwargs): self._check_path(MPLUGIN_PATH) self._check_path(CPLUGIN_PATH) self._check_path(CACHE_PATH) - + # Clean old cache files self._cache_clean() if not os.path.isdir(MPLUGIN_PATH): return retval - + # Foreach plugin inside mplugins and custom for plugin_path in [MPLUGIN_PATH, CPLUGIN_PATH]: if not os.path.isdir(plugin_path): continue - + for p_path in os.listdir(plugin_path): p_path = os.path.join(plugin_path, p_path) @@ -96,7 +101,7 @@ def cmd_monitor_get(self, *argv, **kwargs): # Skip disabled plugins if os.path.basename(p_path).startswith('.'): continue - + runas = None scripts = [] interval = COMMAND_INTERVAL @@ -104,30 +109,30 @@ def cmd_monitor_get(self, *argv, **kwargs): # Search for plugin files if plugin_path == MPLUGIN_PATH: mplugin = MPlugin(p_path) - + # Update config and re-read if config: mplugin.write_config(config.get(mplugin.id)) mplugin = MPlugin(p_path) - + runas = mplugin.data.get('runas', None) interval = mplugin.data.get('interval', COMMAND_INTERVAL) script = os.path.join(plugin_path, p_path, mplugin.id) - + if not os.path.exists(script): script += '.py' - + if not os.path.exists(script): continue # Add as valid mplugin script scripts = [script] - + # Executable plugin base if not os.access(script, os.X_OK): os.chmod(script, 0755) - + # Custom plugins path else: # Set default interval or read from path name @@ -137,47 +142,51 @@ def cmd_monitor_get(self, *argv, **kwargs): _tmp = os.path.join(plugin_path, p_path, filename) if os.access(_tmp, os.X_OK): scripts.append(_tmp) - + for script in scripts: # Read last result from cache (even if void) from_cache = self._cache_read(script, interval + CACHE_SOFT_TIME) retval.append(self._parse_script_name(script) + GLUE + str(interval) + GLUE + from_cache) - + # Execute script if cache wont be valid on next command_get execution if not self._cache_read(script, interval - COMMAND_INTERVAL - CACHE_SOFT_TIME): to_execute.append({'script': script, 'runas': runas}) - - for data in to_execute: - _run_background_file(data['script'], data['runas']) + for data in to_execute: + if ecm.is_win(): + p = Process(target=_run_background_file, args=(data['script'], data['runas'],)) + p.start() + p.join() + else: + _run_background_file(data['script'], data['runas']) return retval - + def cmd_monitor_plugin_install(self, *argv, **kwargs): """ Installs a plugin [url=plugin_url] """ url = kwargs.get('url', None) content = None - + if not url: raise ecm.InvalidParameters(self.cmd_monitor_plugin_install.__doc__) - + try: content = ecm.get_url(url) except: - pass - + raise Exception("Unable to get URL: %s" % url) + if not content: raise Exception("Unable to get URL: %s" % url) - - try: + + try: plugin = json.loads(content) except: raise Exception("Invalid data received") id = plugin.get('id') runas = plugin.get('runas') - + arg_config = plugin.get('config') arg_script_b64 = plugin.get('script') @@ -214,7 +223,7 @@ def cmd_monitor_plugin_install(self, *argv, **kwargs): script = b64decode(arg_script_b64) except: pass - + config = { 'id': id, 'runas': runas, @@ -223,16 +232,16 @@ def cmd_monitor_plugin_install(self, *argv, **kwargs): 'version': plugin.get('version'), 'config': arg_config } - + if id and config and script: mplugin = MPlugin(MPLUGIN_PATH) if mplugin.install(id, config, script): # Installation ok, run it script_file = os.path.abspath(os.path.join(MPLUGIN_PATH, id, id)) _run_background_file(script_file, runas) - + return True - + return False def cmd_monitor_plugin_uninstall(self, *argv, **kwargs): @@ -241,20 +250,20 @@ def cmd_monitor_plugin_uninstall(self, *argv, **kwargs): """ plugin_id = kwargs.get('id', None) - + if not plugin_id: raise ecm.InvalidParameters(self.cmd_monitor_plugin_uninstall.__doc__) - + mplugin = MPlugin(MPLUGIN_PATH) return mplugin.uninstall(plugin_id) - + @staticmethod def _interval_from_path(my_path): interval = os.path.split(my_path)[-1] - + if not ecm.is_integer(interval): interval = COMMAND_INTERVAL - + return int(interval) @staticmethod @@ -279,13 +288,13 @@ def _cache_read(command, max_old): # check updated cache if os.path.isfile(cache_file): modified = os.path.getmtime(cache_file) - + how_old = int(time() - modified) - - if(how_old > max_old): + + if (how_old > max_old): # Invalid cache file os.remove(cache_file) - + else: # Return cache content f = open(cache_file, 'r') @@ -294,7 +303,7 @@ def _cache_read(command, max_old): f.close() return content - + @staticmethod def _cache_clean(): for f in os.listdir(CACHE_PATH): @@ -319,20 +328,22 @@ def _run_background_file(script, run_as=None): script_name = os.path.basename(fullpath) workdir = os.path.dirname(script) - # Create child process and return if parent - if ecm.fork(workdir): - return + + if ecm.is_linux(): + # Create child process and return if parent + if ecm.fork(workdir): + return # Write timeout to cache file _write_cache(script_name, CRITICAL, 'Timeout') - + try: command = [fullpath] sys.path.append(MY_PATH) - + env = os.environ.copy() env['PYTHONPATH'] = MY_PATH + ':' + env.get('PYTHONPATH', '') - + retval, stdout, stderr = ecm.run_command(command, runas=run_as, envars=env) _write_cache(script_name, retval, stdout) From 2275b204ce04d14462526d49efedf1e88dfaffe4 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Wed, 7 Sep 2016 17:35:08 +0200 Subject: [PATCH 48/57] Update __helper.py added is_linux --- plugins/__helper.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/__helper.py b/plugins/__helper.py index 752bd06..90a8969 100644 --- a/plugins/__helper.py +++ b/plugins/__helper.py @@ -45,7 +45,7 @@ _FLUSH_WORKER_SLEEP_TIME = 0.2 -AGENT_VERSION = 3.0 +AGENT_VERSION = 4.0 def is_win(): @@ -56,6 +56,15 @@ def is_win(): return False + +def is_linux(): + """ Returns True if is a windows system + """ + if platform.startswith("linux"): + return True + + return False + def file_write(file_path, content=None): """ Writes a file """ @@ -100,6 +109,7 @@ def clean_stdout(std_output): try: r = re.compile("\033\[[0-9;]*m", re.MULTILINE) return r.sub('', std_output) + except: return std_output @@ -678,7 +688,7 @@ def command(self, command, args=None, std_input=None, run_as=None, working_dir=N if run_as and not is_win(): if not check_sudo(): - return 255, '', 'Sudo is not available:' + return 255, '', 'sudo is not available:' # don't use su - xxx or env variables will not be available command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))] From 7783a56cdb46b15e921229cb593b613b900503a4 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 9 Sep 2016 11:24:03 +0200 Subject: [PATCH 49/57] Update __base__.py windows --- monitor/mplugin/__base__/__base__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index b5c9ea2..dc35c09 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -19,10 +19,14 @@ # Base monitor for agent please do not modify it import psutil +import os +import sys -from os import getloadavg, getpid +from os import getpid from time import time, sleep +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'plugins')) + from __mplugin import MPlugin from __mplugin import OK, CRITICAL @@ -144,6 +148,7 @@ def _get_disk(self): return retval def _get_disk_io(self): + retval = {} from os.path import basename try: @@ -161,7 +166,6 @@ def _get_disk_io(self): continue res[device] = disk_io[device] - retval = {} retval = self.counters(self._to_data(res), 'disk_io') except: pass @@ -286,7 +290,6 @@ def _get_inodes(self): i += 6 return inode_list - def _get_netstat(self): try: From 1738a7e029fbccc2e4bd86713e323ce57e6f5280 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 13 Sep 2016 13:16:28 +0200 Subject: [PATCH 50/57] Pool(20) only on windows --- plugins/plugin_monitor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/plugin_monitor.py b/plugins/plugin_monitor.py index 0b34591..840b6a4 100644 --- a/plugins/plugin_monitor.py +++ b/plugins/plugin_monitor.py @@ -64,7 +64,9 @@ def cmd_monitor_get(self, *argv, **kwargs): config = None b64_config = kwargs.get('config', None) - thread_pool = Pool(20) + # Use Pool on windows systems + if ecm.is_win(): + thread_pool = Pool(20) try: if isinstance(b64_config, str): From d308194ae7d06af91625213ac7ff479a3bf75b71 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 13 Sep 2016 18:04:23 +0200 Subject: [PATCH 51/57] refactor get_inodes --- monitor/mplugin/__base__/__base__.py | 54 +++++----------------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index dc35c09..0685454 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -237,60 +237,22 @@ def _get_network(self): return retval def _get_inodes(self): - """ - runs df -i - Filesystem Inodes IUsed IFree IUse% Mounted on - /dev/mapper/fedora-root 2436448 335632 2100816 14% / - /dev/mapper/fedora-home 1310720 49938 1260782 4% /home - - and returns output as list of dictionaries - - [ { 'Filesystem': '/dev/mapper/fedora-root', - 'IFree': '2100817', - 'IUse%': '14%', - 'IUsed': '335631', - 'Inodes': '2436448', - 'Mounted on': '/'}, - { 'Filesystem': '/dev/mapper/fedora-home', - 'IFree': '1260404', - 'IUse%': '4%', - 'IUsed': '50316', - 'Inodes': '1310720', - 'Mounted on': '/home'}] - """ if self.is_win(): return [] - from commands import getstatusoutput - - retcode, inodes = getstatusoutput('df -i') - - inodes = inodes.split() - inode = inodes[7:] - - partitions_list = [] - - for part in psutil.disk_partitions(all=False): - partitions_list.append(part.mountpoint) - inode_list = [] - i = 0 - - while i < len(inode): + for part in psutil.disk_partitions(all=False): try: - if str(inode[i + 2]) == 'bindfs': - i += 6 - continue - - if inode[i + 5] in partitions_list: - inode_list.append({'Filesystem': inode[i], 'Inodes': inode[i + 1], 'IUsed': inode[i + 2], - 'IFree': inode[i + 3], 'IUse%': inode[i + 4], 'Mounted on': inode[i + 5]}) - except Exception as e: + data = statvfs(part.mountpoint) + iused = data.f_files - data.f_ffree + iused_p = int(iused * 100 / data.f_files) + inode_list.append({'Filesystem': part.device, 'Inodes': data.f_files, 'IUsed': iused, + 'IFree': data.f_ffree, 'IUse%': iused_p, 'Mounted on': part.mountpoint}) + except: pass - i += 6 return inode_list - + def _get_netstat(self): try: # Only psutil > v2 From 93f967594bf653e1868aa20c2883fde3e8ef607d Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 20 Sep 2016 18:29:05 +0200 Subject: [PATCH 52/57] Update __mplugin.py --- plugins/__mplugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/__mplugin.py b/plugins/__mplugin.py index 39ed566..7bf0492 100644 --- a/plugins/__mplugin.py +++ b/plugins/__mplugin.py @@ -68,7 +68,7 @@ def __init__(self, plugin_path=None): log.basicConfig( filename=join(self.path, LOG_FILE_NAME), format='%(levelname)s:%(message)s', - level=log.DEBUG + level=log.INFO ) # Read configuration From 7d95ae07021ff64db7dcc0195a26661ba19c68f6 Mon Sep 17 00:00:00 2001 From: Juan Carlos Moreno Date: Tue, 20 Sep 2016 18:29:38 +0200 Subject: [PATCH 53/57] Update __helper.py --- plugins/__helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/__helper.py b/plugins/__helper.py index 90a8969..cfa890f 100644 --- a/plugins/__helper.py +++ b/plugins/__helper.py @@ -162,7 +162,7 @@ def download_file(url, filename=None, user=None, passwd=None): return filename -def get_url(url, timeout=10): +def get_url(url, timeout=30): socket.setdefaulttimeout(timeout) urlopen = urllib.urlopen(url) From c519675cae71b6e5a854c41d6abd5797dff321ee Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 10 Feb 2017 11:53:55 +0100 Subject: [PATCH 54/57] Update init updated home to reflect current working directory --- init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init b/init index 4c9fd99..d0f28ce 100755 --- a/init +++ b/init @@ -2,7 +2,7 @@ NAME=${NAME:-ecagentd} TWISTD="$(which twistd)" -HOME=${DIR:-/opt/ecmanaged/ecagent} +HOME=${DIR:-$(pwd)} PID_FILE=${PID_FILE:-"$HOME/twistd.pid"} TAC=${TAC:-"$HOME/$NAME.tac"} RETVAL=0 From 6c12110c46599c352de438f70546553e1fec11d1 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 23 Jun 2017 14:52:00 +0200 Subject: [PATCH 55/57] change version --- build/debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/debian/changelog b/build/debian/changelog index decd586..a8cac13 100644 --- a/build/debian/changelog +++ b/build/debian/changelog @@ -1,3 +1,9 @@ +ecmanaged-ecagent (3.3-1) stable; urgency=low + + * version 3.3 + + -- Juan Carlos Moreno (ECMANAGED) Fri, 23 May 2017 14:48:25 +0200 + ecmanaged-ecagent (3.2-2) stable; urgency=low * version 3.2 From acbcb062581f63efbd712c43eedd9b9743c39d85 Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Fri, 23 Jun 2017 16:25:32 +0200 Subject: [PATCH 56/57] build for debian --- build/build.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 build/build.txt diff --git a/build/build.txt b/build/build.txt new file mode 100644 index 0000000..978f2e4 --- /dev/null +++ b/build/build.txt @@ -0,0 +1,19 @@ +# Debian + +change build/debian/changelog and push to repo + +sudo apt-get update + +sudo apt-get upgrade + +sudo apt-get install build-essential git devscripts debhelper dh-python python-all python-setuptools + +git clone https://github.com/ecmanaged/ecm-agent.git + +cd ecm-agent + +cp -r build/debian . + +rm -rf build + +dpkg-buildpackage \ No newline at end of file From 8e7769fc4185e263abbde2396c9286fea62ce52c Mon Sep 17 00:00:00 2001 From: Arindam Choudhury Date: Mon, 26 Jun 2017 17:08:29 +0200 Subject: [PATCH 57/57] fix identation --- plugins/__helper.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/plugins/__helper.py b/plugins/__helper.py index cfa890f..b46c0cc 100644 --- a/plugins/__helper.py +++ b/plugins/__helper.py @@ -49,12 +49,13 @@ def is_win(): - """ Returns True if is a windows system - """ - if platform.startswith("win32"): - return True + """ Returns True if is a windows system + """ - return False + if platform.startswith("win32"): + return True + + return False def is_linux(): @@ -148,7 +149,7 @@ def download_file(url, filename=None, user=None, passwd=None): if _header_filename: filename = path.join(path.dirname(filename), _header_filename) except: - pass + pass with open(filename, 'wb') as fp: while True: @@ -265,7 +266,7 @@ def install_package(packages, update=True): if update: run_command(['apt-get', '-y', '-qq', 'update'], runas='root') - + command = ['apt-get', '-o', 'Dpkg::Options::=--force-confold', @@ -279,7 +280,7 @@ def install_package(packages, update=True): elif distribution.lower() in ['centos', 'redhat', 'fedora', 'amazon']: if update: run_command(['yum', '-y', 'clean', 'all'], runas='root') - + command = ['yum', '-y', '--nogpgcheck', @@ -587,12 +588,12 @@ def check_sudo(): return False p = Popen( - [which('sudo'),'-n','id'], - bufsize=0, - stdout=PIPE, - stderr=PIPE, - universal_newlines=True, - close_fds=(os.name == 'posix') + [which('sudo'), '-n', 'id'], + bufsize=0, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + close_fds=(os.name == 'posix') ) retval = p.wait() @@ -657,7 +658,7 @@ def __init__(self): self.thread_stderr = '' self.thread_run = 1 - def command(self, command, args=None, std_input=None, run_as=None, working_dir=None, envars=None, only_stdout = False): + def command(self, command, args=None, std_input=None, run_as=None, working_dir=None, envars=None, only_stdout=False): """ Execute command and flush stdout/stderr using threads """ @@ -689,7 +690,7 @@ def command(self, command, args=None, std_input=None, run_as=None, working_dir=N if run_as and not is_win(): if not check_sudo(): return 255, '', 'sudo is not available:' - + # don't use su - xxx or env variables will not be available command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))]