diff --git a/.gitignore b/.gitignore index 5a101d5..661dc4e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ log monitor/mplugin/__base__/data.json */*.pyc -config/ecagent.cfg monitor/.cache/* monitor/mplugin/__base__/.counter.dat monitor/mplugin/__base__/.touch diff --git a/build/build.txt b/build/build.txt new file mode 100644 index 0000000..978f2e4 --- /dev/null +++ b/build/build.txt @@ -0,0 +1,19 @@ +# Debian + +change build/debian/changelog and push to repo + +sudo apt-get update + +sudo apt-get upgrade + +sudo apt-get install build-essential git devscripts debhelper dh-python python-all python-setuptools + +git clone https://github.com/ecmanaged/ecm-agent.git + +cd ecm-agent + +cp -r build/debian . + +rm -rf build + +dpkg-buildpackage \ No newline at end of file diff --git a/build/debian.old/changelog b/build/debian.old/changelog new file mode 100644 index 0000000..eac9407 --- /dev/null +++ b/build/debian.old/changelog @@ -0,0 +1,5 @@ +ecmanaged-ecagent (3.0.0-1) stable; urgency=low + + * version 3.0 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 diff --git a/build/debian.old/compat b/build/debian.old/compat new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/build/debian.old/compat @@ -0,0 +1 @@ +5 diff --git a/build/debian/templates b/build/debian.old/conffiles similarity index 100% rename from build/debian/templates rename to build/debian.old/conffiles diff --git a/build/debian.old/control b/build/debian.old/control new file mode 100644 index 0000000..5e40af8 --- /dev/null +++ b/build/debian.old/control @@ -0,0 +1,13 @@ +Source: ecmanaged-ecagent +Section: admin +Priority: optional +Maintainer: Juan Carlos Moreno +Build-Depends: cdbs, debhelper (>= 7.0.50) +Standards-Version: 3.9.2 +Homepage: http://www.ecmanaged.com + +Package: ecmanaged-ecagent +Architecture: all +Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-apt, python-httplib2 +Description: ecmanaged-ecagent + ECM Agent - ECMANAGED Monitor and deploy agent diff --git a/build/debian.old/copyright b/build/debian.old/copyright new file mode 100644 index 0000000..bc8f769 --- /dev/null +++ b/build/debian.old/copyright @@ -0,0 +1,18 @@ +Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=166 +Upstream-Name: ecmanaged-ecagent + +Files: * +Copyright: 2012 Juan Carlos Moreno +License: Apache License, Version 2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. diff --git a/build/debian/cron.d b/build/debian.old/cron.d similarity index 100% rename from build/debian/cron.d rename to build/debian.old/cron.d diff --git a/build/debian/dirs b/build/debian.old/dirs similarity index 100% rename from build/debian/dirs rename to build/debian.old/dirs diff --git a/build/debian.old/ecmanaged-ecagent.dirs b/build/debian.old/ecmanaged-ecagent.dirs new file mode 100644 index 0000000..eb6ebf4 --- /dev/null +++ b/build/debian.old/ecmanaged-ecagent.dirs @@ -0,0 +1 @@ +opt/ecmanaged/ecagent diff --git a/build/debian/install b/build/debian.old/install similarity index 100% rename from build/debian/install rename to build/debian.old/install diff --git a/build/debian.old/postinst b/build/debian.old/postinst new file mode 100644 index 0000000..54c2e8e --- /dev/null +++ b/build/debian.old/postinst @@ -0,0 +1,10 @@ +#! /bin/bash +# postinst script for ecmanaged-ecagent +# +# see: dh_installdeb(1) + +# Stop ecagent +INIT=/opt/ecmanaged/ecagent/init +if [ -x ${INIT} ]; then + ${INIT} start +fi diff --git a/build/debian/prerm b/build/debian.old/prerm similarity index 78% rename from build/debian/prerm rename to build/debian.old/prerm index c8bfa2f..49cbde6 100644 --- a/build/debian/prerm +++ b/build/debian.old/prerm @@ -5,13 +5,14 @@ set -e -CONF=/opt/ecmanaged/ecagent/config/ecagent.cfg +# Stop ecagent +INIT=/opt/ecmanaged/ecagent/init +if [ -x ${INIT} ]; then + ${INIT} stop +fi case "$1" in remove) - if test ! -f "$CONF"; then - rm -f $CONF - fi ;; upgrade|deconfigure) ;; diff --git a/build/debian.old/rules b/build/debian.old/rules new file mode 100755 index 0000000..4bf4b42 --- /dev/null +++ b/build/debian.old/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f + +# Exclude git directory +export DH_ALWAYS_EXCLUDE=GIT:ecagent/.git + +%: + dh $@ + +override_dh_fixperms: + dh_fixperms + chmod 0700 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config + chmod 0400 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config/ecagent.cfg + chmod 0755 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent diff --git a/build/debian.old/templates b/build/debian.old/templates new file mode 100644 index 0000000..e69de29 diff --git a/build/debian/changelog b/build/debian/changelog index e69de29..a8cac13 100644 --- a/build/debian/changelog +++ b/build/debian/changelog @@ -0,0 +1,17 @@ +ecmanaged-ecagent (3.3-1) stable; urgency=low + + * version 3.3 + + -- Juan Carlos Moreno (ECMANAGED) Fri, 23 May 2017 14:48:25 +0200 + +ecmanaged-ecagent (3.2-2) stable; urgency=low + + * version 3.2 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 + +ecmanaged-ecagent (3.2-1) stable; urgency=low + + * version 3.2 + + -- Juan Carlos Moreno (ECMANAGED) Tue, 05 May 2015 17:42:25 +0200 diff --git a/build/debian/compat b/build/debian/compat index 7ed6ff8..f11c82a 100644 --- a/build/debian/compat +++ b/build/debian/compat @@ -1 +1 @@ -5 +9 \ No newline at end of file diff --git a/build/debian/conffiles b/build/debian/conffiles deleted file mode 100644 index b1edf78..0000000 --- a/build/debian/conffiles +++ /dev/null @@ -1 +0,0 @@ -/opt/ecmanaged/ecagent/config/ecagent.cfg diff --git a/build/debian/control b/build/debian/control index 5e40af8..5d4b641 100644 --- a/build/debian/control +++ b/build/debian/control @@ -1,13 +1,13 @@ Source: ecmanaged-ecagent +Maintainer: Arindam Choudhury Section: admin Priority: optional -Maintainer: Juan Carlos Moreno -Build-Depends: cdbs, debhelper (>= 7.0.50) -Standards-Version: 3.9.2 +Build-Depends: dh-python, python (>= 2.6.6-3), debhelper (>= 9) +Standards-Version: 3.9.6 Homepage: http://www.ecmanaged.com Package: ecmanaged-ecagent Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-apt, python-httplib2 -Description: ecmanaged-ecagent - ECM Agent - ECMANAGED Monitor and deploy agent +Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo +Description: ECM Agent - ECMANAGED Monitor and deploy agent + Deploy and monitor agent for ECmanaged cloud management tool, based on XMPP protocol. diff --git a/build/debian/copyright b/build/debian/copyright index bc8f769..70738bb 100644 --- a/build/debian/copyright +++ b/build/debian/copyright @@ -1,18 +1,45 @@ -Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=166 +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: ecmanaged-ecagent +Upstream-Contact: Juan Carlos Moreno +Source: http://www.ecmanaged.com Files: * -Copyright: 2012 Juan Carlos Moreno -License: Apache License, Version 2.0 +Copyright: 2012, ACKSTORM, S.L. All rights reserved. +License: GPL-3+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at + your option) any later version. + . + In addition, as a special exception, the copyright holders give + permission to link the code of portions of this program with the + OpenSSL library under certain conditions as described in each + individual source file, and distribute linked combinations including + the two. + . + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the + file(s), but you are not obligated to do so. If you do not wish to do + so, delete this exception statement from your version. If you delete + this exception statement from all source files in the program, then + also delete it here. + . + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian GNU/Linux systems, the complete text of the GNU General + Public License (GPL) version 3 can be found at + /usr/share/common-licenses/GPL-3. - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. +Files: debian/* +Copyright: 2010-2012, ACK STORM, S.L. +License: GPL-3+ + On Debian GNU/Linux systems, the complete text of the GNU General + Public License (GPL) version 3 can be found at + /usr/share/common-licenses/GPL-3 \ No newline at end of file diff --git a/build/debian/ecmanaged-ecagent.dirs b/build/debian/ecmanaged-ecagent.dirs deleted file mode 100644 index eca14de..0000000 --- a/build/debian/ecmanaged-ecagent.dirs +++ /dev/null @@ -1,2 +0,0 @@ -etc/apt/sources.list.d -opt/ecmanaged/ecagent diff --git a/build/debian/ecmanaged-ecagent.init b/build/debian/ecmanaged-ecagent.init deleted file mode 100755 index 53341fc..0000000 --- a/build/debian/ecmanaged-ecagent.init +++ /dev/null @@ -1,124 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: ecagentd -# Required-Start: $remote_fs $network -# Required-Stop: $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start ECM Agent daemon -### END INIT INFO - -set +e - -NAME=ecagentd -TWISTD=/usr/bin/twistd -TAC=$NAME.tac -DESC="ECM Agent" - -test -x $TWISTD || exit 0 - -DIR=/opt/ecmanaged/ecagent -PID_FILE=$DIR/twistd.pid - -# Check working dir -if test ! -d "$DIR"; then - echo "Unable to access work dir: $DIR" - exit 1; -fi - -export LANG="C" -export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" - -# clean APT related environment when started from cloud-init -# bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=439763 - -unset PERL_DL_NONLAZY -unset DEBCONF_REDIR -unset DEBIAN_FRONTEND -unset DEBIAN_HAS_FRONTEND -unset DPKG_NO_TSTP - -# define LSB log_* functions. -. /lib/lsb/init-functions - -getpid() { - echo `cat $PID_FILE 2>/dev/null` -} - -killagent() { - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) kill -9 $PID >/dev/null 2>&1; pkill -f ${TAC} >/dev/null 2>&1 ;; - *) pkill -f ${TAC} >/dev/null 2>&1 ;; - esac -} - -zombies() { - PID=$(getpid) - for i in $(ps axo ppid,state,etimes|grep -w Z|awk '{if ($3 >= 300) print $1 }'); do - if [ $i -eq ${PID} ]; then - $0 restart - fi - done -} - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - killagent - /bin/rm -f $PID_FILE > /dev/null 2>&1 - start-stop-daemon --oknodo --start --pidfile $PID_FILE \ - --background --chdir $DIR --exec $TWISTD -- -ny $TAC >/dev/null 2>&1 - case "$?" in - 0) log_end_msg 0 ;; - *) log_end_msg 1; exit 1 ;; - esac - ;; - - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - start-stop-daemon --retry 5 --oknodo --stop --signal 9 --quiet --pidfile $PID_FILE 2>/dev/null - case "$?" in - 0) log_end_msg 0; killagent ;; - *) log_end_msg 1; killagent ; exit 1 ;; - esac - ;; - - status) - log_daemon_msg "Checking status of $DESC" "$NAME" - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) log_end_msg 0 ;; - *) log_end_msg 1; exit 1 ;; - esac - ;; - - check) - log_daemon_msg "Checking status of $DESC" "$NAME" - PID=$(getpid) - ls -l /proc/$PID/exe > /dev/null 2>&1 - case "$?" in - 0) zombies; log_end_msg 0 ;; - *) $0 restart ;; - esac - ;; - - kill) - killagent - ;; - - restart) - $0 stop - $0 start - ;; - - *) - echo "Usage: /etc/init.d/$NAME {start|stop|restart|status|check}" >&2 - exit 1 - ;; - -esac - -exit 0 diff --git a/build/debian/postinst b/build/debian/postinst index 6ef5e00..14f2c79 100644 --- a/build/debian/postinst +++ b/build/debian/postinst @@ -1,5 +1,16 @@ -#! /bin/bash -# postinst script for ecmanaged-ecagent -# -# see: dh_installdeb(1) +#!/bin/sh +set -e + +if getent passwd ecmanaged >/dev/null 2>&1; then + chown -R ecmanaged:ecmanaged /opt/ecmanaged + mkdir -p /etc/ecmanaged + chown -R ecmanaged:ecmanaged /etc/ecmanaged +fi + +case "$1" in + configure) + if [ -f /opt/ecmanaged/ecagent/init ]; then + /opt/ecmanaged/ecagent/init start > /dev/null 2>&1 + fi +esac \ No newline at end of file diff --git a/dist_build/debian/preinst b/build/debian/preinst similarity index 100% rename from dist_build/debian/preinst rename to build/debian/preinst diff --git a/build/debian/rules b/build/debian/rules index 188aeb6..3b90b0d 100755 --- a/build/debian/rules +++ b/build/debian/rules @@ -1,17 +1,15 @@ #!/usr/bin/make -f - -# Exclude git directory -export DH_ALWAYS_EXCLUDE=GIT:ecagent/.git - +DH_VERBOSE = 1 +export DH_OPTIONS=-v +# This file was automatically generated by stdeb 0.8.2 at +# Thu, 30 Jul 2015 08:59:04 +0000 +export PYBUILD_NAME=ecmanaged-ecagent %: - dh $@ + dh $@ --with python2 --buildsystem=pybuild + +override_dh_auto_install: + python setup.py install --root=debian/ecmanaged-ecagent --install-layout=deb --no-compile -override_dh_installinit: - dh_installinit --init-script=ecagentd -- "start 90 2 3 4 5 . stop 10 0 1 6 ." +override_dh_auto_build: -override_dh_fixperms: - dh_fixperms - chmod 0700 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config - chmod 0400 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent/config/ecagent.cfg - chmod 0755 debian/ecmanaged-ecagent/opt/ecmanaged/ecagent diff --git a/build/redhat/ecmanaged-ecagent.spec b/build/redhat/ecmanaged-ecagent.spec index 9024a3a..74d0644 100644 --- a/build/redhat/ecmanaged-ecagent.spec +++ b/build/redhat/ecmanaged-ecagent.spec @@ -18,7 +18,7 @@ Vendor: Juan Carlos Moreno Packager: Arindam Choudhury Provides: ecmanaged-ecagent #fedora -Requires: python2 python-devel python-twisted python-protocols python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils dbus-python sudo +Requires: python2 python-devel python-twisted python-protocols python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils sudo #centos Requires: python2 python-devel python-twisted-core python-twisted-web python-twisted-words python-configobj python-psutil libxml2-python python-simplejson rpm-python python-crypto python-httplib2 shadow-utils dbus-python sudo Url: www.ecmanaged.com diff --git a/config/ecagent.init.cfg b/config/ecagent.cfg.init similarity index 100% rename from config/ecagent.init.cfg rename to config/ecagent.cfg.init diff --git a/configure.py b/configure.py index 1ed0562..f105fc7 100755 --- a/configure.py +++ b/configure.py @@ -32,23 +32,23 @@ sys.path.append(".") configure_account = None -configure_groups = None +configure_tags = None try: - optlist, args = getopt.getopt(sys.argv[1:], 'a:s:', ["account=", "server-groups="]) + optlist, args = getopt.getopt(sys.argv[1:], 'a:t:', ["account=", "tags="]) except getopt.GetoptError: - print 'Please configure agent with ./configure --account=XXXXX' + print 'Please configure agent with ./configure.py --account=XXXXX --tags=XXX,XXX' sys.exit(-1) for option, value in optlist: if option in ("-a", "--account"): configure_account = value - elif option in ("-s", "--server-groups"): - configure_groups = value + elif option in ("-t", "--tags"): + configure_tags = value -if not configure_account and not configure_groups: - print 'Please configure agent with ./configure --account=XXXXX' +if not configure_account and not configure_tags: + print 'Please configure agent with ./configure.py --account=XXXXX --tags=XXX' sys.exit(-1) root_dir = os.path.dirname(os.path.realpath(__file__)) @@ -56,13 +56,12 @@ # Parse config file or end execution config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') -config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.init.cfg') - -# Is initial config (move init to cfg) -if not os.path.exists(config_file) and os.path.exists(config_file_init): - os.rename(config_file_init, config_file) +config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init') # manipulate configuration file +if not os.path.isfile(config_file): + config_file = config_file_init + if not os.path.isfile(config_file): print 'Unable to read the config file at %s' % config_file print 'Agent will now quit' @@ -73,8 +72,8 @@ if configure_account: config['XMPP']['account'] = configure_account -if configure_groups: - config['XMPP']['groups'] = configure_groups +if configure_tags: + config['XMPP']['tags'] = configure_tags config.write() print 'Manual configuration override succeeded.' diff --git a/cron.d/ecmanaged-ecagent b/cron.d/ecmanaged-ecagent index e24a3cd..4706770 100644 --- a/cron.d/ecmanaged-ecagent +++ b/cron.d/ecmanaged-ecagent @@ -1,4 +1,8 @@ SHELL=/bin/bash HOME=/opt/ecmanaged/ecagent/ -*/5 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 +# Start agent on reboot +@reboot root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 + +# Check agent every two minutes +*/2 * * * * root /opt/ecmanaged/ecagent/init check > /dev/null 2>&1 diff --git a/cron.d/ecmanaged-ecagent-init b/cron.d/ecmanaged-ecagent-init deleted file mode 100755 index f123cff..0000000 --- a/cron.d/ecmanaged-ecagent-init +++ /dev/null @@ -1 +0,0 @@ -*/5 * * * * root /etc/init.d/ecagentd check > /dev/null 2>&1 diff --git a/cron.d/ecmanaged-ecagent-systemd b/cron.d/ecmanaged-ecagent-systemd deleted file mode 100644 index 66f2950..0000000 --- a/cron.d/ecmanaged-ecagent-systemd +++ /dev/null @@ -1 +0,0 @@ -*/5 * * * root service ecagentd status || service ecagentd start > /dev/null 2>&1 diff --git a/dist_build/debian/changelog b/dist_build/debian/changelog deleted file mode 100644 index a47addb..0000000 --- a/dist_build/debian/changelog +++ /dev/null @@ -1,17 +0,0 @@ -ecmanaged-ecagent (3.0-1) UNRELEASED; urgency=medium - - * better system integration - - -- Arindam Choudhury Thu, 30 Jul 2015 09:06:14 +0000 - -ecmanaged-ecagent (2.2-2) unstable; urgency=low - - * Support for systemd - - -- Arindam Choudhury Thu, 30 Jul 2015 09:06:14 +0000 - -ecmanaged-ecagent (2.2-1) unstable; urgency=low - - * source package automatically created by stdeb 0.8.2 - - -- Arindam Choudhury Thu, 30 Jul 2015 08:59:04 +0000 \ No newline at end of file diff --git a/dist_build/debian/compat b/dist_build/debian/compat deleted file mode 100644 index f11c82a..0000000 --- a/dist_build/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 \ No newline at end of file diff --git a/dist_build/debian/control b/dist_build/debian/control deleted file mode 100644 index b6aeec7..0000000 --- a/dist_build/debian/control +++ /dev/null @@ -1,13 +0,0 @@ -Source: ecmanaged-ecagent -Maintainer: Arindam Choudhury -Section: admin -Priority: optional -Build-Depends: dh-python, python (>= 2.6.6-3), debhelper (>= 9) -Standards-Version: 3.9.6 -Homepage: http://www.ecmanaged.com - -Package: ecmanaged-ecagent -Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, lsb-base (>= 3.2-13), python, python-crypto, debconf, python-twisted-core, python-protocols, python-twisted-web, python-configobj, python-twisted-words, python-psutil, python-libxml2, python-simplejson, python-httplib2, sudo, python-dbus -Description: ECM Agent - ECMANAGED Monitor and deploy agent - Deploy and monitor agent for ECmanaged cloud management tool, based on XMPP protocol. diff --git a/dist_build/debian/copyright b/dist_build/debian/copyright deleted file mode 100644 index 70738bb..0000000 --- a/dist_build/debian/copyright +++ /dev/null @@ -1,45 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: ecmanaged-ecagent -Upstream-Contact: Juan Carlos Moreno -Source: http://www.ecmanaged.com - -Files: * -Copyright: 2012, ACKSTORM, S.L. All rights reserved. -License: GPL-3+ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or (at - your option) any later version. - . - In addition, as a special exception, the copyright holders give - permission to link the code of portions of this program with the - OpenSSL library under certain conditions as described in each - individual source file, and distribute linked combinations including - the two. - . - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the - file(s), but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. If you delete - this exception statement from all source files in the program, then - also delete it here. - . - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - . - You should have received a copy of the GNU General Public License - along with this program. If not, see . - . - On Debian GNU/Linux systems, the complete text of the GNU General - Public License (GPL) version 3 can be found at - /usr/share/common-licenses/GPL-3. - -Files: debian/* -Copyright: 2010-2012, ACK STORM, S.L. -License: GPL-3+ - On Debian GNU/Linux systems, the complete text of the GNU General - Public License (GPL) version 3 can be found at - /usr/share/common-licenses/GPL-3 \ No newline at end of file diff --git a/dist_build/debian/postinst b/dist_build/debian/postinst deleted file mode 100644 index 14f2c79..0000000 --- a/dist_build/debian/postinst +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -set -e - -if getent passwd ecmanaged >/dev/null 2>&1; then - chown -R ecmanaged:ecmanaged /opt/ecmanaged - mkdir -p /etc/ecmanaged - chown -R ecmanaged:ecmanaged /etc/ecmanaged -fi - -case "$1" in - configure) - if [ -f /opt/ecmanaged/ecagent/init ]; then - /opt/ecmanaged/ecagent/init start > /dev/null 2>&1 - fi -esac \ No newline at end of file diff --git a/dist_build/debian/rules b/dist_build/debian/rules deleted file mode 100644 index 3b90b0d..0000000 --- a/dist_build/debian/rules +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/make -f -DH_VERBOSE = 1 -export DH_OPTIONS=-v -# This file was automatically generated by stdeb 0.8.2 at -# Thu, 30 Jul 2015 08:59:04 +0000 -export PYBUILD_NAME=ecmanaged-ecagent -%: - dh $@ --with python2 --buildsystem=pybuild - -override_dh_auto_install: - python setup.py install --root=debian/ecmanaged-ecagent --install-layout=deb --no-compile - -override_dh_auto_build: - - diff --git a/ecagent/agent.py b/ecagent/agent.py index 2e355ed..5a0b3bf 100644 --- a/ecagent/agent.py +++ b/ecagent/agent.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import resource from time import time # Twisted imports @@ -28,7 +27,7 @@ import ecagent.twlogging as log from ecagent.verify import ECVerify -from ecagent.functions import mem_clean, mem_usage +from ecagent.functions import mem_clean from message import AGENT_VERSION_PROTOCOL @@ -38,6 +37,8 @@ _CHECK_RAM_MAX_RSS_MB = 125 _CHECK_RAM_INTERVAL = 60 +KEEPALIVED_TIMEOUT = 180 + class SMAgent: def __init__(self, config): reactor.callWhenRunning(self._check_config) @@ -78,65 +79,75 @@ def __init__(self, config): self.running_commands = {} self.num_running_commands = 0 + self.timestamp = 0 self.memory_checker = LoopingCall(self._check_memory, self.running_commands) self.memory_checker.start(_CHECK_RAM_INTERVAL) + + self.keepalive = LoopingCall(self._reconnect) log.debug("Loading XMPP...") Client.__init__( self, config['XMPP'], - [('/iq', self.__onIq), ], + [("/iq[@type='set']", self.__onIq), ], resource='ecm_agent-%d' % AGENT_VERSION_PROTOCOL) + + def _reconnect(self): + """ + Disconnect the current reactor to try to connect again + """ + log.info("No data received in %ss: Trying to reconnect" % KEEPALIVED_TIMEOUT) + reactor.disconnectAll() def __onIq(self, msg): """ A new IQ message has been received and we should process it. """ log.debug('__onIq') - mem_clean('__onIq [start]') log.debug("q Message received: \n%s" % msg.toXml()) log.debug("Message type: %s" % msg['type']) - if msg['type'] == 'set': - message = IqMessage(msg) - recv_command = message.command.replace('.', '_') + if self.keepalive.running: + log.debug("Stop keepalived") + self.keepalive.stop() + + log.debug("Starting keepalived") + self.keepalive.start(KEEPALIVED_TIMEOUT, now=False) - if recv_command in self.running_commands: - if time() > self.running_commands[recv_command]: - del self.running_commands[recv_command] - self.num_running_commands -= 1 - log.debug("Deleted %s from running_commands dict as should have been completed" % (recv_command)) + self.timestamp = time() - if recv_command not in self.running_commands: - log.debug('recieved new command: %s with message: %s' % (message.command, message)) + message = IqMessage(msg) + recv_command = message.command.replace('.', '_') - if hasattr(message, 'command') and hasattr(message, 'from_'): - log.debug('online contacts: %s' % self._online_contacts) + if recv_command in self.running_commands: + if self.timestamp > self.running_commands[recv_command]: + del self.running_commands[recv_command] + self.num_running_commands -= 1 + log.debug("Deleted %s from running_commands dict as should have been completed" % (recv_command)) - if message.from_ not in self._online_contacts: - log.warn('IQ sender not in roster (%s), dropping message' % message.from_) - else: - self.running_commands[recv_command] = time() + int(message.command_args['timeout']) - self.num_running_commands += 1 - log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) - self._processCommand(message) + if recv_command not in self.running_commands: + log.debug('recieved new command: %s with message: %s' % (message.command, message)) - else: - log.warn('Unknown ecm_message received: Full XML:\n%s' % (msg.toXml())) + if hasattr(message, 'command') and hasattr(message, 'from_'): + log.debug('online contacts: %s' % self._online_contacts) - del message - else: - log.debug("already running given command %s" %recv_command) - result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) - self._send(result, message) + if message.from_ not in self._online_contacts: + log.warn('IQ sender not in roster (%s), dropping message' % message.from_) + else: + self.running_commands[recv_command] = self.timestamp + int(message.command_args['timeout']) + self.num_running_commands += 1 + log.debug("Running commands: names: %s numbers: %i" % (self.running_commands, self.num_running_commands)) + self._processCommand(message) else: - log.warn('Unknown IQ type received: Full XML:\n%s' % (msg.toXml())) + log.debug("already running given command %s" %recv_command) + result = (_E_RUNNING_COMMAND, '', 'another command is running', 0) + self._send(result, message) del msg - mem_clean('__onIq [end]') + del message def _processCommand(self, message): if not self.verify.signature(message): @@ -147,7 +158,6 @@ def _processCommand(self, message): flush_callback = self._flush message.command_name = message.command.replace('.', '_') - mem_clean('run_command [start]') d = self.command_runner.run_command(message, flush_callback) # Clean message @@ -159,7 +169,6 @@ def _processCommand(self, message): errbackKeywords={'message': message}, ) del message - mem_clean('run_command [end]') return d else: @@ -168,11 +177,9 @@ def _processCommand(self, message): self._onCallFinished(result, message) del message - mem_clean('run_command [end]') return def _onCallFinished(self, result, message): - mem_clean('agent._onCallFinished') log.debug('Call Finished') self._send(result, message) del self.running_commands[message.command_name] @@ -196,18 +203,16 @@ def _flush(self, result, message): def _send(self, result, message): log.debug('Send Response') - mem_clean('agent._send') message.toResult(*result) del result - mem_clean('agent._send') self.send(message.toEtree()) def _check_memory(self, num_running_commands): - rss, vms = mem_usage() - log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) + rss = mem_clean('periodic memory clean') if not num_running_commands and rss > _CHECK_RAM_MAX_RSS_MB: log.critical("Max allowed RSS memory exceeded: %s MB, exiting." % _CHECK_RAM_MAX_RSS_MB) - reactor.stop() \ No newline at end of file + reactor.stop() + del rss diff --git a/ecagent/client.py b/ecagent/client.py index 9bc71d8..a7b9868 100644 --- a/ecagent/client.py +++ b/ecagent/client.py @@ -14,8 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import logging as log - +import ecagent.twlogging as log # Twisted imports from twisted.internet.defer import DeferredSemaphore from twisted.words.xish.domish import Element @@ -37,7 +36,7 @@ def __init__(self, config, observers, resource='XMPPClient'): """ my_observers = [ ('/presence', self._onPresence), - ('/iq', self._onPossibleErrorIq), + ("/iq[@type='error']", self._onPossibleErrorIq), ] my_observers.extend(observers) @@ -63,15 +62,14 @@ def __init__(self, config, observers, resource='XMPPClient'): ) def _onPossibleErrorIq(self, elem): - if elem['type'] == "error": - sender = elem['from'] - for el in elem.elements(): - if el.name == 'error' and el['code'] == '404': - log.warn('Received a 404 code from the server, setting the target user as offline') - if sender in self._online_contacts: - self._online_contacts.remove(sender) - else: - log.debug('Received a 404 from %s which not (anymore?) in the online contacts list.') + sender = elem['from'] + for el in elem.elements(): + if el.name == 'error' and el['code'] == '404': + log.warn('Received a 404 code from the server, setting the target user as offline') + if sender in self._online_contacts: + self._online_contacts.remove(sender) + else: + log.debug('Received a 404 from %s which not (anymore?) in the online contacts list.') def _onPresence(self, elem): """ @@ -84,6 +82,7 @@ def _onPresence(self, elem): log.debug("%s is now available" % presence.sender) #Store full jid. self._online_contacts.add(presence.sender) + else: log.debug("%s is not available anymore" % presence.sender) if presence.jid in self._online_contacts: diff --git a/ecagent/config.py b/ecagent/config.py index 2051c0c..d202662 100644 --- a/ecagent/config.py +++ b/ecagent/config.py @@ -110,6 +110,7 @@ def _get_config2(self, unique_id): address = self._get_ip() uuid = self._get_stored_uuid() account = self.get_stored_account() + agent_groups = self.get_stored_agent_groups params = "?uuid=%s&ipaddress=%s&hostname=%s&unique_id=%s&account=%s" \ % (uuid, address, hostname, unique_id, account) @@ -148,6 +149,10 @@ def _get_stored_unique_id(self): def get_stored_account(self): return self['XMPP'].get('account', '') + + def get_stored_agent_groups(self): + return self['XMPP'].get('agent_groups', '') + def parse_meta_data(self, json_data): meta_data = None try: @@ -217,3 +222,5 @@ def _get_unique_id(): unique_id = 'mac::' + ':'.join(findall('..', '%012x' % getnode())) return unique_id + + diff --git a/ecagent/core.py b/ecagent/core.py index 352ed5f..6d3969b 100644 --- a/ecagent/core.py +++ b/ecagent/core.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -KEEPALIVED_TIMEOUT = 60 MAX_FAILED_LOGINS = 5 XMPP_PORT = 5222 @@ -24,13 +23,11 @@ from twisted.words.protocols.jabber import client, jid, xmlstream from twisted.words.xish.domish import Element from twisted.internet import reactor -from twisted.internet.task import LoopingCall from twisted.words.protocols.jabber.xmlstream import STREAM_END_EVENT from twisted.words.protocols.jabber.client import IQ # Local import twlogging as log -from functions import mem_clean # Add registerAccount to XMPPAuthenticator @@ -84,8 +81,6 @@ def __init__(self, user, password, host, observers, self.failed_count = 0 self._xs = None - self._keep_alive_lc = None - self._user = user self._password = password self._host = host @@ -131,7 +126,7 @@ def _failed_auth(self, error): def _stream_end(self, error): """ overwrite in derivated class """ - log.info("XMPPClient stream end: %s" % error) + log.info("XMPPClient stream end") def _connected(self, xml_stream): log.info("XMPPClient connected") @@ -145,13 +140,6 @@ def _authd(self, xml_stream): """ log.info("XMPPClient authenticated") - #Keepalive: Send a newline every 60 seconds - #to avoid server disconnect - self._keep_alive_lc = LoopingCall(self._xs.send, '\n') - self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) - self._xs.addObserver(STREAM_END_EVENT, - lambda _: self._keep_alive_lc.stop()) - for message, callable in self._observers: self._xs.addObserver(message, callable) @@ -162,17 +150,8 @@ def _newid(self): return str(int(random() * (10 ** 31))) def send(self, elem): - mem_clean('core.send [start]') - if not elem.getAttribute('id'): log.debug('No message ID in message, creating one') elem['id'] = self._newid() self._xs.send(elem.toXml()) - - #Reset keepalive looping call timer - if self._keep_alive_lc.running: - self._keep_alive_lc.stop() - self._keep_alive_lc.start(KEEPALIVED_TIMEOUT) - - mem_clean('core.send [stop]') diff --git a/ecagent/functions.py b/ecagent/functions.py index c401d01..91f238f 100644 --- a/ecagent/functions.py +++ b/ecagent/functions.py @@ -36,20 +36,18 @@ def mem_usage(): vms /= 1000000.0 except: pass + log.info("Current Memory usage: rss=%sMB | vms=%sMB" % (rss, vms)) return rss, vms -def mem_clean(where='', dolog=False): +def mem_clean(where=''): _collect = collect() rss, vms = mem_usage() string = "_mem_clean: %s collected %d objects. (current mem: rss=%sMB | vms=%sMB)" % (where, _collect, rss, vms) - if dolog: - log.info(string) + log.debug(string) - else: - log.debug(string) - - del _collect, where, dolog, rss, vms, string + del _collect, where, vms, string + return rss \ No newline at end of file diff --git a/ecagent/message.py b/ecagent/message.py index a578cb1..99e3aaa 100644 --- a/ecagent/message.py +++ b/ecagent/message.py @@ -23,7 +23,6 @@ # Local import ecagent.twlogging as log -from ecagent.functions import mem_clean AGENT_VERSION_CORE = 3 AGENT_VERSION_PROTOCOL = 1 @@ -121,7 +120,6 @@ def toEtree(self): return msg def toXml(self): - mem_clean('toXml [start]') return self.toXml() def toResult(self, retvalue, stdout, stderr, timed_out, partial=0): diff --git a/ecagent/runner.py b/ecagent/runner.py index 3d25c50..13c92f6 100644 --- a/ecagent/runner.py +++ b/ecagent/runner.py @@ -33,7 +33,6 @@ from twisted.internet.error import ProcessTerminated, ProcessDone import ecagent.twlogging as log -from ecagent.functions import mem_clean class CommandRunner(): @@ -66,10 +65,6 @@ def __init__(self, config): self._commands = {} reactor.callWhenRunning(self._load_commands) - def __del__(self): - log.debug("__del__ CommandRunner()") - mem_clean("__del__: CommandRunner") - def _load_commands(self): for path in self.command_paths: log.debug("Processing dir: %s" % path) @@ -78,7 +73,8 @@ def _load_commands(self): for filename in os.listdir(path): if not filename.startswith('plugin_'): continue - if os.path.splitext(filename)[1] != '.py': + + if os.path.splitext(filename)[1] not in ['.py','.exe']: continue log.debug(" Queuing plugin %s for process." % filename) @@ -112,15 +108,16 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None need_sudo = ['plugin_pip.py', 'plugin_service.py', 'plugin_update.py', 'plugin_haproxy.py', 'plugin_monitor.py', 'plugin_pip_extra.py', 'plugin_puppet.py', 'plugin_saltstack.py', 'plugin_proc.py'] ext = os.path.splitext(filename)[1] if ext == '.py': - if os.path.split(filename)[1] in need_sudo: - command = 'sudo' - # -u: sets unbuffered output - args = [command, self._python_runner, '-u', '-W ignore::DeprecationWarning', filename, command_name] - else: + from sys import platform + if platform.startswith("win32") or os.path.split(filename)[1] not in need_sudo: command = self._python_runner - # -u: sets unbuffered output args = [command, '-u', '-W ignore::DeprecationWarning', filename, command_name] + else: + command = 'sudo' + # -u: sets unbuffered output + args = [command, self._python_runner, '-u', '-W ignore::DeprecationWarning', filename, command_name] + else: command = filename args = [command, command_name] @@ -134,7 +131,6 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None else: log.info("[INIT] Loading commands from %s" % filename) - mem_clean('spawnProcess [start]') crp = CommandRunnerProcess(cmd_timeout, command_args, flush_callback, message) d = crp.getDeferredResult() reactor.spawnProcess(crp, command, args, env=self.env) @@ -142,7 +138,6 @@ def _run_process(self, filename, command_name, command_args, flush_callback=None del cmd_timeout, filename, command_name, command_args del flush_callback, message, args - mem_clean('spawnProcess [end]') return d diff --git a/ecagentd.service b/ecagentd.service deleted file mode 100644 index ed21258..0000000 --- a/ecagentd.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=ECManaged Agent for monitoring and deployment - -[Service] -Type=simple -User=ecmanaged -Group=ecmanaged -PIDFile=/opt/ecmanaged/ecagent/twistd.pid -#ExecStart=/usr/bin/twistd -y /opt/ecmanaged/ecagent/ecagentd.tac -ExecStart=\ - /usr/bin/twistd \ - --nodaemon \ - --pidfile=/opt/ecmanaged/ecagent/twistd.pid \ - --no_save \ - --python=/opt/ecmanaged/ecagent/ecagentd.tac - -WorkingDirectory=/opt/ecmanaged/ecagent -Restart=always -RestartSec=300 - -[Install] -WantedBy=multi-user.target diff --git a/ecagentd.tac b/ecagentd.tac index bb0ef0f..73f1095 100644 --- a/ecagentd.tac +++ b/ecagentd.tac @@ -19,12 +19,17 @@ import os import gc import sys import stat -import random # Twisted from twisted.application.service import Application -root_dir = os.path.dirname(os.path.realpath(__file__)) + +try: + root_dir = os.path.dirname(os.path.abspath(__file__)) +except NameError: # We are the main py2exe script, not a module + import sys + root_dir = os.path.dirname(os.path.abspath(sys.argv[0])) + os.chdir(root_dir) if root_dir not in sys.path: @@ -59,12 +64,20 @@ open(pid_file, 'w').write(str(os.getpid())) # Start agent and setup logging config_file = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg') -config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.init.cfg') +config_file_init = os.path.join(os.path.sep, root_dir, 'config', 'ecagent.cfg.init') + +# rename config/ecagent.cfg.init to config/ecagent.cfg for fresh install -# Is initial config (move init to cfg) -if not os.path.exists(config_file) and os.path.exists(config_file_init): +if os.path.exists(config_file) and os.path.exists(config_file_init): + os.remove(config_file_init) + +if os.path.exists(config_file_init) and not os.path.exists(config_file): os.rename(config_file_init, config_file) +if not os.path.isfile (config_file): + print "Failed to find config file." + sys.exit(-1) + os.chmod(config_file, stat.S_IRWXU) if not os.path.isfile(config_file): diff --git a/init b/init index 4c9fd99..d0f28ce 100755 --- a/init +++ b/init @@ -2,7 +2,7 @@ NAME=${NAME:-ecagentd} TWISTD="$(which twistd)" -HOME=${DIR:-/opt/ecmanaged/ecagent} +HOME=${DIR:-$(pwd)} PID_FILE=${PID_FILE:-"$HOME/twistd.pid"} TAC=${TAC:-"$HOME/$NAME.tac"} RETVAL=0 diff --git a/monitor/mplugin/__base__/__base__.py b/monitor/mplugin/__base__/__base__.py index 9617f42..0685454 100755 --- a/monitor/mplugin/__base__/__base__.py +++ b/monitor/mplugin/__base__/__base__.py @@ -19,10 +19,14 @@ # Base monitor for agent please do not modify it import psutil +import os +import sys -from os import getloadavg, getpid +from os import getpid from time import time, sleep +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'plugins')) + from __mplugin import MPlugin from __mplugin import OK, CRITICAL @@ -50,6 +54,7 @@ def run(self): 'net': self._get_network(), 'netstat': self._get_netstat(), 'disk_io': self._get_disk_io(), + 'inodes' : self._get_inodes(), 'cputimes': self._get_cpu_times(), 'process': self._get_processes(), 'swap': self._get_swap(), @@ -130,7 +135,7 @@ def _get_disk(self): if hasattr(usage, 'free'): tmp['free'] = self.to_gb(usage.free) if hasattr(usage, 'percent'): - tmp['percent'] = usage.percent + tmp['percent'] = (float(usage.total - usage.free) / float(usage.total)) * 100 retval[part.mountpoint] = tmp @@ -143,6 +148,7 @@ def _get_disk(self): return retval def _get_disk_io(self): + retval = {} from os.path import basename try: @@ -160,7 +166,6 @@ def _get_disk_io(self): continue res[device] = disk_io[device] - retval = {} retval = self.counters(self._to_data(res), 'disk_io') except: pass @@ -230,7 +235,24 @@ def _get_network(self): pass return retval - + + def _get_inodes(self): + if self.is_win(): + return [] + + inode_list = [] + for part in psutil.disk_partitions(all=False): + try: + data = statvfs(part.mountpoint) + iused = data.f_files - data.f_ffree + iused_p = int(iused * 100 / data.f_files) + inode_list.append({'Filesystem': part.device, 'Inodes': data.f_files, 'IUsed': iused, + 'IFree': data.f_ffree, 'IUse%': iused_p, 'Mounted on': part.mountpoint}) + except: + pass + + return inode_list + def _get_netstat(self): try: # Only psutil > v2 @@ -420,14 +442,17 @@ def _boot_time_windows(): except: return - @staticmethod - def _get_load(): - m1, m5, m15 = getloadavg() - return { - '1m': m1, - '5m': m5, - '15m': m15 - } + def _get_load(self): + if not self.is_win(): + from os import getloadavg + m1, m5, m15 = getloadavg() + return { + '1m': m1, + '5m': m5, + '15m': m15 + } + + return 0 @staticmethod def _to_dict(obj): diff --git a/plugins/__ecmhaproxy.py b/plugins/__ecmhaproxy.py deleted file mode 100644 index 9c5b80d..0000000 --- a/plugins/__ecmhaproxy.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Copyright (C) 2013 - Aybuke Ozdemir - -this file is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -this file is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see - -""" -# -*- coding: utf-8 -*- - -import warnings -from os import path -from tempfile import mktemp -import socket -import csv - -import simplejson as json -from __haproxy import HAProxyConfig, Option - -NAME = 'haproxy' -RECV_SIZE = 1024 -DEFAULT_SOCKET = '/tmp/haproxy' -HAPROXY_BIN = '/usr/sbin/haproxy' - -TEMPLATE_FILE = '/etc/haproxy/haproxy.tpl' -TEMPLATE_CONFIG = """ -global - maxconn 4096 - user haproxy - group haproxy - daemon - -defaults - retries 3 - option redispatch - option http-server-close - option forwardfor if-none - timeout connect 5000 - timeout client 50000 - timeout server 50000 - timeout check 5000 - balance leastconn -""" - - -class ECMHAConfig: - def __init__(self, ecm_hash, template=TEMPLATE_FILE, as_json=False): - if as_json: - ecm_hash = json.loads(ecm_hash) - - if not self._check(ecm_hash): - raise Exception('Invalid hash received') - - if not path.isfile(template): - warnings.warn("Template file not found, using default config") - template = mktemp() - f = open(template, 'w') - f.write(TEMPLATE_CONFIG) - f.close() - - self.template = template - self.ecm_hash = ecm_hash - self.config = HAProxyConfig(template) - self._loads() - - def show(self, as_json=False): - config_to_show = self.config.createHash() - if as_json: - config_to_show = json.dumps(config_to_show) - print config_to_show - - def read(self, read_file, as_json=True): - f = open(read_file, 'r') - first_line = f.readline() - f.close() - - if first_line.startswith('#'): - json_config = first_line.split('#')[1] - - if not as_json: - json_config = json.loads(json_config) - return json_config - - return False - - def valid(self): - config = self.config.getConfig() - return is_valid(config) - - def write(self, write_file): - if write_file == self.template: - raise Exception("Can't use template as final file") - - config = self.config.getConfig() - f = open(write_file, 'w') - f.write("#" + json.dumps(self.ecm_hash) + "\n") - f.write(config) - f.close() - - def _check(self, ecm_hash): - if ecm_hash.get('health', None) is None: - return False - if ecm_hash.get('listeners', None) is None: - return False - if ecm_hash.get('backends', None) is None: - return False - - return True - - # private functions - - def _loads(self): - health_timeout = self.ecm_hash['health']['timeout'] - health_threshold = self.ecm_hash['health']['threshold'] - health_interval = self.ecm_hash['health']['threshold'] - health_check = self.ecm_hash['health']['check'] - - listeners = False - - # Get check data - (hproto, hport, hpath) = health_check.split('::') - - for listener in self.ecm_hash['listeners']: - listeners = True - (fproto, fport, bproto, bport) = listener.split('::') - self.config.addListen(name="front-%s" % fport, port=fport, mode=self._protocol_dic(fproto)) - listen = self.config.getListen("front-%s" % fport) - - # Add check timeout configuration - listen.addOption(Option('timeout', ('check %i' % (health_timeout*1000),))) - - # Add cookie only if there are backend (config crashes otherwise) - if len(self.ecm_hash['backends']) and self._protocol_dic(fproto) == 'http': - listen.addOption(Option('cookie', ('ECM-HA-ID insert indirect nocache maxidle 30m maxlife 8h',))) - - if hpath: - check = 'httpchk HEAD %s' % hpath - listen.addOption(Option('option', (check,))) - - for backend in self.ecm_hash['backends']: - uuid = backend.keys()[0] - server = '%s %s:%s check inter %i rise %i fall %i' % (uuid, backend[uuid], fport, health_interval*1000, - health_threshold, health_threshold) - if self._protocol_dic(fproto) == 'http': - server += ' cookie %s' % uuid - listen.addOption(Option('server', (server,))) - - if not listeners: - # Add default listener - self.config.addListen(name="empty", port=80, mode='http') - - @staticmethod - def _protocol_dic(proto): - retval = 'tcp' - if proto.lower() == 'http': - retval = 'http' - - return retval - - -class ECMHASocket(object): - def __init__(self, socket_file=DEFAULT_SOCKET): - self.socket_file = socket_file - - def connect(self): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.connect(self.socket_file) - return s - - def communicate(self, command): - """ Send a single command to the socket and return a single response (raw string) """ - s = self.connect() - if not command.endswith('\n'): command += '\n' - s.send(command) - result = '' - buf = s.recv(RECV_SIZE) - while buf: - result += buf - buf = s.recv(RECV_SIZE) - s.close() - return result - - def get_server_info(self): - result = {} - output = self.communicate('show info') - for line in output.splitlines(): - try: - key, val = line.split(':') - except ValueError, e: - continue - result[key.strip()] = val.strip() - return result - - def get_server_stats(self): - output = self.communicate('show stat -1 4 -1') - #sanitize and make a list of lines - output = output.lstrip('# ').strip() - output = [l.strip(',') for l in output.splitlines()] - csvreader = csv.DictReader(output) - result = [d.copy() for d in csvreader] - return result - - -def is_valid(config): - import subprocess - - tmp_file = mktemp() - f = open(tmp_file, 'w') - f.write(config) - f.close() - - p = subprocess.Popen([HAPROXY_BIN, "-c", "-f", tmp_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if p.returncode == 0: - return True - - warnings.warn("Invalid configuration file: %s" % err) - return False - diff --git a/plugins/__haproxy.py b/plugins/__haproxy.py deleted file mode 100644 index 6c2b014..0000000 --- a/plugins/__haproxy.py +++ /dev/null @@ -1,490 +0,0 @@ -""" -Copyright (C) 2013 - Aybuke Ozdemir - -This file is part of python-haproxy-tools - -python-haproxy-tools is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -python-haproxy-tools is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see - -""" -# -*- coding: utf-8 -*- - -SECTIONS = ['global', 'defaults', 'listen', 'frontend', 'backend'] - - -class HAProxyConfig(): - def __init__(self, config_path): - self.config_path = config_path - self.config = self.__readConfig() - - # Set globals section. - gs = self.__getSection('global') - if gs: - self.globalh = Global(gs) - else: - self.globalh = None - - # Set defaults section. - ds = self.__getSection('defaults') - if ds: - self.defaults = Defaults(ds) - else: - self.defaults = None - - #Set Listen. - self.listens = [] - for name in self.__getSectionNames('listen'): - l = Listen(self.__getSectionWithName('listen', name)) - self.listens.append(l) - - # Set Frontends. - self.frontends = [] - for name in self.__getSectionNames('frontend'): - f = Frontend(self.__getSectionWithName('frontend', name)) - self.frontends.append(f) - - #Set Backend. - self.backends = [] - for name in self.__getSectionNames('backend'): - b = Backend(self.__getSectionWithName('backend', name)) - self.backends.append(b) - - def getConfig(self): - out = "# This file created by python-haproxy-tools\n" - if self.globalh: - out += self.globalh.getConfig() - if self.defaults: - out += self.defaults.getConfig() - for l in self.listens: - out += l.getConfig() - for f in self.frontends: - out += f.getConfig() - for b in self.backends: - out += b.getConfig() - return out - - def createHash(self): - hash_config = {'global': [], 'defaults': [], 'backend': {}, 'frontend': {}, 'listen': {}} - - b = self.getGlobal() - for a in b.options: - hash_config['global'].append({a.name: ' '.join(a.params)}) - - b = self.getDefaults() - for a in b.options: - hash_config['defaults'].append({a.name: ' '.join(a.params)}) - - for b in self.getBackends(): - hash_config['backend'][b.name] = [] - for a in b.options: - hash_config['backend'][b.name].append({a.name: ' '.join(a.params)}) - - for b in self.getFrontends(): - hash_config['frontend'][b.name] = [] - for a in b.options: - hash_config['frontend'][b.name].append({a.name: ' '.join(a.params)}) - - for b in self.getListens(): - hash_config['listen'][b.name] = [] - for a in b.options: - hash_config['listen'][b.name].append({a.name: ' '.join(a.params)}) - - return hash_config - - def writeConfig(self,ha_file): - hash_config = self.createHash() - f = open(ha_file, 'w') - for key in SECTIONS: - if not hash_config.get(key): - continue - - if key in ['global','defaults']: - f.write("%s\n" %key) - - for param in hash_config[key]: - if isinstance(param, dict): - f.write(" %s %s\n" % (param.keys()[0], param[param.keys()[0]])) - - else: - f.write("%s %s\n" % (key,param)) - for value in hash_config[key][param]: - f.write(" %s %s\n" % (value.keys()[0], value[value.keys()[0]])) - f.write("\n") - - f.write("\n") - f.close() - - def getGlobal(self): - return self.globalh - - def getDefaults(self): - return self.defaults - - def getFrontend(self, name): - name = name.strip() - for fe in self.frontends: - if fe.description.name == name: - return fe - - def getBackend(self, name): - name = name.strip() - for be in self.backends: - if be.description.name == name: - return be - - def getListen(self, name): - name = name.strip() - for l in self.listens: - if l.description.name == name: - return l - - def getFnames(self): - fa = [] - for f in self.frontends: - fa.append(f.name) - return fa - - def getLnames(self): - la = [] - for l in self.listens: - la.append(l.name) - return la - - def getBnames(self): - ba = [] - for b in self.backends: - ba.append(b.name) - return ba - - def addListen(self, name, ip=None, port=None, mode=None): - ca = [] - ca.append('listen %s' %name) - coption = "" - if ip: - cip = ip - else: - cip = "*" - if port: - cport = port - else: - cport = "80" - coption = "bind %s:%s" %(cip,cport) - ca.append(coption) - if mode: - ca.append('mode %s' %mode) - - listen = Listen(ca) - self.listens.append(listen) - - def addFrontend(self, name, ip=None, port=None, mode=None): - ca = [] - ca.append('frontend %s' %name) - if ip: - cip = ip - else: - cip = ip - if port: - cport = port - else: - cport = "80" - coption = "bind %s:%s" %(cip, cport) - ca.append(coption) - if mode: - ca.append('mode:%s' %mode) - - frontend = Frontend(ca) - self.frontends.append(frontend) - - def addBackend(self, name, balance): - ca = [] - ca.append('backend %s' %name) - coption = "balance %s" %balance - ca.append(coption) - backend = Backend(ca) - self.backends.append(backend) - - def addGlobal(self, option, value): - goption = Option(option, value) - self.globalh.addOption(goption) - - def addDefaults(self, option, value): - doption = Option(option, value) - self.defaults.addOption(doption) - - def delBackend(self, name): - for be in self.backends: - if be.description.name == name: - self.backends.remove(be) - - def delFrontend(self, name): - for fe in self.frontends: - if fe.description.name == name: - self.frontends.remove(fe) - - def delListen(self, name): - for l in self.listens: - if l.description.name == name: - self.listens.remove(l) - - def getFrontends(self): - return self.frontends - - def getBackends(self): - return self.backends - - def getListens(self): - return self.listens - - def __getSectionNames(self, title): - l_names = [] - for row in self.config: - row = row.strip() - if row.startswith(title): - name = row.split()[1] - l_names.append(name) - return l_names - - def __getSectionWithName(self, title, name): - config_array = [] - title = title.strip() - start_flag = False - - for line in self.config: - line = line.strip() - - if line == '': - continue - - line_array = line.split() - ltitle = line_array[0].strip() - - if len(line_array) > 1: - lname = line_array[1].strip() - - if ltitle == title.strip(): - if name == lname: - config_array.append(line) - start_flag = True - continue - - if ltitle in SECTIONS: - start_flag = False - - if start_flag: - config_array.append(line) - return config_array - - def __getSection(self, title): - config_array = [] - title = title.strip() - start_flag = False - - for line in self.config: - line = line.strip() - - if line == '': - continue - - ltitle = line.split()[0].strip() - if ltitle == title: - config_array.append(line) - start_flag = True - continue - - if ltitle in SECTIONS: - start_flag = False - - if start_flag: - config_array.append(line) - if config_array: - return config_array - else: - return None - - def __readConfig(self): - config_file = open(self.config_path) - config = config_file.readlines() - config_file.close() - return config - - -class Option(): - def __init__(self, param_name, params): - self.name = param_name - self.params = params - - def getRow(self): - return self.__repr__() - - def getParamName(self): - return self.name - - def getParams(self): - return self.params - - def getConfig(self): - return "%s %s" %(self.name, " ".join(self.params)) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.params))) - - -class Section(): - def __init__(self, config_array): - self.config_array = config_array - self.options = [] - self.description = None - - is_first_row = True - for row in self.config_array: - if is_first_row: - is_first_row = False - items = row.split() - title = items[0] - - if len(items) > 1: - name = items[1] - else: - name = None - if len(items) > 2: - params = items[2:] - else: - params = None - - self.description = Description(title, name) - - if params: - for p in params: - o = Option('bind', tuple([p])) - self.options.append(o) - - continue - - param_name = self.getParamName(row).strip() - params = self.getParams(row) - option = Option(param_name, params) - self.options.append(option) - - def getParamName(self, row): - return row.split()[0] - - def getParams(self, row): - return tuple(row.split()[1:]) - - def getConfig(self): - opts = self.options - des = self.description - config_output = "" - config_output += des.getConfig() + '\n' - for opt in opts: - config_output += ' ' + opt.getConfig() + '\n' - config_output += '\n' - return config_output - - def addOption(self, option): - self.options.append(option) - return True - - def delOption(self, option): - for opt in self.options: - if opt == option.name: - self.options.remove(opt) - return True - - def setOption(self, option): - for opt in self.options: - if opt.name == option.name: - opt.params = option.params - return self.options - - -class Description(): - def __init__(self, title, name=None): - self.title = title - self.name = name - - def getConfig(self): - out = [] - out.append(self.title) - if self.name: - out.append(self.name) - return " ".join(out) - - -class Global(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.options))) - - -class Defaults(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.options))) - - -class Listen(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - -class Frontend(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - -class Backend(Section): - def __init__(self, config_array): - Section.__init__(self, config_array) - self.name = self.description.name - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return (('' - % (self.name, self.options))) - - diff --git a/plugins/__helper.py b/plugins/__helper.py index 904462f..b46c0cc 100644 --- a/plugins/__helper.py +++ b/plugins/__helper.py @@ -45,16 +45,26 @@ _FLUSH_WORKER_SLEEP_TIME = 0.2 -AGENT_VERSION = 3.0 +AGENT_VERSION = 4.0 def is_win(): - """ Returns True if is a windows system - """ - if platform.startswith("win32"): - return True + """ Returns True if is a windows system + """ - return False + if platform.startswith("win32"): + return True + + return False + + +def is_linux(): + """ Returns True if is a windows system + """ + if platform.startswith("linux"): + return True + + return False def file_write(file_path, content=None): """ Writes a file @@ -100,6 +110,7 @@ def clean_stdout(std_output): try: r = re.compile("\033\[[0-9;]*m", re.MULTILINE) return r.sub('', std_output) + except: return std_output @@ -138,7 +149,7 @@ def download_file(url, filename=None, user=None, passwd=None): if _header_filename: filename = path.join(path.dirname(filename), _header_filename) except: - pass + pass with open(filename, 'wb') as fp: while True: @@ -152,7 +163,7 @@ def download_file(url, filename=None, user=None, passwd=None): return filename -def get_url(url, timeout=10): +def get_url(url, timeout=30): socket.setdefaulttimeout(timeout) urlopen = urllib.urlopen(url) @@ -254,7 +265,8 @@ def install_package(packages, update=True): envars['DEBIAN_FRONTEND'] = 'noninteractive' if update: - run_command(['apt-get', '-y', '-qq', 'update']) + run_command(['apt-get', '-y', '-qq', 'update'], runas='root') + command = ['apt-get', '-o', 'Dpkg::Options::=--force-confold', @@ -267,7 +279,8 @@ def install_package(packages, update=True): elif distribution.lower() in ['centos', 'redhat', 'fedora', 'amazon']: if update: - run_command(['yum', '-y', 'clean', 'all']) + run_command(['yum', '-y', 'clean', 'all'], runas='root') + command = ['yum', '-y', '--nogpgcheck', @@ -283,7 +296,7 @@ def install_package(packages, update=True): elif distribution.lower() in ['arch']: if update: - run_command(['pacman', '-S', '--noconfirm', 'pacman']) + run_command(['pacman', '-S', '--noconfirm', 'pacman'], runas='root') command = ['pacman', '-S', '--noconfirm', @@ -292,7 +305,7 @@ def install_package(packages, update=True): else: return 1, '', "Distribution not supported: %s" % distribution - return run_command(command, envars=envars) + return run_command(command, envars=envars, runas='root') except Exception as e: return 1, '', "Error installing packages %s" % e @@ -575,12 +588,12 @@ def check_sudo(): return False p = Popen( - [which('sudo'),'-n','id'], - bufsize=0, - stdout=PIPE, - stderr=PIPE, - universal_newlines=True, - close_fds=(os.name == 'posix') + [which('sudo'), '-n', 'id'], + bufsize=0, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + close_fds=(os.name == 'posix') ) retval = p.wait() @@ -645,7 +658,7 @@ def __init__(self): self.thread_stderr = '' self.thread_run = 1 - def command(self, command, args=None, std_input=None, run_as=None, working_dir=None, envars=None, only_stdout = False): + def command(self, command, args=None, std_input=None, run_as=None, working_dir=None, envars=None, only_stdout=False): """ Execute command and flush stdout/stderr using threads """ @@ -676,15 +689,10 @@ def command(self, command, args=None, std_input=None, run_as=None, working_dir=N if run_as and not is_win(): if not check_sudo(): - return 255, '', 'Sudo is not available:' - # don't use su - xxx or env variables will not be available - if run_as == _ROOT: - command = [which('sudo'), ' '.join(map(str, command))] + return 255, '', 'sudo is not available:' - else: - command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))] - - # :TODO: Run_as for windows :S + # don't use su - xxx or env variables will not be available + command = [which('sudo'), 'su', run_as, '-c', ' '.join(map(str, command))] try: p = Popen( diff --git a/plugins/__mplugin.py b/plugins/__mplugin.py index f5c4606..7bf0492 100644 --- a/plugins/__mplugin.py +++ b/plugins/__mplugin.py @@ -53,8 +53,9 @@ class MPlugin: def __init__(self, plugin_path=None): # set alarm for timeout - signal.signal(signal.SIGALRM, _timeout) - signal.alarm(CHECK_TIMEOUT) + if not self.is_win(): + signal.signal(signal.SIGALRM, _timeout) + signal.alarm(CHECK_TIMEOUT) if plugin_path: self.path = abspath(plugin_path) @@ -67,7 +68,7 @@ def __init__(self, plugin_path=None): log.basicConfig( filename=join(self.path, LOG_FILE_NAME), format='%(levelname)s:%(message)s', - level=log.DEBUG + level=log.INFO ) # Read configuration diff --git a/plugins/plugin_configfile.py b/plugins/plugin_configfile.py deleted file mode 100644 index 8d5bb78..0000000 --- a/plugins/plugin_configfile.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding:utf-8 -*- - -RUN_AS_ROOT = False - -import os - -from base64 import b64decode -from shutil import move - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMConfigfile(ECMPlugin): - def cmd_configfile_run(self, *argv, **kwargs): - """ - Deploy a file - Syntax: configfile.run[configfile,file,chown_user,chown_group,chmod,rotate,command,runas] - """ - code_base64 = kwargs.get('configfile', None) - filename = kwargs.get('path', None) - chown_user = kwargs.get('chown_user', None) - chown_group = kwargs.get('chown_group', None) - chmod = kwargs.get('chmod', None) - rotate = kwargs.get('rotate', False) - - command = kwargs.get('command', None) - runas = kwargs.get('command_runas', None) - - if not code_base64 or not filename: - raise ecm.InvalidParameters(self.cmd_configfile_run.__doc__) - - ret = {'out': 0, 'stdout': '', 'stderr': ''} - try: - if rotate and os.path.isfile(filename): - new_file = filename + '_rotated_' + ecm.utime() - move(filename, new_file) - ret['stdout'] = ecm.output("Old configfile moved to '%s'" % new_file) - - # Write down file - ecm.file_write(filename, b64decode(code_base64)) - ret['stdout'] += ecm.output("Configfile created successfully at '%s'" % filename) - - except Exception as e: - raise Exception("Unable to write configfile: %s" % e) - - try: - # Chown to specified user/group - if chown_user and chown_group and os.path.isfile(filename): - ecm.chown(filename, chown_user, chown_group) - ret['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - # Chown to specified user/group - if chmod and os.path.isfile(filename): - ecm.chmod(filename, chmod) - ret['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - except Exception as e: - raise Exception("Unable to change permissions for configfile: %s" % e) - - if command: - working_dir = os.path.dirname(filename) - out, stdout, stderr = ecm.run_command(command, runas=runas, workdir=working_dir) - ret = ecm.format_output(out, stdout, stderr) - - return ret - - -ECMConfigfile().run() diff --git a/plugins/plugin_haproxy.py b/plugins/plugin_haproxy.py deleted file mode 100644 index 6ce5695..0000000 --- a/plugins/plugin_haproxy.py +++ /dev/null @@ -1,114 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -RUN_AS_ROOT = True - -import simplejson as json -from base64 import b64decode -from __ecmhaproxy import ECMHAConfig, ECMHASocket - -HAPROXY_CONFIG = '/etc/haproxy/haproxy.cfg' -HAPROXY_INIT = '/etc/init.d/haproxy' - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMHaproxy(ECMPlugin): - def cmd_haproxy_config_get(self, *argv, **kwargs): - """ - haproxy.config_get[] - """ - f = open(HAPROXY_CONFIG, 'r') - first_line = f.readline() - f.close() - - if first_line.startswith('#'): - config = first_line.split('#')[1].rstrip('\n') - status = self._status() - return { - 'config': json.loads(config), - 'status': status - } - - raise Exception("Unable to get config") - - def cmd_haproxy_config_set(self, *argv, **kwargs): - """ - haproxy.config_set[config=base64_balancer_config] - """ - config = kwargs.get('config', None) - if not config: - raise ecm.InvalidParameters(self.cmd_haproxy_config_set.__doc__) - - try: - config = b64decode(config) - except Exception: - raise ecm.InvalidParameters('Invalid base64 configuration') - - ha_config = ECMHAConfig(config, as_json=True) - if ha_config.valid(): - ha_config.write(HAPROXY_CONFIG) - self._restart() - - return self.cmd_haproxy_config_get() - - else: - raise Exception('Invalid configuration') - - def cmd_haproxy_stats(self, *argv, **kwargs): - """ - haproxy.stats[] - """ - try: - ha = ECMHASocket() - return { - 'stats': ha.get_server_stats(), - 'info': ha.get_server_info() - } - - except: - raise Exception("Unable to get config") - - - @staticmethod - def _status(): - try: - status_array = {} - ha = ECMHASocket() - for line in ha.get_server_stats(): - status_array.setdefault(line['pxname'], {})[line['svname']] = line['status'] - return status_array - - except: - raise Exception("Unable to get config") - - @staticmethod - def _restart(): - import subprocess - p = subprocess.Popen([HAPROXY_INIT, 'reload'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if p.returncode == 0: - return True - - raise Exception("Error restarting service: %s" % err) - - -ECMHaproxy().run() - diff --git a/plugins/plugin_monitor.py b/plugins/plugin_monitor.py index bdde257..840b6a4 100644 --- a/plugins/plugin_monitor.py +++ b/plugins/plugin_monitor.py @@ -19,6 +19,9 @@ import os import sys +from multiprocessing import Process +from multiprocessing import Pool + import simplejson as json from base64 import b64decode from time import time @@ -53,20 +56,24 @@ class ECMMonitor(ECMPlugin): - def cmd_monitor_get(self, *argv, **kwargs): """ Runs monitor commands from monitor path """ - + config = None b64_config = kwargs.get('config', None) - + + # Use Pool on windows systems + if ecm.is_win(): + thread_pool = Pool(20) + try: - config = json.loads(b64decode(b64_config)) + if isinstance(b64_config, str): + config = json.loads(b64decode(b64_config)) except: pass - + retval = [] to_execute = [] @@ -74,18 +81,18 @@ def cmd_monitor_get(self, *argv, **kwargs): self._check_path(MPLUGIN_PATH) self._check_path(CPLUGIN_PATH) self._check_path(CACHE_PATH) - + # Clean old cache files self._cache_clean() if not os.path.isdir(MPLUGIN_PATH): return retval - + # Foreach plugin inside mplugins and custom for plugin_path in [MPLUGIN_PATH, CPLUGIN_PATH]: if not os.path.isdir(plugin_path): continue - + for p_path in os.listdir(plugin_path): p_path = os.path.join(plugin_path, p_path) @@ -96,7 +103,7 @@ def cmd_monitor_get(self, *argv, **kwargs): # Skip disabled plugins if os.path.basename(p_path).startswith('.'): continue - + runas = None scripts = [] interval = COMMAND_INTERVAL @@ -104,30 +111,30 @@ def cmd_monitor_get(self, *argv, **kwargs): # Search for plugin files if plugin_path == MPLUGIN_PATH: mplugin = MPlugin(p_path) - + # Update config and re-read if config: mplugin.write_config(config.get(mplugin.id)) mplugin = MPlugin(p_path) - + runas = mplugin.data.get('runas', None) interval = mplugin.data.get('interval', COMMAND_INTERVAL) script = os.path.join(plugin_path, p_path, mplugin.id) - + if not os.path.exists(script): script += '.py' - + if not os.path.exists(script): continue # Add as valid mplugin script scripts = [script] - + # Executable plugin base if not os.access(script, os.X_OK): os.chmod(script, 0755) - + # Custom plugins path else: # Set default interval or read from path name @@ -137,47 +144,51 @@ def cmd_monitor_get(self, *argv, **kwargs): _tmp = os.path.join(plugin_path, p_path, filename) if os.access(_tmp, os.X_OK): scripts.append(_tmp) - + for script in scripts: # Read last result from cache (even if void) from_cache = self._cache_read(script, interval + CACHE_SOFT_TIME) retval.append(self._parse_script_name(script) + GLUE + str(interval) + GLUE + from_cache) - + # Execute script if cache wont be valid on next command_get execution - if not self._cache_read(script, interval - COMMAND_INTERVAL + CACHE_SOFT_TIME): + if not self._cache_read(script, interval - COMMAND_INTERVAL - CACHE_SOFT_TIME): to_execute.append({'script': script, 'runas': runas}) - - for data in to_execute: - _run_background_file(data['script'], data['runas']) + for data in to_execute: + if ecm.is_win(): + p = Process(target=_run_background_file, args=(data['script'], data['runas'],)) + p.start() + p.join() + else: + _run_background_file(data['script'], data['runas']) return retval - + def cmd_monitor_plugin_install(self, *argv, **kwargs): """ Installs a plugin [url=plugin_url] """ url = kwargs.get('url', None) content = None - + if not url: raise ecm.InvalidParameters(self.cmd_monitor_plugin_install.__doc__) - + try: content = ecm.get_url(url) except: - pass - + raise Exception("Unable to get URL: %s" % url) + if not content: raise Exception("Unable to get URL: %s" % url) - - try: + + try: plugin = json.loads(content) except: raise Exception("Invalid data received") id = plugin.get('id') runas = plugin.get('runas') - + arg_config = plugin.get('config') arg_script_b64 = plugin.get('script') @@ -214,7 +225,7 @@ def cmd_monitor_plugin_install(self, *argv, **kwargs): script = b64decode(arg_script_b64) except: pass - + config = { 'id': id, 'runas': runas, @@ -223,16 +234,16 @@ def cmd_monitor_plugin_install(self, *argv, **kwargs): 'version': plugin.get('version'), 'config': arg_config } - + if id and config and script: mplugin = MPlugin(MPLUGIN_PATH) if mplugin.install(id, config, script): # Installation ok, run it script_file = os.path.abspath(os.path.join(MPLUGIN_PATH, id, id)) _run_background_file(script_file, runas) - + return True - + return False def cmd_monitor_plugin_uninstall(self, *argv, **kwargs): @@ -241,20 +252,20 @@ def cmd_monitor_plugin_uninstall(self, *argv, **kwargs): """ plugin_id = kwargs.get('id', None) - + if not plugin_id: raise ecm.InvalidParameters(self.cmd_monitor_plugin_uninstall.__doc__) - + mplugin = MPlugin(MPLUGIN_PATH) return mplugin.uninstall(plugin_id) - + @staticmethod def _interval_from_path(my_path): interval = os.path.split(my_path)[-1] - + if not ecm.is_integer(interval): interval = COMMAND_INTERVAL - + return int(interval) @staticmethod @@ -279,13 +290,13 @@ def _cache_read(command, max_old): # check updated cache if os.path.isfile(cache_file): modified = os.path.getmtime(cache_file) - + how_old = int(time() - modified) - - if(how_old > max_old): + + if (how_old > max_old): # Invalid cache file os.remove(cache_file) - + else: # Return cache content f = open(cache_file, 'r') @@ -294,7 +305,7 @@ def _cache_read(command, max_old): f.close() return content - + @staticmethod def _cache_clean(): for f in os.listdir(CACHE_PATH): @@ -319,20 +330,22 @@ def _run_background_file(script, run_as=None): script_name = os.path.basename(fullpath) workdir = os.path.dirname(script) - # Create child process and return if parent - if ecm.fork(workdir): - return + + if ecm.is_linux(): + # Create child process and return if parent + if ecm.fork(workdir): + return # Write timeout to cache file _write_cache(script_name, CRITICAL, 'Timeout') - + try: command = [fullpath] sys.path.append(MY_PATH) - + env = os.environ.copy() env['PYTHONPATH'] = MY_PATH + ':' + env.get('PYTHONPATH', '') - + retval, stdout, stderr = ecm.run_command(command, runas=run_as, envars=env) _write_cache(script_name, retval, stdout) diff --git a/plugins/plugin_puppet.py b/plugins/plugin_puppet.py deleted file mode 100644 index d1624d2..0000000 --- a/plugins/plugin_puppet.py +++ /dev/null @@ -1,232 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -from base64 import b64decode -from tempfile import mkdtemp -from shutil import rmtree -import tarfile - -# Local -from __plugin import ECMPlugin -import __helper as ecm - -MODULES_PATH = '/etc/puppet/modules' -MODULES_PATH_WINDOWS = 'c:\ECM\puppet\modules' - -BOOTSTRAP = 'http://bootstrap.ecmanaged.com/puppet/linux/' -BOOTSTRAP_ALT = 'http://bootstrap-devel.ecmanaged.com/puppet/linux/' - -BOOTSTRAP_WINDOWS = 'http://bootstrap.ecmanaged.com/puppet/windows/' -BOOTSTRAP_WINDOWS_ALT = 'http://bootstrap-devel.ecmanaged.com/puppet/windows/' - -class ECMPuppet(ECMPlugin): - def cmd_puppet_available(self, *argv, **kwargs): - """ Checks if puppet commands are available - """ - return bool(self._is_available()) - - def cmd_puppet_install(self, *argv, **kwargs): - """ Installs saltstack using bootstrap scripts - """ - if self._is_available(): - return True - - bootstrap = BOOTSTRAP - bootstrap_file = 'bootstrap.sh' - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS - bootstrap_file = 'bootstrap.ps1' - - if not self._install(bootstrap, bootstrap_file): - # Try alternative bootstrap - bootstrap = BOOTSTRAP_ALT - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS_ALT - - if not self._install(bootstrap,bootstrap_file): - raise Exception("Unable to install puppet") - - return True - - def cmd_puppet_apply(self, *argv, **kwargs): - """ - Syntax: puppet.appy[recipe_code,evars,facts] - """ - recipe_base64 = kwargs.get('recipe_code', None) - metadata_platform = kwargs.get('metadata_platform', None) - metadata_server = kwargs.get('metadata_server', None) - - if not recipe_base64: - raise ecm.InvalidParameters(self.cmd_puppet_apply.__doc__) - - # Set module path - module_path = kwargs.get('module_path', None) - if module_path is None: - module_path = MODULES_PATH - if ecm.is_win(): - module_path = MODULES_PATH_WINDOWS - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata_server) - - # Update metadata - ecm.write_metadata_platform(metadata_b64=metadata_platform) - ecm.write_metadata(metadata_b64=metadata_server) - - try: - catalog = b64decode(recipe_base64) - except: - raise ecm.InvalidParameters("Unable to decode recipe") - - try: - command = ['puppet', - 'apply', - '--modulepath', - module_path, - '--detailed-exitcodes', - '--debug'] - - out, stdout, stderr = ecm.run_command(command, stdin=catalog, envars=envars) - ret = ecm.format_output(out, stdout, stderr) - - # exit code of '2' means there were changes - if ret['out'] == 2: ret['out'] = 0 - if "\nError: " in ret['stderr']: ret['out'] = 4 - - return ret - - except Exception as e: - raise Exception("Error running puppet apply: %s" % e) - - def cmd_puppet_apply_file(self, *argv, **kwargs): - """ - Syntax: puppet.apply_file[recipe_url,envars,facts] - """ - recipe_url = kwargs.get('recipe_url', None) - metadata = kwargs.get('metadata', None) - - if not recipe_url: - raise ecm.InvalidParameters(self.cmd_puppet_apply.__doc__) - - recipe_file = None - recipe_path = None - module_path = MODULES_PATH - if ecm.is_win(): - module_path = MODULES_PATH_WINDOWS - module_path = kwargs.get('module_path', module_path) - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - try: - # Download recipe url - recipe_path = mkdtemp() - tmp_file = recipe_path + '/recipe.tar.gz' - - if ecm.download_file(recipe_url, tmp_file): - if tarfile.is_tarfile(tmp_file): - tar = tarfile.open(tmp_file) - tar.extractall(path=recipe_path) - - for file_name in tar.getnames(): - if file_name.endswith('.catalog.pson'): - recipe_file = file_name - tar.close() - - # Apply puppet catalog - return self._run_catalog(recipe_file, recipe_path, module_path=module_path, envars=envars) - else: - raise Exception("Invalid recipe tgz file") - else: - raise Exception("Unable to download file") - - except: - raise Exception("Unable to get recipe") - - finally: - rmtree(recipe_path, ignore_errors=True) - - def _is_available(self): - """ which puppet - """ - if ecm.is_win(): - return ecm.which('puppet.exe') - - return ecm.which('puppet') - - def _run_catalog(self, recipe_file, recipe_path, module_path, envars=None): - """ Execute catalog file - """ - retval = self._run_puppet(recipe_file, recipe_path, module_path, 'catalog', envars) - - # Try old way - if 'invalid option' in retval.get('stdout', ''): - retval = self._run_puppet(recipe_file, recipe_path, module_path, 'apply', envars) - - return retval - - def _run_puppet(self, recipe_file, recipe_path, module_path, catalog_cmd='catalog', envars=None): - """ Real puppet execution - """ - puppet_cmd = self._is_available() - if not puppet_cmd: - raise Exception("Puppet is not available") - - command = [puppet_cmd, 'apply', '--detailed-exitcodes', '--modulepath', module_path, '--debug', - '--' + catalog_cmd, recipe_file] - - out, std_out, std_err = ecm.run_command(command, workdir=recipe_path, envars=envars) - ret = ecm.format_output(out, std_out, std_err) - - # --detailed-exitcodes - # Provide transaction information via exit codes. If this is enabled, - # an exit code of '2' means there were changes, - # an exit code of '4' means there were failures during the transaction, - # and an exit code of '6' means there were both changes and failures. - - # bug in exitcodes in some version even with errors return 0 - # http://projects.puppetlabs.com/issues/6322 - - if ret['out'] == 2: - ret['out'] = 0 - if "\nError: " in ret['stderr']: - ret['out'] = 4 - - return ret - - def _install(self, bootstrap_url, bootstrap_file = 'bootstrap.sh'): - """ Installs puppet using bootstrap url - """ - tmp_dir = mkdtemp() - bootstrap_file = tmp_dir + '/' + bootstrap_file - ecm.download_file(bootstrap_url, bootstrap_file) - - # wget -O - http://bootstrap.ecmanaged.com/puppet/linux/ | sudo sh - - if ecm.file_read(bootstrap_file): - envars = {'DEBIAN_FRONTEND': 'noninteractive'} - ecm.run_file(bootstrap_file, envars=envars) - - rmtree(tmp_dir) - return bool(self._is_available()) - - -ECMPuppet().run() diff --git a/plugins/plugin_reboot_require.py b/plugins/plugin_reboot_require.py deleted file mode 100644 index 2454a9b..0000000 --- a/plugins/plugin_reboot_require.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -import os -from commands import getstatusoutput -from pkg_resources import parse_version - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -pkg_kit = True - -try: - from gi.repository import PackageKitGlib -except ImportError: - pkg_kit = False - - -class ECMRebootRequirePackageKit(ECMPlugin): - def cmd_reboot_require(self, *argv, **kwargs): - from platform import release, machine - from gi.repository import PackageKitGlib - - working_kernel = release() - - client = PackageKitGlib.Client() - client.refresh_cache(False, None, lambda p, t, d: True, None) - res = client.resolve(PackageKitGlib.FilterEnum.INSTALLED, ['kernel'], None, lambda p, t, d: True, None) - - if res.get_exit_code() != PackageKitGlib.ExitEnum.SUCCESS: - return False - - package_ids = res.get_package_array() - - if len(package_ids) == 0: - return False - - installed_kernel = None - - for pkg in package_ids: - if pkg.get_arch() == machine(): - if installed_kernel is None: - installed_kernel = pkg - else: - if parse_version(pkg.get_version()) > parse_version(installed_kernel.get_version()): - installed_kernel = pkg - - installed_kernel = installed_kernel.get_version() + '.' +installed_kernel.get_arch() - - return parse_version(installed_kernel) > parse_version(working_kernel) - -class ECMRebootRequire(ECMPlugin): - def cmd_reboot_require(self, *argv, **kwargs): - distribution, _version = ecm.get_distribution() - if distribution.lower() in ['debian', 'ubuntu']: - if os.path.exists('/var/run/reboot-required.pkgs') or os.path.exists('/var/run/reboot-required'): - return True - else: - retval, current_kernel = getstatusoutput('uname -r') - if retval != 0: - return False - retval, latest_kernel = getstatusoutput("dpkg --list | grep linux-image | head -n1 | cut -d ' ' -f3 | perl -pe 's/^linux-image-(\S+).*/$1/'") - if retval != 0: - return False - return parse_version(latest_kernel) > parse_version(current_kernel) - return False - - elif distribution.lower() in ['centos', 'redhat', 'fedora', 'amazon']: - retval, current_kernel = getstatusoutput('uname -r') - if retval != 0: - return False - retval, latest_kernel = getstatusoutput("rpm -q --last kernel | perl -pe 's/^kernel-(\S+).*/$1/' | head -1") - if retval != 0: - return False - return parse_version(latest_kernel) > parse_version(current_kernel) - return False - - - -if pkg_kit: - ECMRebootRequirePackageKit().run() -else: - ECMRebootRequire().run() \ No newline at end of file diff --git a/plugins/plugin_saltstack.py b/plugins/plugin_saltstack.py deleted file mode 100644 index 243b5bd..0000000 --- a/plugins/plugin_saltstack.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = True - -from tempfile import mkdtemp -from shutil import rmtree -from base64 import b64decode - -# Local -from __plugin import ECMPlugin -import __helper as ecm - -DEFAULT_SALT_PATH = '/srv/salt' -DEFAULT_SALT_PATH_WINDOWS = 'C:\ECM\SALTSTACK\salt' - -DEFAULT_PILLAR_PATH = '/srv/pillar' -DEFAULT_PILLAR_PATH_WINDOWS = 'C:\ECM\SALTSTACK\pillar' - -BOOTSTRAP = 'http://bootstrap.ecmanaged.com/saltstack/linux' -BOOTSTRAP_ALT = 'http://bootstrap.saltstack.org' - -BOOTSTRAP_WINDOWS = 'http://bootstrap.ecmanaged.com/saltstack/windows' -BOOTSTRAP_WINDOWS_ALT = 'http://bootstrap.saltstack.org' - -TOP_CONTENT = """base: - '*': - - ecmanaged -""" - -class ECMSaltstack(ECMPlugin): - def cmd_saltstack_available(self, *argv, **kwargs): - """ Checks if saltstack commands are available - """ - return bool(self._is_available()) - - def cmd_saltstack_install(self, *argv, **kwargs): - """ Installs saltstack using bootstrap scripts - """ - if self._is_available(): return True - - bootstrap = BOOTSTRAP - bootstrap_file = 'bootstrap.sh' - - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS - bootstrap_file = 'bootstrap.ps1' - - if not self._install(bootstrap,bootstrap_file): - # Try alternative bootstrap - bootstrap = BOOTSTRAP_ALT - if ecm.is_win(): - bootstrap = BOOTSTRAP_WINDOWS_ALT - - if not self._install(bootstrap,bootstrap_file): - raise Exception("Unable to install saltstack") - - return True - - def cmd_saltstack_apply(self, *argv, **kwargs): - """ - Apply a saltstack manifest - Syntax: saltstack.apply[recipe_code,pillar_code,envars,facts] - """ - recipe_b64 = kwargs.get('recipe_code', None) - pillar_b64 = kwargs.get('pillar_code', None) - metadata = kwargs.get('metadata', None) - - if not recipe_b64: - raise ecm.InvalidParameters(self.cmd_saltstack_apply.__doc__) - - saltstack_cmd = self._is_available() - if not saltstack_cmd: - raise Exception('Saltstack no available') - - # Get default paths - default_path = DEFAULT_SALT_PATH - if ecm.is_win(): - default_path = DEFAULT_SALT_PATH_WINDOWS - module_path = kwargs.get('module_path', default_path) - - default_pillar_path = DEFAULT_PILLAR_PATH - if ecm.is_win(): - default_pillar_path = DEFAULT_PILLAR_PATH_WINDOWS - pillar_path = kwargs.get('pillar_path', default_pillar_path) - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - try: - # Create top file - self._create_top_file(module_path) - - recipe_file = module_path + '/ecmanaged.sls' - ecm.file_write(recipe_file, b64decode(recipe_b64)) - - if pillar_b64: - self._create_top_file(pillar_path) - pillar_file = pillar_path + '/ecmanaged.sls' - ecm.file_write(pillar_file, b64decode(pillar_b64)) - - except: - raise Exception("Unable to write recipe") - - try: - # salt-call state.highstate - command = [saltstack_cmd, 'state.highstate', '--local', '--no-color', '-l debug'] - - out, stdout, stderr = ecm.run_command(command, envars=envars, workdir=module_path) - return ecm.format_output(out, stdout, stderr) - - except Exception as e: - raise Exception("Error running saltstack state.highstate: %s" % e) - - def _create_top_file(self, path): - top_file = path + '/top.sls' - ecm.file_write(top_file, TOP_CONTENT) - - def _is_available(self): - """ it's salt-call on path? - """ - if ecm.is_win(): - return ecm.which('salt-call.exe') - return ecm.which('salt-call') - - def _install(self, bootstrap_url, bootstrap_file = 'bootstrap.sh'): - """ Installs saltstack using bootstrap url - """ - - tmp_dir = mkdtemp() - bootstrap_file = tmp_dir + '/' + bootstrap_file - ecm.download_file(bootstrap_url, bootstrap_file) - - # wget -O - http://bootstrap.saltstack.org | sudo sh - - # Options: - #-h Display this message - #-v Display script version - #-n No colours. - #-D Show debug output. - #-c Temporary configuration directory - #-g Salt repository URL. (default: git://github.com/saltstack/salt.git) - #-k Temporary directory holding the minion keys which will pre-seed the master. - #-M Also install salt-master - #-S Also install salt-syndic - #-N Do not install salt-minion - #-X Do not start daemons after installation - #-C Only run the configuration function. This option automatically bypasses any installation. - #-P Allow pip based installations. On some distributions the required salt packages or its dependencies are not available as a package for that distribution. Using this flag allows the script to use pip as a last resort method. NOTE: This works for functions which actually implement pip based installations. - #-F Allow copied files to overwrite existing(config, init.d, etc) - #-U If set, fully upgrade the system prior to bootstrapping salt - #-K If set, keep the temporary files in the temporary directories specified with -c and -k. - - if ecm.file_read(bootstrap_file): - envars = {'DEBIAN_FRONTEND': 'noninteractive'} - ecm.run_file(bootstrap_file, args=['-n', '-P', '-X'], envars=envars) - - rmtree(tmp_dir) - return bool(self._is_available()) - - -ECMSaltstack().run() diff --git a/plugins/plugin_source.py b/plugins/plugin_source.py deleted file mode 100644 index 40adcac..0000000 --- a/plugins/plugin_source.py +++ /dev/null @@ -1,488 +0,0 @@ -# -*- coding:utf-8 -*- - -# Copyright (C) 2012 Juan Carlos Moreno -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -RUN_AS_ROOT = False - -import os - -from tempfile import mkdtemp, NamedTemporaryFile -from urlparse import urlparse -from shutil import move, rmtree -from base64 import b64decode - -try: - import tarfile - import zipfile - import gzip - import bz2 - -except ImportError: - pass - -# Local -from __plugin import ECMPlugin -import __helper as ecm - - -class ECMSource(ECMPlugin): - def cmd_source_run(self, *argv, **kwargs): - """ - Syntax: source.run[path,source,branch,envars,facts,username,password,private_key,chown_user,chown_group,rotate,type] - """ - path = kwargs.get('path', None) - url = kwargs.get('source', None) - branch = kwargs.get('branch', None) - user = kwargs.get('username', None) - passwd = kwargs.get('password', None) - private_key = kwargs.get('private_key', None) - chown_user = kwargs.get('chown_user', None) - chown_group = kwargs.get('chown_group', None) - rotate = kwargs.get('rotate', True) - extract = kwargs.get('extract', True) - - stype = kwargs.get('type', None) - metadata = kwargs.get('metadata', None) - - if not path or not url or not stype: - raise ecm.InvalidParameters(self.cmd_source_run.__doc__) - - if private_key: - try: - private_key = b64decode(private_key) - except: - raise ecm.InvalidParameters("Invalid private key format") - - if stype.upper() in ('URL', 'FILE'): - source = FILE(path, rotate, extract) - - elif stype.upper() == 'GIT': - source = GIT(path, rotate) - - elif stype.upper() == 'SVN': - source = SVN(path, rotate) - - else: - raise ecm.InvalidParameters("Unknown source") - - # Set environment variables before execution - envars = ecm.metadata_to_env(metadata_b64=metadata) - - # Update metadata - ecm.write_metadata(metadata_b64=metadata) - - retval = source.clone(url=url, - branch=branch, - envars=envars, - username=user, - password=passwd, - private_key=private_key) - - # Chown to specified user/group - if chown_user and chown_group and os.path.isdir(path): - ecm.chown(path, chown_user, chown_group, recursive=True) - retval['stdout'] += ecm.output("Owner changed to '%s':'%s'" % (chown_user, chown_group)) - - return self._return(retval) - - def _return(self, ret): - output = { - 'out': ret.get('out', 1), - 'stderr': ret.get('stderr', ''), - 'stdout': ret.get('stdout', '') - } - return output - - -class GIT: - def __init__(self, working_dir, rotate=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - # Get git path - self.git_cmd = self._is_available() - - if not self.git_cmd: - if not self._install(): - raise Exception('Unable to find or install git') - self.git_cmd = self._is_available() - - def clone(self, url, branch, envars, username, password, private_key): - """ runs git clone URL - """ - command = self._get_command(url,branch) - - # Create git command with user and password - if username and password: - parsed = urlparse(url) - if parsed.scheme in ('http', 'https'): - command = command.replace('://', '://' + username + ':' + password + '@') - - elif parsed.scheme == 'ssh': - command = command.replace('://', '://' + username + '@') - - elif private_key: - helper, indetity = self._certificate_helper(private_key) - envars['GIT_SSH'] = helper - - out, stdout, stderr = ecm.run_command(command=command, workdir=self.working_dir, envars=envars) - result_exec = ecm.format_output(out, stdout, stderr) - - if not result_exec['out']: - extra_msg = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - if self.old_dir: - extra_msg += ecm.output("Old source files moved to '%s'" % self.old_dir) - result_exec['stdout'] += extra_msg - - if private_key: - try: - os.unlink(helper) - os.unlink(indetity) - except: - pass - - return result_exec - - def _get_command(self, url, branch): - param = '' - if branch and branch != 'master': - param = " -b " + str(branch) - - command = self.git_cmd + " clone" + param + " --quiet --verbose '" + url + "' ." - command_pull = self.git_cmd + " pull --quiet --verbose" - - if os.path.isdir(self.working_dir + '/.git'): - # GIT clone on already .git repo, do a pull and hope... - command = command_pull - - return command - - def _certificate_helper(self, private_key=None): - """ - Returns the path to a helper script which can be used in the GIT_SSH env - var to use a custom private key file. - """ - opts = { - 'StrictHostKeyChecking': 'no', - 'PasswordAuthentication': 'no', - 'KbdInteractiveAuthentication': 'no', - 'ChallengeResponseAuthentication': 'no', - } - - # Create identity file - identity = NamedTemporaryFile(delete=False) - ecm.chmod(identity.name, 0600) - identity.writelines([private_key]) - identity.close() - - # Create helper script - helper = NamedTemporaryFile(delete=False) - helper.writelines([ - '#!/bin/sh\n', - 'exec ssh ' + - ' '.join('-o%s=%s' % (key, value) for key, value in opts.items()) + - ' -i ' + identity.name + - ' $*\n' - ]) - - helper.close() - ecm.chmod(helper.name, 0750) - - return helper.name, identity.name - - def _is_available(self): - """ checks if git is on path - """ - if ecm.is_win(): - return ecm.which('git.exe') - - return ecm.which('git') - - def _install(self): - """ Try to install git - """ - ecm.install_package('git') - return bool(self._is_available()) - - -class SVN: - def __init__(self, working_dir, rotate=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - # Get git path - self.svn_cmd = self._is_available() - - if not self.svn_cmd: - if not self._install(): - raise Exception('Unable to find or install subversion') - self.svn_cmd = self._is_available() - - def clone(self, url, branch, envars, username, password, private_key): - """ svn co URL - """ - # Add username and password to url - if username and password: - url = url.replace('://', '://' + username + ':' + password + '@') - - command = self.svn_cmd + " co '" + url + "' ." - - out, stdout, stderr = ecm.run_command(command=command, workdir=self.working_dir, envars=envars) - result_exec = ecm.format_output(out, stdout, stderr) - - if not result_exec['out']: - extra_msg = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - if self.old_dir: - extra_msg += ecm.output("Old source files moved to '%s'" % self.old_dir) - - if result_exec['stdout']: - result_exec['stdout'] = extra_msg + result_exec['stdout'] - - else: - result_exec['stdout'] = extra_msg - - return result_exec - - def _is_available(self): - """ is svn on path - """ - if ecm.is_win(): - return ecm.which('svn.cmd') - return ecm.which('svn') - - def _install(self): - """ try to install subversion - """ - ecm.install_package('subversion') - return self._is_available() - - -class FILE: - def __init__(self, working_dir, rotate=False, extract=False): - if not working_dir: - raise ecm.InvalidParameters("Invalid path") - - self.working_dir = working_dir - self.rotate = rotate - self.extract = extract - - # Create or rename working_dir - deploy = Deploy(self.working_dir, rotate) - self.old_dir = deploy.prepare() - - def clone(self, branch, envars, url, username, password, private_key): - """ Downloads a file from a remote url and decompress it - """ - file_downloaded = ecm.download_file( - url=url, - user=username, - passwd=password - ) - - tmp_dir = os.path.dirname(file_downloaded) - - if file_downloaded: - if self.extract: - extract = self._extract(file_downloaded) - - if extract: - extract['head'] = '' - if extract.get('stdout', None): - extract['head'] = ecm.output("Source deployed successfully to '%s'" % self.working_dir) - - if extract.get('stdout', None) and self.old_dir: - extract['head'] += ecm.output("Old source files moved to '%s'" % self.old_dir) - else: - move(file_downloaded, self.working_dir) - extract = { - 'stdout': ecm.output("Source deployed successfully to '%s'" % self.working_dir), - 'stderr': '', - 'out': 0 - } - - else: - rmtree(tmp_dir, ignore_errors=True) - raise Exception("Unable to download file") - - # Clean and output - rmtree(tmp_dir, ignore_errors=True) - ret = { - 'stdout': extract.get('head', '') + extract.get('stdout', ''), - 'stderr': extract.get('stderr', 'Unable to download file'), - 'out': extract.get('out', 1) - } - - return ret - - def _extract(self, filename): - """ extractor helper - """ - try: - file_type = self._get_file_type(filename) - opener = mode = None - - if file_type == 'zip': - opener, mode = zipfile.ZipFile, 'r' - - elif file_type == 'gz': - if tarfile.is_tarfile(filename): - opener, mode = tarfile.open, 'r:gz' - - elif file_type == 'bz2': - if tarfile.is_tarfile(filename): - opener, mode = tarfile.open, 'r:bz2' - - if not opener: - raise Exception("Unsupported file compression") - - cfile = opener(filename, mode) - - # if first member is dir, skip 1st container path - if file_type == 'zip': - members = cfile.namelist() - else: - members = cfile.getmembers() - - stdout = '' - for member in members: - if file_type == 'zip': - member_name = member - else: - member_name = member.name - - stdout += "Extracted " + member_name + "\n" - cfile.extractall(self.working_dir) - cfile.close() - - except Exception as e: - try: - return self._extract_alternative(filename) - except: - raise Exception("Could not extract file: %s" % e) - - ret = {'out': 0, 'stderr': '', 'stdout': stdout} - return ret - - def _extract_alternative(self, filename): - """ extractor helper: Try to extract file using system commands - """ - from shutil import move - from os import path - - file_type = self._get_file_type(filename) - - # Move file before decompress - move(filename, self.working_dir) - filename = path.join(self.working_dir, path.basename(filename)) - - if file_type == 'zip': - package = 'unzip' - command = 'unzip' - args = [filename] - - elif file_type == 'gz': - package = 'gzip' - command = 'gunzip' - args = [filename] - - elif file_type == 'bz2': - package = 'bzip2' - command = 'bzip2' - args = ['-d', filename] - - else: - raise Exception("Unsupported file compression") - - exists = ecm.which(command) - if not exists: - # Try to install package - ecm.install_package(package) - exists = ecm.which(command) - - if exists and command: - # Decompress - out, stdout, stderr = ecm.run_command(command, args, workdir=self.working_dir) - ret = {'out': out, 'stderr': stderr, 'stdout': stdout} - return ret - - raise Exception("Could not extract file") - - def _get_file_type(self, filename): - """ get compressed file type based on marks - """ - magic_dict = { - "\x1f\x8b\x08": "gz", - "\x42\x5a\x68": "bz2", - "\x50\x4b\x03\x04": "zip" - } - - max_len = max(len(x) for x in magic_dict) - with open(filename) as f: - file_start = f.read(max_len) - for magic, filetype in magic_dict.items(): - if file_start.startswith(magic): - return filetype - - return False - - -class Deploy: - def __init__(self, working_dir, rotate): - self.working_dir = os.path.abspath(working_dir) - self.rotate = rotate - - def prepare(self): - """ Common function to create and rotate - """ - to_dir = None - if self.rotate and os.path.isdir(self.working_dir): - drive, path = os.path.splitdrive(self.working_dir) - split_path = ecm.split_path(path) - - try: - a = split_path[1] - except IndexError: - # Unsafe rotate - self.rotate = False - - if self.rotate: - to_dir = self.working_dir + '_rotated_' + ecm.utime() - move(self.working_dir, to_dir) - - # create working dir - if not os.path.isdir(self.working_dir): - ecm.mkdir_p(self.working_dir) - - return to_dir - - def rollback(self, path): - return - - -ECMSource().run() diff --git a/setup.cfg b/setup.cfg index 586e0cd..a7a39d2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ provides = ecmanaged-ecagent build_requires = systemd requires = python2 - python-devel python-twisted python-protocols python-configobj @@ -27,8 +26,4 @@ requires = python2 python-crypto python-httplib2 shadow-utils - python-pip - pygobject3 - PolicyKit - PackageKit - dbus-python \ No newline at end of file + dbus-python diff --git a/setup.py b/setup.py index b712bec..1161d28 100644 --- a/setup.py +++ b/setup.py @@ -16,30 +16,8 @@ from distutils.core import setup - -def _create_data_files(): - data_files=[('config', ['config/ecagent.init.cfg', 'config/xmpp_cert.pub']), - ('monitor/mplugin/__base__', ['monitor/mplugin/__base__/data.json']), - ('/etc/sudoers.d', ['sudoers.d/ecmanaged']), - ('',['configure.py','ecagent.bat', 'ecagent.sh', 'ecagentd.tac']) - ] - - from commands import getstatusoutput - retcode, systemd_system_unit_dir = getstatusoutput('pkg-config systemd --variable=systemdsystemunitdir') - - if retcode == 0: - # systemd - data_files.append((systemd_system_unit_dir, ['ecagentd.service'])) - data_files.append(('/etc/cron.d',['cron.d/ecmanaged-ecagent-systemd'])) - else: - data_files.append(('/etc/init.d',['ecagentd'])) - data_files.append(('/etc/cron.d',['cron.d/ecmanaged-ecagent-init'])) - - return data_files - - setup(name='ecmanaged-ecagent', - version='3.0', + version='3.2', license='Apache v2', description='ECManaged Agent - Monitoring and deployment agent', long_description='ECManaged Agent - Monitoring and deployment agent', @@ -50,14 +28,13 @@ def _create_data_files(): maintainer = 'Arindam Choudhury', maintainer_email = 'arindam@live.com', - url='www.ecmanaged.com', - + platforms=['All'], packages=['ecagent', 'plugins','monitor.mplugin.__base__'], - data_files=[('config', ['config/ecagent.init.cfg', 'config/xmpp_cert.pub']), + data_files=[('config', ['config/ecagent.cfg.init', 'config/xmpp_cert.pub']), ('monitor/mplugin/__base__', ['monitor/mplugin/__base__/data.json']), ('/etc/sudoers.d', ['sudoers.d/ecmanaged']), ('/etc/cron.d', ['cron.d/ecmanaged-ecagent']),