Merge branch 'dev'

Author: alexander popov

mamonsu/lib/parser.py

@@ -10,6 +10,8 @@
     -c, --config       <file>
     -p, --pid          <pid-file>
     -d                 daemonize
+        --version      output version information, then exit
+        --help         show this help, then exit
 
 Export example config with default variables:
 Command: export
@@ -38,31 +40,37 @@
 Export template for native zabbix agent:
 Command: export zabbix-template 
 Examples:
-    {prog} export zabbix-template <file>
+    {prog} export zabbix-template [options] <file>
 Options:
     --template-name <template name>
     --plugin-type <plugin_type> (pg,sys,all)
     --application  <application name in template>
     --pg-version <pg_version>
     --add-plugins <directory>
     --config  <file>
+    --old-zabbix
 Default plugin_type = all, template name = PostgresPro-<platform name>,
 application = App-PostgresPro-<platform name>, pg-version = 10,
 Note: default pg-version is vanilla, but with PGPRO or PGEE before version number it can be changed. Supported version 
 numbers are 10, 11, 9.6, 9.5
+default template export for currently supported zabbix -server versions, if template for unsupported versions 
+is needed use --old zabbix flag
 Example: PGPRO_10 or PGEE_11 or PGPRO_9.6
 
 
 Export zabbix template with additional plugins included in config file:
 Command: export template 
 Examples:
-    {prog} export template <file>
+    {prog} export template [options] <file>
 Options:
     --add-plugins <directory>
     --template-name <template name>
     --application <application name in template>
     --config <file>
+    --old-zabbix
 Default template name = PostgresPro-<platform name>, application = App-PostgresPro-<platform name>
+Note: default template export for currently supported zabbix -server versions, if template for unsupported versions 
+is needed use --old zabbix flag
 
 
 Bootstrap DDL for monitoring:
@@ -117,6 +125,17 @@
     {prog} zabbix item error <host name>
     {prog} zabbix item lastvalue <host name>
     {prog} zabbix item lastclock <host name>
+    
+Export metrics to zabbix server
+Command: --send-data-zabbix 
+Example:
+    {prog} --send-data-zabbix --zabbix-file=localhost.log --zabbix-address=localhost 
+Options:    
+    --zabbix-address <name of the Zabbix host to send metrics>
+    --zabbix-port <port of Zabbix server to send metrics> by default 10051
+    --zabbix-file <path to file with metrics to send metrics>
+    --zabbix-client <name of the Zabbix host to send metrics> by default localhost 
+    --zabbix-log-level <log level to send metrics> (INFO|DEBUG|WARN) by default INFO
 """
 
 if platform.LINUX:
@@ -261,4 +280,18 @@ def parse_args():
     parser.add_option(
         '--application', dest='application',
         default='App-PostgresPro-{0}'.format(sys.platform.title()))
+    parser.add_option(
+        '--old-zabbix',
+        dest='old_zabbix', action='store_true',
+        help='Create special template for currently unsupported zabbix versions')
+    # Zabbix server to send metrics
+    parser.add_option('--zabbix-address', dest='zabbix_address', default=None)
+    # port of Zabbix server to send metrics
+    parser.add_option('--zabbix-port', dest='zabbix_port', default='10051')
+    # name of the Zabbix host to send metrics
+    parser.add_option('--zabbix-client', dest='zabbix_client', default='localhost')
+    # path to file with metrics to send metrics
+    parser.add_option('--zabbix-file', dest='zabbix_file', default=None)
+    # log level to send metrics
+    parser.add_option('--zabbix-log-level', dest='zabbix_log_level', default='INFO')
     return parser.parse_args()

mamonsu/lib/plugin.py

@@ -42,6 +42,9 @@ class Plugin(object):
     # for all childs
     is_child = True
 
+    # old_zabbix_server_version
+    old_zabbix = False
+
     # const
     PATH = "/etc/zabbix/zabbix_agent.d/scripts"
     DELTA = Template.DELTA

mamonsu/lib/runner.py

@@ -51,12 +51,34 @@ def quit_handler(_signo=None, _stack_frame=None):
             from mamonsu.tools.agent.start import run_agent
             sys.argv.remove('agent')
             return run_agent()
+        elif tool == '--send-data-zabbix':
+            args, _ = parse_args()
+            if not args.zabbix_address:
+                print('To send metrics to zabbix, please, add option --zabbix-addres')
+                exit(125)
+
+            if not args.zabbix_file:
+                print ('To send metrics to zabbix, please, add option --zabbix_file')
+                exit(123)
+
+            cfg = Config(args.config_file, args.plugins_dirs)
+            cfg.config.set('zabbix','address',args.zabbix_address)
+            cfg.config.set('zabbix','port',args.zabbix_port)
+            cfg.config.set('zabbix', 'client', args.zabbix_client)
+            cfg.config.set('log', 'level', args.zabbix_log_level)
+
+            supervisor = Supervisor(cfg)
+            supervisor.send_file_zabbix(cfg,args.zabbix_file)
+            exit(0)
+
         elif tool == 'export':
             args, commands = parse_args()
             # get PG version
             version_args = args.pg_version.split('_')
             define_pg_version(version_args)
             cfg = Config(args.config_file, args.plugins_dirs)
+            if args.old_zabbix:
+                Plugin.old_zabbix = True
             if not len(commands) == 2 and not len(commands) == 3:
                 print_total_help()
             if commands[1] == 'zabbix-parameters':
@@ -122,7 +144,7 @@ def quit_handler(_signo=None, _stack_frame=None):
                     commands.append('postgrespro.xml')
                     print('Template for mamonsu have been saved in postgrespro.conf file')
                 for klass in Plugin.only_child_subclasses():
-                    if klass.__name__ == "PgWaitSampling":  # check if plugin is for EE
+                    if klass.__name__ == "PgWaitSampling" or klass.__name__ == "Cfs" :  # check if plugin is for EE
                         if Plugin.VersionPG['type'] == 'PGEE':
                             plugins.append(klass(cfg))
                     else:

mamonsu/lib/senders/zbx.py

@@ -12,7 +12,7 @@
 import mamonsu.lib.platform as platform
 from mamonsu.lib.plugin import Plugin
 from mamonsu.lib.queue import Queue
-
+from itertools import islice
 
 class ZbxSender(Plugin):
 
@@ -64,6 +64,40 @@ def _flush(self):
         })
         self._send_data(data)
 
+    def send_file_to_zabbix (self,path):
+        zabbix_client = self.config.fetch('zabbix', 'client')
+        self.log.setLevel((self.config.fetch('log', 'level')).upper())
+
+        metrics = []
+        with open(path, 'r') as f:
+            while True:
+                lines=list(islice(f, 100))
+                for line in lines:
+                    try:
+                        split_line = line.rstrip('\n').split('\t')
+                        if len(split_line) == 3:
+                            metric = {
+                                'host': zabbix_client,
+                                'key': split_line[2],
+                                'value': split_line[1],
+                                'clock': int(split_line[0])}
+                            metrics.append(metric)
+                        else:
+                            self.log.error('Can\'t load metric in line: "{0}". The line must have the format: time <tab> value <tab> metric\'s name.'.format(line.rstrip('\n')))
+                    except Exception as e:
+                        self.log.error('Can\'t load metric in line: "{0}". Error : {1} '.format(line.rstrip('\n'),e,))
+
+                data = json.dumps({
+                    'request': 'sender data',
+                    'data': metrics,
+                    'clock': int(time.time())
+                })
+                self._send_data(data)
+                self.log.info('sended {0} metrics'.format(str(len(metrics))))
+                metrics = []
+                if not lines:
+                    break
+
     def _send_data(self, data):
         data_len = struct.pack('<Q', len(data))
         if platform.PY3:

mamonsu/lib/supervisor.py

@@ -6,6 +6,7 @@
 from mamonsu.lib.senders import *
 from mamonsu.tools.agent import *
 from mamonsu.plugins import *
+from mamonsu.lib.senders.zbx import ZbxSender
 
 
 class Supervisor(object):
@@ -62,3 +63,7 @@ def _loop(self):
                 else:
                     self._sender.send('mamonsu.plugin.errors[]', '')
                 plugin_errors, plugin_probes = 0, 0
+
+    def send_file_zabbix(self,cfg,path):
+        zbxSender = ZbxSender (cfg)
+        zbxSender.send_file_to_zabbix(path)

mamonsu/lib/zbx_template.py

@@ -55,7 +55,7 @@ class ZbxTemplate(object):
         ('snmpv3_contextname', None), ('snmpv3_securityname', None),
         ('snmpv3_securitylevel', 0), ('snmpv3_authprotocol', 0),
         ('snmpv3_authpassphrase', None), ('snmpv3_privprotocol', 0),
-        ('snmpv3_privpassphrase', None), ('formula', 1),
+        ('snmpv3_privpassphrase', None), ('logtimefmt', None), ('formula', 1),
         ('delay_flex', None), ('params', None),
         ('ipmi_sensor', None), ('data_type', 0), ('authtype', 0),
         ('username', None), ('password', None), ('publickey', None),
@@ -89,14 +89,20 @@ class ZbxTemplate(object):
         ('calc_fnc', 2), ('type', Template.GRAPH_TYPE.normal)
     ]
 
+    condition_defaults = [
+        ('macro', None), ('value', 0),
+        ('operator', None),
+        ('formulaid', None)
+    ]
+
     discovery_defaults = [
         ('name', None), ('type', 2), ('snmp_community', None),
         ('snmp_oid', None), ('delay', 60), ('status', 0),
         ('allowed_hosts', None), ('snmpv3_contextname', None),
         ('snmpv3_securityname', None), ('snmpv3_securitylevel', 0),
         ('snmpv3_authprotocol', 0), ('snmpv3_authpassphrase', None),
         ('snmpv3_privprotocol', 0), ('snmpv3_privpassphrase', None),
-        ('delay_flex', None), ('params', None), ('filter', None),
+        ('delay_flex', None), ('params', None),
         ('ipmi_sensor', None), ('authtype', 0),
         ('username', None), ('password', None), ('publickey', None),
         ('privatekey', None), ('port', None), ('lifetime', 7),
@@ -194,7 +200,23 @@ def graph(self, args={}, xml_key='graph'):
             self._format_args(self.graph_values_defaults, args),
             graph_items, xml_key)
 
-    def discovery_rule(self, rule={}, items=[], triggers=[], graphs=[]):
+    # condition for template creation for zabbix version 4.4
+    def condition(self, args={}, xml_key='condition'):
+        try:
+            conditions = args['condition']
+        except KeyError:
+            raise LookupError(
+                'Miss item in conditions: {0}.'.format(args))
+        res = ''
+        for idx, item in enumerate(conditions):
+            res += '<{1}>{0}</{1}>'.format(
+                self._format_args(self.condition_defaults, item),
+                xml_key)
+        res = '<conditions>' + res + '</conditions>' + '<formula/><evaltype>0</evaltype>'
+
+        return res
+
+    def discovery_rule(self, rule={},conditions=[], items=[], triggers=[], graphs=[]):
 
         result_items = '<item_prototypes>'
         for item in items:
@@ -214,10 +236,25 @@ def discovery_rule(self, rule={}, items=[], triggers=[], graphs=[]):
                 graph, xml_key='graph_prototype')
         result_graphs += '</graph_prototypes>'
 
-        result = '<discovery_rule>{0}{1}{2}{3}</discovery_rule>'
+        if len(conditions) > 0:
+
+            result_conditions = '<filter>'
+            for condition in conditions:
+                result_conditions += self.condition(
+                    condition, xml_key='condition')
+            result_conditions += '</filter>'
+        else:
+            result = '<discovery_rule>{0}{1}{2}{3}</discovery_rule>'
+            if  ('filter', None) not in self.discovery_defaults:
+                self.discovery_defaults.append(('filter', None))
+            return result.format(
+                self._format_args(self.discovery_defaults, rule),
+                result_items, result_triggers, result_graphs)
+
+        result = '<discovery_rule>{0}{1}{2}{3}{4}</discovery_rule>'
         return result.format(
             self._format_args(self.discovery_defaults, rule),
-            result_items, result_triggers, result_graphs)
+            result_conditions, result_items, result_triggers, result_graphs)
 
     def _application(self):
         result = '<applications><application><name>{0}'

mamonsu/plugins/pgsql/cfs.py

@@ -159,21 +159,36 @@ def graphs(self, template):
     def discovery_rules(self, template):
         rule = {
             'name': 'Compressed relations discovery',
-            'key': 'pgsql.cfs.discovery_compressed_relations[]',
-            'filter': '{#COMPRESSED_RELATION}:.*'
+            'key': 'pgsql.cfs.discovery_compressed_relations[]'
+            # 'filter': '{#COMPRESSED_RELATION}:.*'
         }
+        if Plugin.old_zabbix:
+            rule['filter'] = '{#DATABASE}:.*'
+            conditions = []
+        else:
+            conditions = [
+                {
+                    'condition': [
+                        {'macro': '{#COMPRESSED_RELATION}',
+                         'value': '.*',
+                         'operator': '',
+                         'formulaid': 'A'}
+                    ]
+                }
+
+            ]
         items = [
             {'key': 'pgsql.cfs.compress_ratio[{#COMPRESSED_RELATION}]',
-                'name': 'Relation {#COMPRESSED_RELATION}: compress ratio',
-                'delay': self.timeRatioInterval}
+             'name': 'Relation {#COMPRESSED_RELATION}: compress ratio',
+             'delay': self.timeRatioInterval}
         ]
         graphs = [
             {
                 'name': 'Relation {#COMPRESSED_RELATION}: compress ratio',
                 'delay': self.timeRatioInterval,
                 'items': [
                     {'color': '00CC00',
-                        'key': 'pgsql.cfs.compress_ratio[{#COMPRESSED_RELATION}]'}]
+                     'key': 'pgsql.cfs.compress_ratio[{#COMPRESSED_RELATION}]'}]
             },
         ]
-        return template.discovery_rule(rule=rule, items=items, graphs=graphs)
+        return template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs)

mamonsu/plugins/pgsql/databases.py

@@ -64,8 +64,23 @@ def discovery_rules(self, template):
         rule = {
             'name': 'Database discovery',
             'key': self.key_db_discovery.format('[{0}]'.format(self.Macros[self.Type])),
-            'filter': '{#DATABASE}:.*'
+
         }
+        if Plugin.old_zabbix:
+            conditions = []
+            rule['filter'] = '{#DATABASE}:.*'
+        else:
+            conditions = [
+                {
+                    'condition': [
+                        {'macro': '{#DATABASE}',
+                        'value': '.*',
+                        'operator': '',
+                        'formulaid': 'A'}
+                    ]
+                }
+
+            ]
         items = [
             {'key': self.right_type(self.key_db_size, var_discovery="{#DATABASE},"),
              'name': 'Database {#DATABASE}: size',
@@ -106,7 +121,7 @@ def discovery_rules(self, template):
                      'yaxisside': 1}]
             }
         ]
-        return template.discovery_rule(rule=rule, items=items, graphs=graphs)
+        return template.discovery_rule(rule=rule, conditions=conditions, items=items, graphs=graphs)
 
     def keys_and_queries(self, template_zabbix):
         result = []

mamonsu/plugins/pgsql/driver/connection.py

@@ -74,7 +74,7 @@ def _close(self):
                 self.log.error('close error: {0}'.format(e))
 
     def _connect(self):
-        self.log.debug('connecting')
+        self.log.info('connecting')
         host, unix_sock = self.host, None
         if host.startswith('/'):
             unix_sock, host = host, None
@@ -87,7 +87,7 @@ def _connect(self):
             database=self.db,
             application_name=self.appname
         )
-        self.log.debug('connected')
+        self.log.info('connected')
         self.conn.autocommit = True
         cur = self.conn.cursor()
         cur.execute('set statement_timeout to {0}'.format(self.timeout * 1000))

mamonsu/plugins/pgsql/driver/pool.py

@@ -32,9 +32,9 @@ class Pool(object):
             'select public.mamonsu_count_autovacuum()'
         ),
         'buffer_cache': (
-            "select sum(1) * 8 * 1024 as size, "
-            " sum(case when usagecount > 1 then 1 else 0 end) * 8 * 1024 as twice_used, "
-            " sum(case isdirty when true then 1 else 0 end) * 8 * 1024 as dirty "
+            "select sum(1) * (current_setting('block_size')::int8) as size, "
+            " sum(case when usagecount > 1 then 1 else 0 end) * (current_setting('block_size')::int8) as twice_used, "
+            " sum(case isdirty when true then 1 else 0 end) * (current_setting('block_size')::int8) as dirty "
             " from public.pg_buffercache",
             'select size, twice_used, dirty from public.mamonsu_buffer_cache()'
         ),