From c73bab9a0e44af068570d1cab67e3f1e5bc6f988 Mon Sep 17 00:00:00 2001 From: francisco-core Date: Sat, 15 Jul 2023 16:51:56 +0100 Subject: [PATCH] Add monitoring metrics exporter to snowflake role Add snowflake metrics exporter via prometheus node-exporter [1]. It extracts snowflake metrics from the snowflake journalctl logs and exposes them via the node-exporter. Metrics are available on the address http://:1900/metrics [1] https://prometheus.io/docs/guides/node-exporter/ --- ansible/inventory/production/hosts | 28 +-- .../roles/snowflake/files/export_metrics.sh | 2 + .../snowflake/files/snowflake2exporter.py | 179 ++++++++++++++++++ ansible/roles/snowflake/tasks/main.yml | 3 + ansible/roles/snowflake/tasks/monitoring.yml | 35 ++++ 5 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 ansible/roles/snowflake/files/export_metrics.sh create mode 100644 ansible/roles/snowflake/files/snowflake2exporter.py create mode 100644 ansible/roles/snowflake/tasks/monitoring.yml diff --git a/ansible/inventory/production/hosts b/ansible/inventory/production/hosts index c5b79e7..5ea379a 100644 --- a/ansible/inventory/production/hosts +++ b/ansible/inventory/production/hosts @@ -1,12 +1,18 @@ $ANSIBLE_VAULT;1.1;AES256 -62343439313634643734343430323633663263613337613763343634656666306262663735336630 -3230643063323937323862373265306362646230653637380a303036653033643631623932303564 -37623632383336623030303064666466353266323935626564666265623964656534393538356162 -3866636137373966360a653761376461356435396132306631653363616164633764313561373533 -32666239303463323366636132383136346464626331303633383862653333633136376539386266 -32376162373765656465646132623336333835636431643134653136356264623636656266393632 -38646430306333373731366462346639376238323330626165623466613363343761353461633333 -35316331366663323730656461613333633466663237316630656331303964656231633964653436 -66326464393063373566353761306533346639366166663531323235323463393566343836356239 -32303861303962663932356538616530613034633834353766303431633935326434363335326561 -353166366163376631303666313730343532 +63343430313630366136666531373634356532393637663063356565653834666337316361363934 +6332313864323530333237653830373266396137333561630a323836363163663238653138306165 +65643531303737646639616638623137643837663639636134663066613335633361303738333032 +3166343866653530640a636432373063333537663139316334336662333533303234656638303234 +30653231643633373333333538663137303334643861303762346363333162363436643236623131 +31366533613932616463343538333035303865656633396362336463313633343362313364613533 +61336435653531316363383233336538376630616262353236346134656366616238306366373832 +62376561323161323334353761346237353634383531363661393163613731653964386564343965 +61393135386265303661653434613661313361646636393362653064393535376136356262666662 +31306235386531353931333131333939633166666162393265313238656636323632383432323065 +66626432653663336333646366663737653331396664616266303366393435383339623637323039 +32396634663731326331646534623837313933626235653530356230323763373032643930333330 +32373231323836343431633462633031646266636430623130323233653632633630646234333634 +61366665643330623339323362623861666662316130323863363039323234323439623166363131 +39636134316361633334633932363134386639353530386531663665353337643931623239313566 +38613963613137383231303130313539343836303637353735343864613262623733353831373038 +6131 diff --git a/ansible/roles/snowflake/files/export_metrics.sh b/ansible/roles/snowflake/files/export_metrics.sh new file mode 100644 index 0000000..ddc2d9f --- /dev/null +++ b/ansible/roles/snowflake/files/export_metrics.sh @@ -0,0 +1,2 @@ +sudo journalctl -o cat -u snowflake-proxy > /tmp/snowflake-logs.txt +/usr/local/bin/snowflake2exporter.py --no-serve /tmp/snowflake-logs.txt | sudo tee /var/lib/prometheus/node-exporter/snowflake.prom \ No newline at end of file diff --git a/ansible/roles/snowflake/files/snowflake2exporter.py b/ansible/roles/snowflake/files/snowflake2exporter.py new file mode 100644 index 0000000..0b2e5e8 --- /dev/null +++ b/ansible/roles/snowflake/files/snowflake2exporter.py @@ -0,0 +1,179 @@ +#!/usr/bin/python3 +# This Script uses the following dependencies +# pip install nums-from-string +# pip install datetime +# +# To Run this script type: +# python main.py +# +# The default is ./docker_snowflake.log +# +# Example: +# python main.py snow.log +# +# Written By Allstreamer_ +# Licenced Under MIT +# +# Enhanced by MariusHerget +# Further enhanced and modified by mrdrache333 +# Further enhanced by francisco-core + +import argparse +import sys +import re +from datetime import datetime, timedelta +from http.server import HTTPServer, BaseHTTPRequestHandler + +# Format of your timestamps in the beginning of the log +# e.g. "2022/01/01 16:50:30 " => "%Y/%m/%d %H:%M:%S" +TIMESTAMP_FORMAT = "%Y/%m/%d %H:%M:%S" + +def nums_from_string(string): + return [int(num) for num in re.findall(r"\d+", string)] + +class TextHandler(BaseHTTPRequestHandler): + logfile_path = None + + def do_GET(self): + + if self.path != "/metrics": + # If the request path is not /metrics, return a 404 Not Found error + self.send_error(404) + return + # Set the response status code to 200 OK + self.send_response(200) + + # Set the content type to text/plain + self.send_header("Content-type", "text/plain") + + # End the headers + self.end_headers() + + # Return the metrics + print_stats( + self.logfile_path, + lambda x: self.wfile.write(x.encode()) # encode response + ) + + +def print_stats(logfile_path: str, printer_func): + # Read file + lines_all = readFile(logfile_path) + + # Get the statistics for various time windows + # e.g. all time => getDataFromLines(lines_all, 24) + # e.g. last 24h => getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24)) + # e.g. last Week => getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24 * 7)) + stats = { + 'All time': getDataFromLines(lines_all), + 'Last 24h': getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24)), + 'Last Week': getDataFromLines(filterLinesBasedOnTimeDelta(lines_all, 24 * 7)), + } + + # Print all the results in the Prometheus metric format + for time in stats: + stat = stats[time] + printer_func( + f"snowflake_served_people{{time=\"{time}\"}} {stat['connections']}\n" + + f"snowflake_upload_gb{{time=\"{time}\"}} {round(stat['upload_gb'], 4)}\n" + + f"snowflake_download_gb{{time=\"{time}\"}} {round(stat['download_gb'], 4)}\n" + ) + +def readFile(logfile_path: str): + # Read in log file as lines + lines_all = [] + with open(logfile_path, "r") as file: + lines_all = file.readlines() + return lines_all + + +# Catchphrase for lines who do not start with a timestamp +def catchTimestampException(rowSubString, timestampFormat): + try: + return datetime.strptime(rowSubString, timestampFormat) + except Exception: + return datetime.strptime("1970/01/01 00:00:00", "%Y/%m/%d %H:%M:%S") + + +# Filter the log lines based on a time delta in hours +def filterLinesBasedOnTimeDelta(log_lines, hours): + now = datetime.now() + length_timestamp_format = len(datetime.strftime(now, TIMESTAMP_FORMAT)) + return filter(lambda row: now - timedelta(hours=hours) <= catchTimestampException(row[0:length_timestamp_format], + TIMESTAMP_FORMAT) <= now, + log_lines) + + +# Convert traffic information (in B, KB, MB, or GB) to B (Bytes) and add up to a sum +def get_byte_count(log_lines): + byte_count = 0 + for row in log_lines: + symbols = row.split(" ") + + # Use a dictionary to map units to their byte conversion values + units = { + "B": 1, + "KB": 1024, + "MB": 1024 * 1024, + "GB": 1024 * 1024 * 1024 + } + + # Use the dictionary to get the byte conversion value for the current unit + byte_count += int(symbols[1]) * units[symbols[2]] + return byte_count + + +# Filter important lines from the log +# Extract number of connections, uploaded traffic in GB and download traffic in GB +def getDataFromLines(lines): + # Filter out important lines (Traffic information) + lines = [row.strip() for row in lines if "In the" in row] + lines = [row.split(",", 1)[1] for row in lines] + + # Filter out all traffic log lines who did not had any connection + lines = [row for row in lines if nums_from_string(row)[0] != 0] + + # Extract number of connections as a sum + connections = sum([nums_from_string(row)[0] for row in lines]) + + # Extract upload and download data + lines = [row.split("Relayed")[1] for row in lines] + upload = [row.split(",")[0].strip() for row in lines] + download = [row.split(",")[1].strip()[:-1] for row in lines] + + # Convert upload/download data to GB + upload_gb = get_byte_count(upload) / 1024 / 1024 / 1024 + download_gb = get_byte_count(download) / 1024 / 1024 / 1024 + + # Return information as a dictionary for better structure + return {'connections': connections, 'upload_gb': upload_gb, 'download_gb': download_gb} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--serve", + dest="serve", + action="store_true", + help="Start http server directly on port 8080" + ) + parser.add_argument( + "--no-serve", + dest="serve", + action="store_false", + help="Simply parse the input file" + ) + parser.set_defaults(serve=True) + + # Log file path from arguments (default: ./docker_snowflake.log) + parser.add_argument("logfile_path", default="./docker_snowflake.log") + args = parser.parse_args() + + if args.serve: + # Start the HTTP server on port 8080 + TextHandler.logfile_path = args.logfile_path + httpd = HTTPServer(("", 8080), TextHandler) + httpd.serve_forever() + else: + # Simply parse the file and print the resulting metrics + print_stats(args.logfile_path, sys.stdout.write) diff --git a/ansible/roles/snowflake/tasks/main.yml b/ansible/roles/snowflake/tasks/main.yml index da1d613..f038a56 100644 --- a/ansible/roles/snowflake/tasks/main.yml +++ b/ansible/roles/snowflake/tasks/main.yml @@ -60,3 +60,6 @@ that: "'unrestricted' in sf_nat_type.stdout" fail_msg: "[ERR] snowflake proxy NAT type is not unrestricted" tags: molecule-notest + +- name: Setup monitoring via prometheus node exporter + include_tasks: monitoring.yml diff --git a/ansible/roles/snowflake/tasks/monitoring.yml b/ansible/roles/snowflake/tasks/monitoring.yml new file mode 100644 index 0000000..49c800c --- /dev/null +++ b/ansible/roles/snowflake/tasks/monitoring.yml @@ -0,0 +1,35 @@ +--- + +- name: ensure prometheus-node-exporter is installed + apt: + pkg: prometheus-node-exporter + state: present + +- name: Setup snowflake metrics creation scripts + copy: + src: "{{ role_path }}/files/{{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: root + group: root + mode: 0744 + with_items: + - "snowflake2exporter.py" + - "export_metrics.sh" + +- name: Run CRON job to export snwoflake metrics to node-exporter + cron: + name: "snowflake_exporter" + user: "root" + weekday: "*" + minute: "*" + hour: "*" + job: "/usr/local/bin/export_metrics.sh" + state: present + cron_file: ansible_snowflake-export-metrics + + +- name: Restart service cron to pick up config changes + ansible.builtin.systemd: + state: restarted + daemon_reload: true + name: cron \ No newline at end of file