From f880e147434bb3b4e96ca0d086b6112e139c4458 Mon Sep 17 00:00:00 2001 From: Eric Hankinson Date: Thu, 25 Oct 2018 16:23:21 -0400 Subject: [PATCH 1/4] Initial support to puch CI data as JSON over MQTT --- ci_screen.cfg.example | 1 + ci_screen/screens/status_screen.py | 8 +++++++- ci_screen/service/ci_server_poller.py | 7 +++++-- ci_screen/service/mqtt_service.py | 22 ++++++++++++++++++++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/ci_screen.cfg.example b/ci_screen.cfg.example index 55efaaf..b3cb4f6 100644 --- a/ci_screen.cfg.example +++ b/ci_screen.cfg.example @@ -24,3 +24,4 @@ password= now_playing_topic= online_topic= marquee_topic= +ci_topic= diff --git a/ci_screen/screens/status_screen.py b/ci_screen/screens/status_screen.py index a492d77..8c6ea7a 100755 --- a/ci_screen/screens/status_screen.py +++ b/ci_screen/screens/status_screen.py @@ -13,6 +13,10 @@ from ci_screen.service.ci_server_poller import CIServerPoller +CI_UPDATE_SIGNAL = "CI_UPDATE" +CI_MQTT_UPDATE_SIGNAL = "CI_MQTT_UPDATE" + + class StatusScreen(qt.QQuickItem): holiday_changed = qt.pyqtSignal() @@ -34,7 +38,7 @@ def componentComplete(self): super(StatusScreen, self).componentComplete() self.on_status_updated.connect(self.on_status_update_on_ui_thread) - dispatcher.connect(self.on_status_update, "CI_UPDATE", sender=dispatcher.Any) + dispatcher.connect(self.on_status_update, signal=CI_UPDATE_SIGNAL, sender=dispatcher.Any) self.poller = CIServerPoller() self.poller.start_polling_async() @@ -93,6 +97,8 @@ def on_status_update_on_ui_thread(self, responses, errors): self._synchronize_projects(self.projects, [p for p in new_projects if not p.is_failed()], bad_ci_servers) self._synchronize_projects(self.failed_projects, [p for p in new_projects if p.is_failed()], bad_ci_servers) + dispatcher.send(signal=CI_MQTT_UPDATE_SIGNAL, sender=self, projects=new_projects) + def on_status_update(self, responses, errors): self.on_status_updated.emit(responses, errors) diff --git a/ci_screen/service/ci_server_poller.py b/ci_screen/service/ci_server_poller.py index f39c1f2..429ed3d 100755 --- a/ci_screen/service/ci_server_poller.py +++ b/ci_screen/service/ci_server_poller.py @@ -14,6 +14,9 @@ import ci_screen.service.ci_server_loader as ci_server_loader +CI_UPDATE_SIGNAL = "CI_UPDATE" + + logger = logging.getLogger(__name__) class CIServerPoller(object): @@ -39,7 +42,7 @@ def stop_polling(self): self.unsubscribe_all() self._stop.set() self.polling_thread = None - + def _poll_for_changes(self): while not self._stop.isSet(): @@ -57,7 +60,7 @@ def _poll_for_changes(self): if error is not None: errors[name] = error - dispatcher.send(signal="CI_UPDATE", sender=self, responses=responses, errors=errors) + dispatcher.send(signal=CI_UPDATE_SIGNAL, sender=self, responses=responses, errors=errors) time.sleep(self._poll_rate) def _get_auth(self, username, token): diff --git a/ci_screen/service/mqtt_service.py b/ci_screen/service/mqtt_service.py index 77c13b5..af59809 100644 --- a/ci_screen/service/mqtt_service.py +++ b/ci_screen/service/mqtt_service.py @@ -7,10 +7,13 @@ import paho.mqtt.client as mqtt from pydispatch import dispatcher +import simplejson as json NOW_PLAYING_SIGNAL = "NOW_PLAYING_UPDATE" MARQUEE_SIGNAL = "SHOW_MARQUEE" +CI_MQTT_UPDATE_SIGNAL = "CI_MQTT_UPDATE" + logger = logging.getLogger(__name__) @@ -38,9 +41,14 @@ def start(self): self._client.message_callback_add(self._now_playing_topic, self._on_now_playing) if self._marquee_topic: self._client.message_callback_add(self._marquee_topic, self._on_marquee) + if self._jenkins_topic: + dispatcher.connect(self.on_ci_update, signal=CI_MQTT_UPDATE_SIGNAL, sender=dispatcher.Any) self._client.connect_async(self._settings['host'], self._settings['port']) self._client.loop_start() + def on_ci_update(self, projects): + self._client.publish(self.ci_topic, json.dumps(projects), retain=True) + @property def _now_playing_topic(self): return self._settings['now_playing_topic'] @@ -53,6 +61,10 @@ def _online_topic(self): def _marquee_topic(self): return self._settings['marquee_topic'] + @property + def _ci_topic(self): + return self._settings['ci_topic'] + def _on_disconnect(self, client, userdata, return_code): logger.info('disconnected from mqtt broker: {}'.format(mqtt.error_string(return_code))) self._client.reconnect() @@ -64,6 +76,10 @@ def _on_connect(self, client, userdata, flags, return_code): logger.info('publishing to "{}"'.format(self._online_topic)) self._client.publish(self._online_topic, '1', retain=True) + if self._ci_topic: + logger.info('publishing to "{}"'.format(self.ci_topic)) + self._client.publish(self.ci_topic, '{}', retain=True) + if self._now_playing_topic: logger.info('subscribing to "{}"'.format(self._now_playing_topic)) self._client.subscribe(self._now_playing_topic) @@ -116,6 +132,7 @@ def _get_mqtt_settings(self): settings['now_playing_topic'] = mqtt.get('now_playing_topic', '') settings['online_topic'] = mqtt.get('online_topic', '') settings['marquee_topic'] = mqtt.get('marquee_topic', '') + settings['ci_topic'] = mqtt.get('ci_topic', '{}') return settings @@ -127,5 +144,6 @@ def _get_default_mqtt_settings(self): 'username': '', 'password': '', 'now_playing_topic': '', - 'online_topic': '', - 'marquee_topic': '' } + 'online_topic': '', + 'marquee_topic': '', + 'ci_topic': '' } From 706c3e68c73ffa5792628b0a20b380a6a7a714d5 Mon Sep 17 00:00:00 2001 From: Eric Hankinson Date: Thu, 25 Oct 2018 16:45:28 -0400 Subject: [PATCH 2/4] Oops, forgot some cleanup --- ci_screen/service/mqtt_service.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci_screen/service/mqtt_service.py b/ci_screen/service/mqtt_service.py index af59809..b4abc3c 100644 --- a/ci_screen/service/mqtt_service.py +++ b/ci_screen/service/mqtt_service.py @@ -41,7 +41,7 @@ def start(self): self._client.message_callback_add(self._now_playing_topic, self._on_now_playing) if self._marquee_topic: self._client.message_callback_add(self._marquee_topic, self._on_marquee) - if self._jenkins_topic: + if self._ci_topic: dispatcher.connect(self.on_ci_update, signal=CI_MQTT_UPDATE_SIGNAL, sender=dispatcher.Any) self._client.connect_async(self._settings['host'], self._settings['port']) self._client.loop_start() diff --git a/requirements.txt b/requirements.txt index c3a3055..6bdbc69 100755 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ python-dateutil freezegun tox paho-mqtt +simplejson From 3605cad08aae0f09c39600d06cc73f23d9a0ace6 Mon Sep 17 00:00:00 2001 From: Eric Hankinson Date: Thu, 25 Oct 2018 17:07:07 -0400 Subject: [PATCH 3/4] Pickling was better option --- ci_screen/service/mqtt_service.py | 8 ++++---- requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci_screen/service/mqtt_service.py b/ci_screen/service/mqtt_service.py index b4abc3c..81c31c8 100644 --- a/ci_screen/service/mqtt_service.py +++ b/ci_screen/service/mqtt_service.py @@ -7,7 +7,7 @@ import paho.mqtt.client as mqtt from pydispatch import dispatcher -import simplejson as json +import jsonpickle NOW_PLAYING_SIGNAL = "NOW_PLAYING_UPDATE" @@ -47,7 +47,7 @@ def start(self): self._client.loop_start() def on_ci_update(self, projects): - self._client.publish(self.ci_topic, json.dumps(projects), retain=True) + self._client.publish(self._ci_topic, jsonpickle.encode(projects, unpicklable=False), retain=True) @property def _now_playing_topic(self): @@ -77,8 +77,8 @@ def _on_connect(self, client, userdata, flags, return_code): self._client.publish(self._online_topic, '1', retain=True) if self._ci_topic: - logger.info('publishing to "{}"'.format(self.ci_topic)) - self._client.publish(self.ci_topic, '{}', retain=True) + logger.info('publishing to "{}"'.format(self._ci_topic)) + self._client.publish(self._ci_topic, '{}', retain=True) if self._now_playing_topic: logger.info('subscribing to "{}"'.format(self._now_playing_topic)) diff --git a/requirements.txt b/requirements.txt index 6bdbc69..2518eb2 100755 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ python-dateutil freezegun tox paho-mqtt -simplejson +jsonpickle From ff92914726dfcefb8ea387b47aed2f1cae2ce06e Mon Sep 17 00:00:00 2001 From: Eric Hankinson Date: Fri, 26 Oct 2018 19:39:41 -0400 Subject: [PATCH 4/4] Updated testing for new ci_topic feature --- .gitignore | 1 + README.md | 6 +++--- features/environment.py | 1 + features/mqtt.feature | 13 ++++++++++++- features/steps/mqtt_steps.py | 8 +++++++- features/steps/projects_steps.py | 2 +- features/support/helpers.py | 11 ++++++----- tox.ini | 2 +- 8 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 48e4a69..ba2d774 100755 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ ci_screen.cfg # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +*.qmlc # C extensions *.so diff --git a/README.md b/README.md index c944111..8f40e1a 100755 --- a/README.md +++ b/README.md @@ -52,12 +52,12 @@ token=1042081038133300184013931000109138 ## Prerequisites -QT5, sip, and PyQt5 must be installed. +Mosquitto, QT5, sip, and PyQt5 must be installed. #### OSX Install Install QT5 using the installer from [qt.io](qt.io). -Use homebrew to install python3, sip, and pyqt5. Alternatively you can build sip and pyqt from source. +Use homebrew to install python3, mosquitto, sip, and pyqt5. Alternatively you can build sip and pyqt from source. #### Raspbian Install @@ -69,4 +69,4 @@ Install python requirements using `pip install -r requirements.txt` and run `./m ## Running tests -`pip install tox` and run `tox`. To run tests. Currently they target python 3.4 or 3.5. +`pip install tox` and run `tox`. To run tests. Currently they target python 3.4, 3.5, 3.6 or 3.7. diff --git a/features/environment.py b/features/environment.py index 7b64037..3542820 100755 --- a/features/environment.py +++ b/features/environment.py @@ -25,6 +25,7 @@ def before_scenario(context, scenario): context.mqtt_now_playing_topic = '' context.mqtt_online_topic = '' context.mqtt_marquee_topic = '' + context.mqtt_ci_topic = '' helpers.rebuild_config_file(context) def after_scenario(context, scenario): diff --git a/features/mqtt.feature b/features/mqtt.feature index b821632..257a747 100644 --- a/features/mqtt.feature +++ b/features/mqtt.feature @@ -29,7 +29,7 @@ Feature: MQTT Then I see now playing info: | song | artist | album art | | These Days | The Black Keys | http://lorempixel.com/500/500/abstract/ | - + Scenario: Publishes online status when running Given I have MQTT enabled And I have a CI server with projects: @@ -97,3 +97,14 @@ Feature: MQTT | topic | message | | /testing/marquee | { "image_url":"../../features/assets/2.jpg","duration":5000 } | Then I see "../../features/assets/2.jpg" + + Scenario: Push CI data to MQTT + Given I have MQTT enabled + And I have a CI server with projects: + | name | status | + | My Project | Success | + And ci topic is set to "/testing/builds" + And the app is running + Then I get a message: + | topic | message | + | /testing/builds | [{"_activity": "Sleeping", "_last_build_label": "12 days ago", "_last_build_status": "Success", "_last_build_time": "2014-08-27T16:06:15Z", "_name": "My Project", "ci_server": "0"}] | diff --git a/features/steps/mqtt_steps.py b/features/steps/mqtt_steps.py index 2fd22ce..0f2bc03 100644 --- a/features/steps/mqtt_steps.py +++ b/features/steps/mqtt_steps.py @@ -65,6 +65,11 @@ def online_topic_is_set_to(context, topic): context.mqtt_online_topic = topic helpers.rebuild_config_file(context) +@given(u'ci topic is set to "(?P[^"]*)"') +def ci_topic_is_set_to(context, topic): + context.mqtt_ci_topic = topic + helpers.rebuild_config_file(context) + @step(u'I get a message') def get_a_message(context): row = context.table[0] @@ -73,8 +78,9 @@ def get_a_message(context): mqtt = context.mqtt_service mqtt.subscribe(topic) + if not eventually(lambda: mqtt.get_message(topic) == message): - raise Exception('Expected to get a message "{}" for topic "{}"'.format(message, topic)) + raise Exception('Expected to get a message "{}" for topic "{}". Instead got "{}"'.format(message, topic, mqtt.get_message(topic))) @given(u'marquee topic is set to "(?P[^"]*)"') def marquee_topic_is_set_to(context, topic): diff --git a/features/steps/projects_steps.py b/features/steps/projects_steps.py index a428e13..28279c3 100755 --- a/features/steps/projects_steps.py +++ b/features/steps/projects_steps.py @@ -1,7 +1,7 @@ from behave import * import pqaut.client import features.support.helpers as helpers - + @step(u'I see projects "(?P[^"]*)"') def i_see_projects(context, projects): diff --git a/features/support/helpers.py b/features/support/helpers.py index ae36151..a155a5c 100755 --- a/features/support/helpers.py +++ b/features/support/helpers.py @@ -9,7 +9,7 @@ import tzlocal import dateutil.parser import pqaut.client -from nose.tools import assert_true +from nose.tools import assert_true import features.support.config_helper as config_helper @@ -63,11 +63,11 @@ def get_port(): def rebuild_config_file(context): config = { 'general': { - 'poll_rate_seconds':str(context.poll_rate), - 'rotation':'0', + 'poll_rate_seconds':str(context.poll_rate), + 'rotation':'0', 'holiday':str(context.holiday), 'mqtt':str(context.mqtt_enabled) - }, + }, 'ci_servers': { 'sections':'' @@ -78,7 +78,8 @@ def rebuild_config_file(context): 'port':'52129', 'now_playing_topic':str(context.mqtt_now_playing_topic), 'online_topic':str(context.mqtt_online_topic), - 'marquee_topic':str(context.mqtt_marquee_topic) + 'marquee_topic':str(context.mqtt_marquee_topic), + 'ci_topic':str(context.mqtt_ci_topic) } } diff --git a/tox.ini b/tox.ini index 1649917..ad28581 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py{34,35}-{unit,integration} +envlist = py{34,35,36,37}-{unit,integration} skip_missing_interpreters=True [testenv]