From 2f457423e436ca3642dbba76e8eab62b3c68ecf7 Mon Sep 17 00:00:00 2001 From: John Dunham Date: Tue, 21 May 2019 16:05:15 -0400 Subject: [PATCH 1/5] Fix for bug outlined in Issue #672. Implemented a variant of the suggested solution. --- .../beats/utility-scripts/csmTransactionRebuild.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py b/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py index 4580bcd09..a8e39551f 100755 --- a/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py +++ b/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py @@ -107,7 +107,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["job_submit_time"], "type" : "allocation", "uid" : data["allocation_id"], - "data" : data + "data" : data.decode('utf-8', 'ignore') } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -139,7 +139,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["job_submit_time"], "type" : "allocation", "uid" : data["allocation_id"], - "data" : data + "data" : data.decode('utf-8', 'ignore') } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -160,7 +160,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["begin_time"], "type" : "allocation-step", "uid" : "{0}-{1}".format(data["allocation_id"],data["step_id"]), - "data" : data + "data" : data.decode('utf-8', 'ignore') } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -187,7 +187,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["begin_time"], "type" : "allocation-step", "uid" : "{0}-{1}".format(data["allocation_id"],data["step_id"]), - "data" : data + "data" : data.decode('utf-8', 'ignore') } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) From 7402373deace484790e2b8b8b182419e4ad4846c Mon Sep 17 00:00:00 2001 From: John Dunham Date: Tue, 21 May 2019 16:15:56 -0400 Subject: [PATCH 2/5] Previous commit was buggy, implementing suggested solution directly. --- .../utility-scripts/csmTransactionRebuild.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py b/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py index a8e39551f..e13b5208b 100755 --- a/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py +++ b/csm_big_data/beats/utility-scripts/csmTransactionRebuild.py @@ -26,6 +26,19 @@ DEFAULT_USER="postgres" DEFAULT_OUTPUT="csm-transaction.log" +def sanitize_string(v): + return v.decode('utf-8', 'ignore') + +def sanitize_dict(d): + for k, v in d.iteritems(): + if isinstance(v, dict): + d[k] = sanitize_dict(v) + elif isinstance(v, str): + d[k] = sanitize_string(v) + else: + d[k] = v + return d + def dump_allocations( db, user, output ): try: db_conn= psycopg2.connect("dbname='{0}' user='{1}' host='localhost'".format(db, user)) @@ -107,7 +120,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["job_submit_time"], "type" : "allocation", "uid" : data["allocation_id"], - "data" : data.decode('utf-8', 'ignore') + "data" : sanitize_dict(data) } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -139,7 +152,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["job_submit_time"], "type" : "allocation", "uid" : data["allocation_id"], - "data" : data.decode('utf-8', 'ignore') + "data" : sanitize_dict(data) } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -160,7 +173,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["begin_time"], "type" : "allocation-step", "uid" : "{0}-{1}".format(data["allocation_id"],data["step_id"]), - "data" : data.decode('utf-8', 'ignore') + "data" : sanitize_dict(data) } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) @@ -187,7 +200,7 @@ def dump_allocations( db, user, output ): "timestamp" : data["begin_time"], "type" : "allocation-step", "uid" : "{0}-{1}".format(data["allocation_id"],data["step_id"]), - "data" : data.decode('utf-8', 'ignore') + "data" : sanitize_dict(data) } file.write('{0}\n'.format(json.dumps(data_wrapped, default=str))) From 82e5bc3c5edaabd349d123801043d7176d5a85f5 Mon Sep 17 00:00:00 2001 From: John Dunham Date: Tue, 11 Jun 2019 13:25:02 -0400 Subject: [PATCH 3/5] Added a fix for Kibana 6.8.0 (dropped category), bumped plugin version.. --- csm_big_data/kibana/plugins/cast_vis/package.json | 4 ++-- csm_big_data/kibana/plugins/cast_vis/public/register_vis.js | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/csm_big_data/kibana/plugins/cast_vis/package.json b/csm_big_data/kibana/plugins/cast_vis/package.json index 7354da3f0..361a9427c 100644 --- a/csm_big_data/kibana/plugins/cast_vis/package.json +++ b/csm_big_data/kibana/plugins/cast_vis/package.json @@ -1,6 +1,6 @@ { "name": "cast_visualizer", - "version": "1.0.0", + "version": "1.0.1", "description": "A kibana plugin to facilitate visualization and control of CAST datasets.", "main": "index.js", "kibana": { @@ -11,7 +11,7 @@ "start": "plugin-helpers start", "test:server": "plugin-helpers test:server", "test:browser": "plugin-helpers test:browser", - "build": "plugin-helpers build -k 6.3.2", + "build": "plugin-helpers build -k 6.8.0", "postinstall": "plugin-helpers postinstall" }, "devDependencies": { diff --git a/csm_big_data/kibana/plugins/cast_vis/public/register_vis.js b/csm_big_data/kibana/plugins/cast_vis/public/register_vis.js index 38d5295cb..eb23d7b64 100644 --- a/csm_big_data/kibana/plugins/cast_vis/public/register_vis.js +++ b/csm_big_data/kibana/plugins/cast_vis/public/register_vis.js @@ -6,7 +6,6 @@ import { CastVisController } from './vis_controller'; import CastSearchVisWrapper from './components/vis/cast_search_vis.js'; -import { CATEGORY } from 'ui/vis/vis_category'; import { VisFactoryProvider } from 'ui/vis/vis_factory'; import { VisTypesRegistryProvider } from 'ui/registry/vis_types'; import { defaultFeedbackMessage } from 'ui/vis/default_feedback_message'; @@ -23,7 +22,6 @@ export default function CASTVisProvider(Private) { isAccessible: true, image, description: 'Performs a CAST based filtering', - category: CATEGORY.OTHER, feedbackMessage: defaultFeedbackMessage, visualization: CastVisController, visConfig: { From 1a43337eac579c93ac5e5f088e1e6ff7250ef9b9 Mon Sep 17 00:00:00 2001 From: John Dunham Date: Tue, 11 Jun 2019 14:01:39 -0400 Subject: [PATCH 4/5] Bumping kibana plugin version to match. --- csm_big_data/rpmscripts/kibana.post.install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csm_big_data/rpmscripts/kibana.post.install b/csm_big_data/rpmscripts/kibana.post.install index 0c0bc6c08..10046344b 100644 --- a/csm_big_data/rpmscripts/kibana.post.install +++ b/csm_big_data/rpmscripts/kibana.post.install @@ -1,7 +1,7 @@ echo "Adding Cast Visulizer to kibana plugins." /usr/share/kibana/bin/kibana-plugin remove cast_visualizer -/usr/share/kibana/bin/kibana-plugin install file:///opt/ibm/csm/bigdata/kibana/cast_visualizer-1.0.0.zip +/usr/share/kibana/bin/kibana-plugin install file:///opt/ibm/csm/bigdata/kibana/cast_visualizer-1.0.1.zip echo "Cast Visulizer added to kibana plugins." From d2e28900d5f7dd4603f5d39df44f7c07aaf35231 Mon Sep 17 00:00:00 2001 From: John Dunham Date: Tue, 11 Jun 2019 14:18:40 -0400 Subject: [PATCH 5/5] Bumping requirements. --- csm_big_data/setupRPM.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csm_big_data/setupRPM.cmake b/csm_big_data/setupRPM.cmake index d94f2df37..28768c4df 100644 --- a/csm_big_data/setupRPM.cmake +++ b/csm_big_data/setupRPM.cmake @@ -26,7 +26,7 @@ set( CPACK_RPM_csm-bds-kibana_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/csm_big_data/rpmscripts/kibana.pre.uninstall") set( CPACK_RPM_csm-bds-kibana_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/csm_big_data/rpmscripts/kibana.post.install") -set(CPACK_RPM_csm-bds-kibana_PACKAGE_REQUIRES "kibana >= 6.2.3") +set(CPACK_RPM_csm-bds-kibana_PACKAGE_REQUIRES "kibana >= 6.8.0") # Setup Logstash RPM @@ -35,5 +35,5 @@ set( CPACK_RPM_csm-bds-logstash_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/csm_big_data/rpmscripts/logstash.pre.uninstall") set( CPACK_RPM_csm-bds-logstash_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/csm_big_data/rpmscripts/logstash.post.install") -set(CPACK_RPM_csm-bds-logstash_PACKAGE_REQUIRES "logstash >= 6.2.3") +set(CPACK_RPM_csm-bds-logstash_PACKAGE_REQUIRES "logstash >= 6.8.0")