From 949af1302acd1c43f2a7f6a0c91922b8e4ef3944 Mon Sep 17 00:00:00 2001 From: LedaLima Date: Wed, 21 Sep 2016 17:23:36 -0500 Subject: [PATCH 1/4] Fixed undeclared variables in impact analysis and globe in storyboard (#163) --- ui/flow/js/components/GlobeViewPanel.react.js | 2 +- ui/flow/js/components/ImpactAnalysisPanel.react.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/flow/js/components/GlobeViewPanel.react.js b/ui/flow/js/components/GlobeViewPanel.react.js index 5385250..ff946b5 100755 --- a/ui/flow/js/components/GlobeViewPanel.react.js +++ b/ui/flow/js/components/GlobeViewPanel.react.js @@ -2,7 +2,7 @@ var React = require('react'); var queue = require('d3-queue'); var GlobeViewStore = require('../stores/GlobeViewStore'); -var m0, o0, fill, proj, sky, path, swoosh, links, svg, width, height; +var m0, o0, fill, proj, sky, path, swoosh, links, svg, width, height, arcLines; var dataset, container; function buildGraph(root, ipsrc) { diff --git a/ui/flow/js/components/ImpactAnalysisPanel.react.js b/ui/flow/js/components/ImpactAnalysisPanel.react.js index e9f72ad..d2cc903 100755 --- a/ui/flow/js/components/ImpactAnalysisPanel.react.js +++ b/ui/flow/js/components/ImpactAnalysisPanel.react.js @@ -44,7 +44,7 @@ function buildGraph(root, ipsrc) hierarchy.nodes(root); x.domain([0, root.value]).nice(); - down(root, 0); + down.call(this.getDOMNode(), root, 0); } From 4c1bfb1bf93b9f7ea66d4bb653e8c39eac656a40 Mon Sep 17 00:00:00 2001 From: LedaLima Date: Thu, 22 Sep 2016 11:48:13 -0500 Subject: [PATCH 2/4] Oa readme review (#168) * Updated documentation according to revision * Updated OA documentation according to QA Review #2 --- README.md | 3 +- oa/INSTALL.md | 18 +++++----- oa/components/README.md | 19 +++++----- oa/components/reputation/README.md | 27 +++++--------- oa/dns/README.md | 26 +++++++------- oa/dns/ipynb_templates/EdgeNotebook.md | 10 +++--- oa/dns/ipynb_templates/ThreatInvestigation.md | 8 ++--- oa/flow/README.md | 35 +++++++++---------- oa/flow/ipynb_templates/EdgeNotebook.md | 11 +++--- .../ipynb_templates/ThreatInvestigation.md | 16 ++++----- oa/proxy/README.md | 20 +++++------ oa/proxy/ipynb_templates/EdgeNotebook.md | 10 +++--- .../ipynb_templates/ThreatInvestigation.md | 8 ++--- 13 files changed, 100 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index df8d6e4..0e0fc80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # **Open Network Insight** +ONI Operational Analytics (OA) is a collection of modules, which includes both the data processing and transformation as well as the GUI module for data visualization. -The visualization repository contains all the front-end code and files related to the Open Network Insight visual elements, such as styles, pages, data files, etc. +The visualization repository (UI folder) contains all the front-end code and files related to the Open Network Insight visual elements, such as styles, pages, data files, etc. Some of the technologies used are: - [IPython==3.2.1](https://ipython.org/ipython-doc/3/index.html) diff --git a/oa/INSTALL.md b/oa/INSTALL.md index 7217aa7..f11174c 100644 --- a/oa/INSTALL.md +++ b/oa/INSTALL.md @@ -3,12 +3,13 @@ ONI Operational Analytics (OA) is a set of python modules and utilities with routines to extract and transform data, loading the results into output files. OA represents the last step before users can score connections and analyze data in the UI. -OA scripts are very similar for the different data types supported however the code is divided into 3 -main modules due to differences on the data model and what context information is required for each data type. - The three supported data types are Flow, DNS and Proxy. For more information about the type of information and insights that can be found for each data source please visit ONI [wiki](https://github.com/Open-Network-Insight/open-network-insight/wiki). +OA scripts are very similar for the different data types supported however the code is divided into 3 +main modules due to differences in the data model and what context information is required for each data type. + + ## Folder Structure components -> Set of utilities prepared to provide context to raw data and @@ -39,7 +40,7 @@ In order to execute this process there are a few prerequisites: 2. Components configuration. To find about how to configure each of the extra components included in this project visit oa/components/[README.md](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/components). These components are required to add context or extract additional information that is going to complement your - original data. Each of this components are independent from each other. Based on the data type some components are + original data. Each of these components are independent from each other. Based on the data type some components are required or not. 3. oni-ml results. Operational Analytics works and transforms Machine Learning results. The implementation of Machine Learning in this project is through [oni-ml](https://github.com/Open-Network-Insight/oni-ml). Although the Operational Analytics @@ -54,7 +55,7 @@ In order to execute this process there are a few prerequisites: ##Operational Analytics installation and usage ####Installation - OA installation consists on the configuration of extra modules or components and creation of a set of files. + OA installation consists of the configuration of extra modules or components and creation of a set of files. Depending on the data type that is going to be processed some components are required and other components are not. If users are planning to analyze the three data types supported (Flow, DNS and Proxy) then all components should be configured. @@ -79,9 +80,9 @@ In order to execute this process there are a few prerequisites: 10.192.1.1, MySystem - 3. oni-setup project contains scripts to install hive database but also includes the main configuration file for this tool. - That file is called duxbay.conf which contains different variables that the user can set up to customize their installation, in fact, some - of them are required to be updated in order to have oni-ml and oni-oa working. + 3. The oni-setup project contains scripts to install the hive database and also includes the main configuration file for this tool. + The main file is called duxbay.conf and it which contains different variables that the user can set up to customize their installation. Some variables are + must be updated in order to have oni-ml and oni-oa working. To run the OA process it's required to install oni-setup. If it's already installed just make sure the following configuration are set up in duxbay.conf file. @@ -118,6 +119,7 @@ In order to execute this process there are a few prerequisites: OA process for the corresponding data type. -l Data limit. Usually ML results contains thousands of records. With "Data limit" OA will process top K results. + The execution time of OA varies based on the number of records being processed and the data type. Depending on the number of records being processed and the data type, OA can take long or short time to execute. When the process completes you can go to oni-oa/data/\ folder and check the results. diff --git a/oa/components/README.md b/oa/components/README.md index a06192a..e942be8 100644 --- a/oa/components/README.md +++ b/oa/components/README.md @@ -1,4 +1,4 @@ -# COMPONENTS +# Operational Analytics Components This document will explain the necessary steps to configure the oni-oa components. @@ -27,7 +27,7 @@ This document will explain the necessary steps to configure the oni-oa component ###Data -Data source module +_Data source module._ This module needs to be configured correctly to avoid errors during the oni-oa execution. Here you need to select the correct database engine to obtain the correct results while creating additional details files. Currently oni-oa is prepared to work with Impala, but you can always configure any other database engine and make the corresponding updates in the code. @@ -46,9 +46,9 @@ You need to update the _engine.json_ file accordingly: } Where: -- database engine: Whichever database engine you have installed and configured in your cluster to work with ONI. i.e. "Impala" or "Hive". +- : Whichever database engine you have installed and configured in your cluster to work with ONI. i.e. "Impala" or "Hive". For this key, the value you enter needs to match exactly with one of the following keys, where you'll need to add the corresponding node name. -- node: The node name in your cluster where you have the database service running. +- : The node name in your cluster where you have the database service running. Example: @@ -62,7 +62,7 @@ Example: ###Reputation -Reputation check module. +_Reputation check module._ This module is called during oni-oa execution to check the reputation for any given IP, DNS name or URI (depending on the pipeline). The reputation module makes use of two third-party services, McAfee GTI and Facebook ThreatExchange. Each of these services are represented by a sub-module in this project, McAfee GTI is implemented by sub-module gti and Facebook ThreatExchange by sub-module fb. For more information see Folder Structure section. @@ -77,6 +77,7 @@ Each of these services are represented by a sub-module in this project, McAfee G **Enable/Disable GTI service** It's possible to disable any of the reputation services mentioned above, all it takes is to remove the configuration for the undesired service in gti_config.json. To learn more about it, see the section below. +To add a different reputation service, you can read all about it [here](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/components/reputation) **Configuration** @@ -124,7 +125,7 @@ It's possible to disable any of the reputation services mentioned above, all it - app_secret: App secret to connect to ThreatExchange service. ###IANA -Internet Assigned Numbers Authority codes translation module. +_Internet Assigned Numbers Authority codes translation module._ **Configuration** @@ -146,7 +147,7 @@ default location, your configuration file should look like this: ###Network Context (nc) -Network Context module. +_Network Context module._ **Pre-requisites** @@ -175,11 +176,11 @@ configuration file should look like this: ###Geoloc -Geolocation module. +_Geolocation module._ This is an optional functionality you can enable / disable depending on your preferences. -**Pre-requisites** +**Pre-requisites** To start using this module, you need to include a comma separated file containing the geolocation for most (or all) IPs. To learn more about the expected schema for this file or where to find a full geolocation db, please refer to the [_context_](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/context/README.md) documentation diff --git a/oa/components/reputation/README.md b/oa/components/reputation/README.md index 28b465a..f4003cf 100644 --- a/oa/components/reputation/README.md +++ b/oa/components/reputation/README.md @@ -1,31 +1,22 @@ -###GTI (gti) -DNS Global Threat Intelligence module. - -This module is called in dns_oa.py for IP reputation check. The GTI module makes use of two third-party services, McAfee GTI and Facebook ThreatExchange. Each of these services are represented by a sub-module in this project, McAfee GTI is implemented by sub-module gti and Facebook ThreatExchange by sub-module fb. For more information see [Folder Structure](https://github.com/Open-Network-Insight/oni-oa/blob/1.0.1-dns_oa_readme_creation/ipython/dns/README.md#folder-structure). - -## How to implement a new reputation service for DNS OA - -DNS GTI comes with two sub-modules and they correspond to the reputation services we are supporting by default. - - gti: implements logic to call and return results from McAfee reputation service. - - fb: implements logic to call and return results from facebook ThreatExchange reputation service. +###Reputation +This section describes the functionality of the current reputation service modules and how you can implement your own. It's possible to add new reputation services by implementing a new sub-module, to do that developers should follow these steps: -1. Map the responses of the new reputation service with DNS reputation table. +1. Map the responses of the new reputation service, according to this reputation table. - | Key | Value | - |---|---| + | Key | Value | + |-----|-------| |UNVERIFIED|-1| |NONE |0 | |LOW |1 | |MEDIUM |2 | |HIGH |3 | -2. Add a new configuration for the new reputation service in gti_config.json. +2. Add a new key for the new reputation service in gti_config.json. - { - "targe_columns" : [3], + { "gti" : { … }, "fb" : {… @@ -36,11 +27,11 @@ DNS GTI comes with two sub-modules and they correspond to the reputation service } 3. Create file structure for new sub-module. - [solution-user@edge-server]$ cd ~/ipython/dns/gti/ + [solution-user@edge-server]$ cd ~/oni-oa/components/reputation/ [solution-user@edge-server]$ mkdir mynewreputationservice [solution-user@edge-server]$ cd mynewreputationservice -4. Add _ _init_ _.py file. +4. Create an empty _ _init_ _.py file. 5. Add a new file *reputation.py*. Each sub-module should contain a reputation.py file. 6. Write your code in reputation.py. The code should contain the follow structure: diff --git a/oa/dns/README.md b/oa/dns/README.md index 797096d..1f897b7 100644 --- a/oa/dns/README.md +++ b/oa/dns/README.md @@ -1,16 +1,12 @@ # DNS -oni-oa sub-module for Open-Network-Insight, version 1.1 - -DNS sub-module will extract and transform DNS (Domain Name Service) data already ranked by oni-ml and will load into csv files for presentation layer. +DNS sub-module extracts and transforms DNS (Domain Name Service) data already ranked by oni-ml and will load into csv files for presentation layer. ## DNS Components ###dns_oa.py -DNS oni-oa main script. - -It executes the following steps: +DNS oni-oa main script executes the following steps: 1. Creates the right folder structure to store the data and the ipython notebooks. This is: @@ -45,30 +41,32 @@ It executes the following steps: **Dependencies** -Before running DNS OA users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as Flow and Proxy. - -- python 2.7. [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Proxy OA. +- [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Proxy OA. The following modules are already included but some of them require configuration. See the following sections for more information. -- [components/iana](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#IANA-iana) -- [components/data](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#data) -- [components/nc](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#Network-Context-nc) -- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#Reputation) +- [components/iana](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#IANA-iana) +- [components/data](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#data) +- [components/nc](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#network-context-nc) +- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/reputation) - dns_conf.json + **Prerequisites** +Before running DNS OA users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as Flow and Proxy. + - Configure database engine - Configure GTI services - Configure IANA service - Configure Network Context service +- Configure Geolocation - Generate ML results for DNS **Output** -- dns_scores.csv: Main results file for DNS OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa). +- dns_scores.csv: Main results file for DNS OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/INSTALL.md#usage). Schema with zero-indexed columns: diff --git a/oa/dns/ipynb_templates/EdgeNotebook.md b/oa/dns/ipynb_templates/EdgeNotebook.md index 8e0fb1a..8cf2ce5 100644 --- a/oa/dns/ipynb_templates/EdgeNotebook.md +++ b/oa/dns/ipynb_templates/EdgeNotebook.md @@ -1,9 +1,9 @@ #DNS Edge Investigation Notebook ###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) The following python modules will be imported for the notebook to work correctly: @@ -20,8 +20,8 @@ The following python modules will be imported for the notebook to work correctly ###Pre-requisites - Execution of the oni-oa process for DNS -- Correct setup the duxbay.conf file. You can check this [link](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) -- Have a public key authentication between the current UI node and the ML node. You can follow this [instructions](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) +- Correct setup the duxbay.conf file. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Have a public key authentication between the current UI node and the ML node. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) ##Data source diff --git a/oa/dns/ipynb_templates/ThreatInvestigation.md b/oa/dns/ipynb_templates/ThreatInvestigation.md index 63c2ea2..e1f37c3 100644 --- a/oa/dns/ipynb_templates/ThreatInvestigation.md +++ b/oa/dns/ipynb_templates/ThreatInvestigation.md @@ -1,9 +1,9 @@ #DNS Threat Investigation Notebook ###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) The following python modules will have to be imported for the notebook to work correctly: @@ -22,7 +22,7 @@ The following python modules will have to be imported for the notebook to work c ##Pre-requisites - Execution of the oni-oa process for DNS - Score a set connections in the Edge Investigation Notebook -- Correct setup of the duxbay.conf file. You can check this [link](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Correct setup of the duxbay.conf file. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) ##Additional Configuration diff --git a/oa/flow/README.md b/oa/flow/README.md index d439fff..6c789b7 100644 --- a/oa/flow/README.md +++ b/oa/flow/README.md @@ -1,15 +1,11 @@ # **Flow OA** - -oni-oa sub-module for Open Network Insight, version 1.1 - -Flow sub-module will extract and transform Flow data already ranked by oni-ml and will load into csv files for presentation layer. + +Flow sub-module extracts and transforms Flow data already ranked by oni-ml and will load into csv files for presentation layer. ## **Flow OA Components** ### flow_oa.py -Flow oni-oa main script - -It executes the following steps: +Flow oni-oa main script executes the following steps: 1. Creates required folder structure if does not exist for output files. This is: @@ -26,12 +22,13 @@ It executes the following steps: **Dependencies** -- python 2.7. [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Flow OA. +- [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Flow OA. The following files and modules are already included but some of them require configuration. See the following sections for more information: -- [components/nc](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/components) -- [components/geoloc](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/components) -- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/components) +- [components/iana](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#IANA-iana) +- [components/data](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#data) +- [components/nc](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#network-context-nc) +- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/reputation) - flow_config.json The following files are not included: @@ -40,16 +37,18 @@ The following files are not included: **Prerequisites** -Before running Flow OA users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as DNS and Proxy. -- Configure reputation module components/reputation -- Configure network context module components/nc -- Configure geo localization module components/geo -- Create iploc.csv file context/iploc.csv -- Generate ML results for Flow +Before running Flow OA users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as DNS and Proxy. + +- Configure database engine +- Configure GTI services +- Configure IANA service +- Configure Network Context service +- Configure Geolocation service +- Generate ML results for Flow **Output** -- flow_scores.csv. Main results file for Flow OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa). +- flow_scores.csv. Main results file for Flow OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/INSTALL.md#usage). Schema with zero-indexed columns: 0. sev: int diff --git a/oa/flow/ipynb_templates/EdgeNotebook.md b/oa/flow/ipynb_templates/EdgeNotebook.md index 8e63d4f..0afdd9b 100644 --- a/oa/flow/ipynb_templates/EdgeNotebook.md +++ b/oa/flow/ipynb_templates/EdgeNotebook.md @@ -1,9 +1,10 @@ #Flow Edge Investigation Notebook ###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) +- [pandas](http://pandas.pydata.org/) The following python modules will be imported for the notebook to work correctly: @@ -22,8 +23,8 @@ The following python modules will be imported for the notebook to work correctly ###Pre-requisites - Execution of the oni-oa process for Flow -- Correct setup the duxbay.conf file. You can check this [link](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) -- Have a public key created between the current UI node and the ML node. You can follow this [instructions](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) +- Correct setup the duxbay.conf file. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Have a public key created between the current UI node and the ML node. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) ##Additional Configuration diff --git a/oa/flow/ipynb_templates/ThreatInvestigation.md b/oa/flow/ipynb_templates/ThreatInvestigation.md index d954c40..34d22ea 100644 --- a/oa/flow/ipynb_templates/ThreatInvestigation.md +++ b/oa/flow/ipynb_templates/ThreatInvestigation.md @@ -1,9 +1,9 @@ #Flow Threat Investigation Notebook -###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +###Dependencies +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) The following python modules will have to be imported for the notebook to work correctly: @@ -22,9 +22,9 @@ The following python modules will have to be imported for the notebook to work c ##Pre-requisites - Execution of the oni-oa process for Flow - Score a set connections at the Edge Investigation Notebook -- Correct setup the duxbay.conf file. You can check this [link](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) -- Include a comma separated network context file. **Optional** Yoy can find the schema [here](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#network-context-nc) -- Include a geolocation database file. Yoy can find the schema [here](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#geoloc) +- Correct setup the duxbay.conf file. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Include a comma separated network context file. **Optional** [Schema](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#network-context-nc) +- Include a geolocation database file. [Schema](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#geoloc) ##Additional Configuration @@ -41,7 +41,7 @@ Schema for these files can be found here: [flow_scores.csv](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa/flow) [iploc.csv](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#geoloc) -[networkcontext_1.csv](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#geoloc) +[networkcontext_1.csv](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/README.md#network-context-nc) data/flow//flow_scores.csv diff --git a/oa/proxy/README.md b/oa/proxy/README.md index 4e29c0f..ea1a0de 100644 --- a/oa/proxy/README.md +++ b/oa/proxy/README.md @@ -1,16 +1,12 @@ # PROXY -oni-oa sub-module for Open-Network-Insight, version 1.1 - Proxy sub-module will extract and transform Proxy data already ranked by oni-ml and will load into csv files for presentation layer. ## Proxy Components ###proxy_oa.py -Proxy oni-oa main script. - -It executes the following steps: +Proxy oni-oa main script executes the following steps: 1. Creates the right folder structure to store the data and the ipython notebooks. This is: @@ -43,18 +39,18 @@ It executes the following steps: **Dependencies** -- python 2.7. [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Proxy OA. +- [Python 2.7](https://www.python.org/download/releases/2.7/) should be installed in the node running Proxy OA. The following modules are already included but some of them require configuration. Please refer to the _components_ documentation for more information. -- [components/iana](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#IANA-iana) -- [components/data](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#data) -- [components/nc](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/proxy#Network-Context-nc) -- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#Reputation) +- [components/iana](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#IANA-iana) +- [components/data](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#data) +- [components/nc](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components#network-context-nc) +- [components/reputation](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/components/reputation) - proxy_conf.json **Prerequisites** -Before running Proxy OA, users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as Flow and DNS. +Before running Proxy OA, users need to configure components for the first time. It is important to mention that configuring these components make them work for other data sources as Flow and DNS. - Configure database engine - Configure Reputation services @@ -64,7 +60,7 @@ Before running Proxy OA, users need to configure components for the first time. **Output** -- proxy_scores.tsv: Main results file for Proxy OA. This file is tab separated and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/tree/1.1/oa). +- proxy_scores.tsv: Main results file for Proxy OA. This file is tab separated and it's limited to the number of rows the user selected when running [oa/start_oa.py](https://github.com/Open-Network-Insight/oni-oa/blob/1.1/oa/INSTALL.md#usage). Schema with zero-indexed columns: diff --git a/oa/proxy/ipynb_templates/EdgeNotebook.md b/oa/proxy/ipynb_templates/EdgeNotebook.md index 17a287f..3ee79f2 100644 --- a/oa/proxy/ipynb_templates/EdgeNotebook.md +++ b/oa/proxy/ipynb_templates/EdgeNotebook.md @@ -1,9 +1,9 @@ #PROXY Edge Investigation Notebook ###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) The following python modules will be imported for the notebook to work correctly: @@ -20,8 +20,8 @@ The following python modules will be imported for the notebook to work correctly ###Pre-requisites - Execution of the oni-oa process for Proxy -- Correct setup the duxbay.conf file -- Have a public key authentication between the current UI node and the ML node. You can follow these [instructions](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) +- Correct setup the duxbay.conf file [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Have a public key created between the current UI node and the ML node. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Configure%20User%20Accounts#configure-user-accounts) ###Data diff --git a/oa/proxy/ipynb_templates/ThreatInvestigation.md b/oa/proxy/ipynb_templates/ThreatInvestigation.md index 9375c73..ee034c6 100644 --- a/oa/proxy/ipynb_templates/ThreatInvestigation.md +++ b/oa/proxy/ipynb_templates/ThreatInvestigation.md @@ -1,9 +1,9 @@ #PROXY Threat Investigation Notebook ###Dependencies -- iPython == 3.2.1 [check documentation](https://ipython.org/ipython-doc/3/index.html) -- Python 2.7.6 -- ipywidgets +- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html) +- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/) +- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip) The following python modules will have to be imported for the notebook to work correctly: @@ -23,7 +23,7 @@ The following python modules will have to be imported for the notebook to work c ##Pre-requisites - Execution of the oni-oa process for Proxy - Score a set connections at the Edge Investigation Notebook -- Correct setup the duxbay.conf file. You can check this [link](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) +- Correct setup of the duxbay.conf file. [Read more](https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration) ##Additional Configuration From 3c49f0b519c8f28abe5cbb48068f000643e66cc9 Mon Sep 17 00:00:00 2001 From: Felipe Ruiz Date: Mon, 28 Nov 2016 15:57:43 -0200 Subject: [PATCH 3/4] Reviewing INSTALL.md --- ui/INSTALL.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/INSTALL.md b/ui/INSTALL.md index e3f2bb0..ebaa3ba 100755 --- a/ui/INSTALL.md +++ b/ui/INSTALL.md @@ -1,6 +1,6 @@ # Installation Guide -Open Network Insight User Interface (aka ONI UI or UI) Provides tools for interactive visualization, noise filters, white listing, and attack heuristics. +Open Network Insight User Interface (aka ONI UI or UI) provides tools for interactive visualization, noise filters, white listing, and attack heuristics. Here you will find instructions to get ONI UI up and running. For more information about ONI look [here](/Open-Network-Insight/open-network-insight). @@ -16,11 +16,11 @@ Here you will find instructions to get ONI UI up and running. For more informati 1. Go to ONI UI folder - `$ cd PATH_TO_ONI/ui/` + `$ cd PATH_TO_ONI/oni-oa/ui/` -2. With root privileges, install browserify and uglify as global commands on your system. +2. With root privileges, install [browserify](https://www.npmjs.com/package/browserify) and [uglify](https://www.npmjs.com/package/uglify) as global commands on your system. - `# npm install -g browserify uglifyjs` + `$ npm install -g browserify uglifyjs` 3. Install dependencies and build ONI UI @@ -28,9 +28,9 @@ Here you will find instructions to get ONI UI up and running. For more informati ## How to run ONI UI -1. Go to ONI UI folder +1. Go to ONI OA folder - `$ cd PATH_TO_ONI/` + `$ cd cd PATH_TO_ONI/oni-oa/` 2. Start the web server From 6a395eaa520972a4a2890b668d208439cf22455f Mon Sep 17 00:00:00 2001 From: Felipe Ruiz Date: Mon, 28 Nov 2016 15:58:18 -0200 Subject: [PATCH 4/4] Adding watchify instructions --- ui/INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/INSTALL.md b/ui/INSTALL.md index ebaa3ba..e86fa9a 100755 --- a/ui/INSTALL.md +++ b/ui/INSTALL.md @@ -18,9 +18,9 @@ Here you will find instructions to get ONI UI up and running. For more informati `$ cd PATH_TO_ONI/oni-oa/ui/` -2. With root privileges, install [browserify](https://www.npmjs.com/package/browserify) and [uglify](https://www.npmjs.com/package/uglify) as global commands on your system. +2. With root privileges, install [browserify](https://www.npmjs.com/package/browserify), [uglify](https://www.npmjs.com/package/uglify) and [watchify](https://www.npmjs.com/package/watchify) as global commands on your system. - `$ npm install -g browserify uglifyjs` + `$ npm install -g browserify uglifyjs watchify` 3. Install dependencies and build ONI UI