From 384a906e632c554a3e0540ab442d33a27cec1ee7 Mon Sep 17 00:00:00 2001 From: Adrian Brandemuehl Date: Fri, 1 Apr 2022 18:10:22 +0200 Subject: [PATCH] Remove dropbox code --- README.md | 21 +- deploy/api-server.docker | 1 + openapi/top.yaml | 10 +- rbb_server/README.md | 16 +- rbb_server/src/rbb_server/schema.sql | 2 - .../rbb_server_test/test_configuration.py | 2 +- .../test/rbb_server_test/test_storage.py | 4 +- .../test/rbb_server_test/test_stores.py | 2 +- rbb_storage/requirements.txt | 1 - .../src/rbb_storage_dropbox/__init__.py | 0 rbb_storage/src/rbb_storage_dropbox/plugin.py | 220 ------------------ 11 files changed, 28 insertions(+), 251 deletions(-) delete mode 100644 rbb_storage/src/rbb_storage_dropbox/__init__.py delete mode 100644 rbb_storage/src/rbb_storage_dropbox/plugin.py diff --git a/README.md b/README.md index eb502fe..526b551 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # RBB Core -This repository contains the core of the Rosbag Bazaar (RBB), a tool to index/visualize/manage rosbags on remote storage systems. Additionally it provides a web interface and framework for automated simulations. A work queue similar to most continuous integration systems allows processing of long running compute intensive jobs for both rosbag visualization and simulation. The aim is to keep the underlying storage system completely abstracted to allow development of plugins for most cloud based storage systems. (Currently only AWS S3 compatible storage is fully supported, and Dropbox is partially supported) +This repository contains the core of the Rosbag Bazaar (RBB), a tool to index/visualize/manage rosbags on remote storage systems. Additionally it provides a web interface and framework for automated simulations. A work queue similar to most continuous integration systems allows processing of long running compute intensive jobs for both rosbag visualization and simulation. The aim is to keep the underlying storage system completely abstracted to allow development of plugins for most cloud based storage systems. (Currently only AWS S3 compatible storage is fully supported) If you are using RViz to look at the content of your bag files, and want to have this visualization recorded for every rosbag automatically, and the recordings available on any computer with internet access. This software might be something for you. @@ -9,29 +9,29 @@ If you currently simulate your robot with for example Gazebo, and want to automa ![alt text](docs/s2.png "Screenshot 2") -This software has been developed on **Ubuntu 16.04** with **ROS Kinetic** (Newer versions of ROS (NOT ROS 2) will +This software has been developed on **Ubuntu 16.04** with **ROS Kinetic** (Newer versions of ROS (NOT ROS 2) will most likely work without problems). The web interface is only tested with **Google Chrome**. ## Quickstart -The best way to find out if this software is in anyway useful for your project, is to run it on your own computer. This can be done in a few simple steps. (A fast internet connection is required, since several gigabytes need to be downloaded) +The best way to find out if this software is in anyway useful for your project, is to run it on your own computer. This can be done in a few simple steps. (A fast internet connection is required, since several gigabytes need to be downloaded) This will deploy RBB with the public images from Docker Hub. If you want to build your own images you can do so with the `build-containers.sh` script and change the `docker-compose.yaml` file accordingly. 1. Install docker, if you haven't got it already. [Follow the instructions here](https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-using-the-repository) 2. Install docker-compose, [Follow the instructions here](https://docs.docker.com/compose/install/#install-compose) -3. Clone this repository: +3. Clone this repository: `git clone https://github.com/AMZ-Driverless/rbb_core.git` - -4. Run docker-compose in the local-stack subdirectory: + +4. Run docker-compose in the local-stack subdirectory: `cd rbb_core/deploy/local-stack && docker-compose up` - -5. In your browser go to `http://localhost` and login + +5. In your browser go to `http://localhost` and login with user *admin* and password *admin* - + 6. The demo setup comes pre-configured with several external sources. - + 7. Data is stored in a local [minio](https://min.io/) server. The control panel can be accessed on `http://localhost:9000` (login with MINIOKEY / MINIOSUPERSECRET) ## Development @@ -42,7 +42,6 @@ This will deploy RBB with the public images from Docker Hub. If you want to buil * **rbb_server/rbb_swagger_server** (Python 3.5): Auto generated swagger server * **rbb_client** (Python 2.7 & 3.5): Auto generated swagger API client * **rbb_storage/rbb_storage** (Python 2.7 & 3.5): Storage abstraction layer -* **rbb_storage/rbb_storage_dropbox** (Python 2.7 & 3.5): Integration with dropbox * **rbb_storage/rbb_storage_s3** (Python 2.7 & 3.5): Integration with AWS S3 * **rbb_tools** (Python 2.7): Actual generation of content from the rosbags diff --git a/deploy/api-server.docker b/deploy/api-server.docker index ec6c922..ec13136 100644 --- a/deploy/api-server.docker +++ b/deploy/api-server.docker @@ -9,5 +9,6 @@ COPY ./rbb_storage /var/app/rbb_storage RUN pip install -r /var/app/rbb_server/requirements.txt RUN pip install -r /var/app/rbb_storage/requirements.txt +RUN pip install -r /var/app/rbb_server/test-requirements.txt CMD /var/app/rbb_server/run-server diff --git a/openapi/top.yaml b/openapi/top.yaml index b966364..827212e 100644 --- a/openapi/top.yaml +++ b/openapi/top.yaml @@ -1924,7 +1924,7 @@ definitions: example: 'BagStoreDetailed' name: type: string - example: my-dropbox + example: my-s3 required: - detail_type - name @@ -1941,7 +1941,7 @@ definitions: store_type: type: string description: Type of store. - example: dropbox + example: s3 store_data: type: object description: Data that is specific to the store type. @@ -2154,7 +2154,7 @@ definitions: store_type: type: string description: Type of store. - example: dropbox + example: s3 store_data: type: object description: Data that is specific to the store type. @@ -2178,7 +2178,7 @@ definitions: store_name: type: string description: Name of the file store - example: 'dropbox' + example: 's3' uid: type: integer description: Unique file id @@ -2484,4 +2484,4 @@ definitions: type: boolean required: - identifier - - name \ No newline at end of file + - name diff --git a/rbb_server/README.md b/rbb_server/README.md index 2e8ade1..6b28a35 100644 --- a/rbb_server/README.md +++ b/rbb_server/README.md @@ -1,11 +1,11 @@ # Rosbag Bazaar API Server -The bazaar API server is the central core of the rosbag bazaar. All information about rosbags, -their contents and their location is stored here. It is made accessible to all other services requiring -this information using a "standard" API. The API is documented in swagger/openAPI format and can be -found in the `/openapi` folder. A server stub is generated from this specification and is stored in +The bazaar API server is the central core of the rosbag bazaar. All information about rosbags, +their contents and their location is stored here. It is made accessible to all other services requiring +this information using a "standard" API. The API is documented in swagger/openAPI format and can be +found in the `/openapi` folder. A server stub is generated from this specification and is stored in the `/rbb_server/src/rbb_swagger_server` folder. Do not modify anything in this folder because if we update the specification -we want to be able to just copy a newly generated server stub there. +we want to be able to just copy a newly generated server stub there. The custom non-generated code is in the `/rbb_server/src/rbb_server` folder. ## Requirements @@ -20,10 +20,10 @@ This will use the flasks built-in server. Or, the server can be run in productio as the WSGI server. Use the `/rbb_server/run-server` script for that. ## Testing server -To run the testing server, please execute the script in `bazaar_test/test_server.py`. This will -create a new `unittest` schema in the database filled with the data in `bazaar_test/test-data.sql`. It +To run the testing server, please execute the script in `test/rbb_server_test/test_server.py`. This will +create a new `unittest` schema in the database filled with the data in `test/rbb_server_test/test-data.sql`. It can be reached here: ``` http://localhost:8081/api/v0 -``` \ No newline at end of file +``` diff --git a/rbb_server/src/rbb_server/schema.sql b/rbb_server/src/rbb_server/schema.sql index efaad2c..7a7b1f2 100644 --- a/rbb_server/src/rbb_server/schema.sql +++ b/rbb_server/src/rbb_server/schema.sql @@ -88,8 +88,6 @@ CREATE TABLE "configuration" ( description VARCHAR(255) NOT NULL DEFAULT '' ); -INSERT INTO "configuration" (config_key, value, description) VALUES ('secret.dropbox.app_key' , '', ''); -INSERT INTO "configuration" (config_key, value, description) VALUES ('secret.dropbox.app_secret' , '', ''); INSERT INTO "configuration" (config_key, value, description) VALUES ('secret.jenkins.user' , '', ''); INSERT INTO "configuration" (config_key, value, description) VALUES ('secret.jenkins.password' , '', ''); INSERT INTO "configuration" (config_key, value, description) VALUES ('worker.default.poll_interval' , '10', ''); diff --git a/rbb_server/test/rbb_server_test/test_configuration.py b/rbb_server/test/rbb_server_test/test_configuration.py index 2af3972..b5b8e3a 100644 --- a/rbb_server/test/rbb_server_test/test_configuration.py +++ b/rbb_server/test/rbb_server_test/test_configuration.py @@ -31,7 +31,7 @@ def test_read_all_admin(self): api_instance = self.get_admin_api() config = dict(api_instance.get_configuration_key("*")) - self.assertDictEqual(config['secret']['dropbox'], { + self.assertDictEqual(config['secret']['s3'], { 'app_key': "", 'app_secret': "" }) diff --git a/rbb_server/test/rbb_server_test/test_storage.py b/rbb_server/test/rbb_server_test/test_storage.py index 250102d..9a66fb2 100644 --- a/rbb_server/test/rbb_server_test/test_storage.py +++ b/rbb_server/test/rbb_server_test/test_storage.py @@ -50,7 +50,7 @@ def test_insert_store(self): test_store = RosbagStore( name="some-test-store", description="test", - store_type="dropbox", + store_type="s3", store_data="{}" ) Database.get_session().add(test_store) @@ -62,7 +62,7 @@ def test_update_store(self): test_store = RosbagStore( name="some-test-store-update", description="test", - store_type="dropbox", + store_type="s3", store_data="{}" ) Database.get_session().add(test_store) diff --git a/rbb_server/test/rbb_server_test/test_stores.py b/rbb_server/test/rbb_server_test/test_stores.py index 55112c7..5c93935 100644 --- a/rbb_server/test/rbb_server_test/test_stores.py +++ b/rbb_server/test/rbb_server_test/test_stores.py @@ -200,7 +200,7 @@ def test_create_new_bag(self): bag = BagDetailed() bag.detail_type="BagDetailed" bag.name="unittest.bag" - bag.store_data={'dropbox': {'uuid': 23094, 'path': '/test/test/test'}} + bag.store_data={'s3': {'uuid': 23094, 'path': '/test/test/test'}} bag.is_extracted=True bag.meta_available=True bag.discovered=datetime.datetime.now(datetime.timezone.utc) diff --git a/rbb_storage/requirements.txt b/rbb_storage/requirements.txt index 5750005..900c21f 100644 --- a/rbb_storage/requirements.txt +++ b/rbb_storage/requirements.txt @@ -1,3 +1,2 @@ boto3 == 1.7.1 botocore == 1.10.1 -dropbox == 8.5.1 \ No newline at end of file diff --git a/rbb_storage/src/rbb_storage_dropbox/__init__.py b/rbb_storage/src/rbb_storage_dropbox/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rbb_storage/src/rbb_storage_dropbox/plugin.py b/rbb_storage/src/rbb_storage_dropbox/plugin.py deleted file mode 100644 index 1ba2188..0000000 --- a/rbb_storage/src/rbb_storage_dropbox/plugin.py +++ /dev/null @@ -1,220 +0,0 @@ -# AMZ-Driverless -# Copyright (c) 2019 Authors: -# - Huub Hendrikx -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import os - -import dropbox -import dropbox.files -import dropbox.oauth - -import rbb_storage - - -class DropboxStoragePlugin(rbb_storage.StoragePluginBase): - - def __init__(self, name, data, config_provider): - super(DropboxStoragePlugin, self).__init__(name, data, config_provider) - - if not 'dropbox' in data: - data['dropbox'] = {} - - self._plugin_data = data['dropbox'] - self._dbx = None - - def list_files(self): - dbx = self._get_dbx() - folder = self._plugin_data['path'] - folder_lower_case = folder.lower() - dir = dbx.files_list_folder(folder, recursive=True) - - files=[] - while True: - for file in dir.entries: - if isinstance(file, dropbox.files.FileMetadata): - if folder_lower_case == file.path_lower[0:len(folder_lower_case)]: - relative_name = file.path_lower[len(folder_lower_case):] - - safe_name = relative_name - if safe_name[0] == '/' or safe_name[0] == '\\': - safe_name = safe_name[1:] - - safe_name = safe_name.replace('/', '_') - safe_name = safe_name.replace('\\', '_') - - files.append(rbb_storage.StoredFile( - safe_name, - file.name, - relative_name, - { - 'id': file.id, - 'path': file.path_lower - } - )) - - if dir.has_more: - dir = dbx.files_list_folder_continue(dir.cursor) - else: - break - - return files - - def list_file(self, file_data): - # TODO: Extract this into a function together with the code in list_files - folder = self._plugin_data['path'] - file_path = file_data['path'] - folder_lower_case = folder.lower() - relative_name = file_path[len(folder_lower_case):] - - safe_name = relative_name - if safe_name[0] == '/' or safe_name[0] == '\\': - safe_name = safe_name[1:] - - safe_name = safe_name.replace('/', '_') - safe_name = safe_name.replace('\\', '_') - - return rbb_storage.StoredFile( - safe_name, - os.path.basename(relative_name), - relative_name, - file_data - ) - - def download_link(self, file_data): - dbx = self._get_dbx() - - if 'path' not in file_data: - raise RuntimeError("Path not defined in file data") - - result = dbx.files_get_temporary_link(file_data['path']) - return result.link - - def delete_file(self, file_data): - raise NotImplementedError() - - def store_file(self, local_path, mime_type, name_hint="", directory_hint="", progress_indicator=True): - raise NotImplementedError() - - def _is_linked_to_account(self): - return 'token' in self._plugin_data and self._plugin_data['token'] != ""; - - def _set_account_token(self, token): - self._plugin_data['token'] = token - - def _get_dbx(self): - if self._dbx is None: - self._dbx = dropbox.Dropbox(self._plugin_data['token']) - return self._dbx - - def new(self, command_line_args): - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('-p', '--path', default="", help="Root path of the store") - parser.add_argument('-t', '--token', default="", help="Authorization token") - args = parser.parse_args(command_line_args) - - self._plugin_data['path'] = args.path - - if args.token: - self._plugin_data['token'] = args.token - return False - else: - self._plugin_data['token'] = "" - return True - - def authorize_get_step(self, step, flask_request, url): - if self._is_linked_to_account(): - return "Already authorized" - - import flask - - if step == "0": - return self.authorize_step_0(flask_request, url) - else: - return flask.redirect(url + "0", code=302) - - def authorize_post_step(self, step, flask_request, url): - if self._is_linked_to_account(): - return "Already authorized" - - import flask - - if step == "1": - return self.authorize_step_1(flask_request, url) - else: - return flask.redirect(url + "0", code=302) - - def _get_auth_flow(self): - config = self._config_provider.get_configuration_key("secret.dropbox") - if not 'secret' in config or not 'dropbox' in config['secret']: - raise RuntimeError("Config keys in secret.dropbox missing!") - - consumer_key = config['secret']['dropbox']['app_key'] - consumer_secret = config['secret']['dropbox']['app_secret'] - flow = dropbox.oauth.DropboxOAuth2FlowNoRedirect(consumer_key, consumer_secret) - return flow - - def authorize_step_0(self, flask_request, url): - - flow = self._get_auth_flow() - dropbox_auth_url = flow.start() - next_step_url = url + "1" - - page_html = """ - - Dropbox Authorization - -

Please get a dropbox authorization code at the following link, %s

-

Paste the code here: -

- -
-

- - - """ % (dropbox_auth_url, dropbox_auth_url, next_step_url) - - import flask - return flask.Response(page_html) - - def authorize_step_1(self, flask_request, url): - code = flask_request.form.get('dropbox_code') - - if code: - try: - flow = self._get_auth_flow() - result = flow.finish(code) - - if result.access_token: - self._set_account_token(result.access_token) - self.save() - return "Authorization succeeded!" - else: - return "Authorization failed!" - except Exception as e: - logging.exception("Dropbox authorization failed", e) - return "Error occured (" + str(e) + ")" - else: - return "Invalid code!" - - -plugin = DropboxStoragePlugin \ No newline at end of file