From f9c3aee12f95a6aae7d2adfb7b9cb47b1983ec44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Mon, 19 Aug 2019 20:39:02 +0200 Subject: [PATCH 1/8] Add support for ENS --- plugins/ENS/ConfigPlugin.py | 26 +++++ plugins/ENS/ENSResolver.py | 161 +++++++++++++++++++++++++++++++ plugins/ENS/SiteManagerPlugin.py | 73 ++++++++++++++ plugins/ENS/UiRequestPlugin.py | 23 +++++ plugins/ENS/__init__.py | 3 + plugins/ENS/requirements.txt | 1 + 6 files changed, 287 insertions(+) create mode 100644 plugins/ENS/ConfigPlugin.py create mode 100644 plugins/ENS/ENSResolver.py create mode 100644 plugins/ENS/SiteManagerPlugin.py create mode 100644 plugins/ENS/UiRequestPlugin.py create mode 100644 plugins/ENS/__init__.py create mode 100644 plugins/ENS/requirements.txt diff --git a/plugins/ENS/ConfigPlugin.py b/plugins/ENS/ConfigPlugin.py new file mode 100644 index 000000000..6e6bb1622 --- /dev/null +++ b/plugins/ENS/ConfigPlugin.py @@ -0,0 +1,26 @@ +from Plugin import PluginManager + +@PluginManager.registerTo('ConfigPlugin') +class ConfigPlugin: + def createArguments(self): + localProviders = ['web3py://default-ipc-providers', 'ws://127.0.0.1:8546', 'http://127.0.0.1:8545'] + mainnetProviders = ['https://cloudflare-eth.com', 'wss://mainnet.infura.io/ws', 'https://mainnet.infura.io'] + ropstenProviders = ['wss://ropsten.infura.io/ws', 'https://ropsten.infura.io'] + rinkebyProviders = ['wss://rinkeby.infura.io/ws', 'https://rinkeby.infura.io'] + goerliProviders = ['wss://goerli.infura.io/ws', 'https://goerli.infura.io'] + + group = self.parser.add_argument_group('ENS plugin') + + group.add_argument('--ens_local_providers', help='Ethereum local providers for ENS plugin', default=localProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_mainnet_providers', help='Ethereum mainnet providers for ENS plugin', default=mainnetProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_ropsten_providers', help='Ethereum ropsten providers for ENS plugin', default=ropstenProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_rinkeby_providers', help='Ethereum rinkeby providers for ENS plugin', default=rinkebyProviders, metavar='protocol://address', nargs='*') + group.add_argument('--ens_goerli_providers', help='Ethereum goerli providers for ENS plugin', default=goerliProviders, metavar='protocol://address', nargs='*') + + group.add_argument('--ens_use_local', help='Use local providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_mainnet', help='Use mainnet providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_ropsten', help='Use ropsten providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_rinkeby', help='Use rinkeby providers for ENS plugin', action='store_true', default=True) + group.add_argument('--ens_use_goerli', help='Use goerli providers for ENS plugin', action='store_true', default=True) + + return super(ConfigPlugin, self).createArguments() diff --git a/plugins/ENS/ENSResolver.py b/plugins/ENS/ENSResolver.py new file mode 100644 index 000000000..185fcc5cd --- /dev/null +++ b/plugins/ENS/ENSResolver.py @@ -0,0 +1,161 @@ +from Config import config + +from web3 import HTTPProvider, WebsocketProvider, IPCProvider +from ens import ENS +import gevent + +from urllib.parse import urlparse +import logging +import time +import json +import re +import os + +FILE_SCHEMES = {'file'} +HTTP_SCHEMES = {'http', 'https'} +WS_SCHEMES = {'ws', 'wss'} + +log = logging.getLogger('ENSPlugin') + +class ENSResolver: + loaded = False + cache = {} + + def __init__(self, site_manager, networks): + self.site_manager = site_manager + self.networks = networks + + def load(self): + if not self.loaded: + self.loadCache() + + greenlets = [] + for network in self.networks: + greenlets.append(gevent.spawn(self.loadNetwork, network)) + gevent.joinall(greenlets) + + self.loaded = True + + def loadNetwork(self, network): + if not network['enabled']: + return + + for provider_uri in network['providers']: + provider_scheme = urlparse(provider_uri).scheme + provider_path = urlparse(provider_uri).path + + if provider_uri == 'web3py://default-ipc-providers': + provider = IPCProvider() + + elif provider_scheme in FILE_SCHEMES: + provider = IPCProvider(provider_path) + + elif provider_scheme in HTTP_SCHEMES: + provider = HTTPProvider(provider_uri) + + elif provider_scheme in WS_SCHEMES: + provider = WebsocketProvider(provider_uri) + + try: + if provider.isConnected(): + network['instance'] = ENS(provider) + return + except: + pass + + network['enabled'] = False + + def loadCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): + if os.path.isfile(path): + try: self.cache = json.load(open(path)) + except json.decoder.JSONDecodeError: pass + + def saveCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): + json.dump(self.cache, open(path, 'w'), indent=2) + + def isDomain(self, address): + return re.match(r'(.*?)([A-Za-z0-9_-]+\.eth)$', address) + + def resolveDomain(self, domain): + if not self.loaded: + self.load() + + domain = domain.lower() + + cache_entry = self.lookupCache(domain) + if cache_entry and time.time() < cache_entry['timeout']: + log.info('cache: %s -> %s', domain, cache_entry['address']) + return cache_entry['address'] + + provider_entry = None + provider_error = None + + try: + provider_entry = self.lookupProviders(domain) + except Exception as err: + provider_error = err + + if provider_entry and not provider_error: + log.info('provider: %s -> %s', domain, provider_entry['address']) + self.saveInCache(provider_entry) + return provider_entry['address'] + + if cache_entry and provider_error: + log.info('fallback: %s -> %s', domain, cache_entry['address']) + self.extendInCache(cache_entry) + return cache_entry['address'] + + def lookupCache(self, domain): + entry = self.cache.get(domain) + + if not entry: + return None + + for network in self.networks: + if entry['network'] == network['name'] and not network['enabled']: + return None + + return entry + + def lookupProviders(self, domain): + error = None + + greenlets = [] + for network in self.networks: + greenlets.append(gevent.spawn(self.lookupProvider, domain, network)) + gevent.joinall(greenlets) + + for greenlet in greenlets: + try: + entry = greenlet.get() + except Exception as err: + error = err + + if entry and entry['address']: + return entry + + if error: + raise error + + def lookupProvider(self, domain, network): + if not network['enabled']: + return None + + name = network['name'] + instance = network['instance'] + + content = instance.content(domain) + + if content and content['type'] == 'zeronet': + return {'network': name, 'domain': domain, 'address': content['hash']} + + def saveInCache(self, entry): + entry['timeout'] = time.time() + 60 * 60 + self.cache[entry['domain']] = entry + + self.saveCache() + + def extendInCache(self, entry): + self.cache[entry['domain']]['timeout'] = time.time() + 60 * 15 + + self.saveCache() diff --git a/plugins/ENS/SiteManagerPlugin.py b/plugins/ENS/SiteManagerPlugin.py new file mode 100644 index 000000000..ddb9ab4f9 --- /dev/null +++ b/plugins/ENS/SiteManagerPlugin.py @@ -0,0 +1,73 @@ +from Config import config +from Plugin import PluginManager + +from .ENSResolver import ENSResolver + +allow_reload = False + +@PluginManager.registerTo('SiteManager') +class SiteManagerPlugin: + _ens_resolver = None + + @property + def ens_resolver(self): + if not self._ens_resolver: + local = { + 'name': 'local', + 'providers': config.ens_local_providers, + 'enabled': config.ens_use_local, + 'instance': None, + } + + mainnet = { + 'name': 'mainnet', + 'providers': config.ens_mainnet_providers, + 'enabled': config.ens_use_mainnet, + 'instance': None, + } + + ropsten = { + 'name': 'ropsten', + 'providers': config.ens_ropsten_providers, + 'enabled': config.ens_use_ropsten, + 'instance': None, + } + + rinkeby = { + 'name': 'rinkeby', + 'providers': config.ens_rinkeby_providers, + 'enabled': config.ens_use_rinkeby, + 'instance': None, + } + + goerli = { + 'name': 'goerli', + 'providers': config.ens_goerli_providers, + 'enabled': config.ens_use_goerli, + 'instance': None, + } + + networks = [ + local, + mainnet, + ropsten, + rinkeby, + goerli, + ] + + self._ens_resolver = ENSResolver( + site_manager=self, + networks=networks, + ) + + return self._ens_resolver + + def load(self, *args, **kwargs): + super(SiteManagerPlugin, self).load(*args, **kwargs) + self.ens_resolver.load() + + def isDomain(self, address): + return self.ens_resolver.isDomain(address) or super(SiteManagerPlugin, self).isDomain(address) + + def resolveDomain(self, domain): + return self.ens_resolver.resolveDomain(domain) or super(SiteManagerPlugin, self).resolveDomain(domain) diff --git a/plugins/ENS/UiRequestPlugin.py b/plugins/ENS/UiRequestPlugin.py new file mode 100644 index 000000000..4eb449335 --- /dev/null +++ b/plugins/ENS/UiRequestPlugin.py @@ -0,0 +1,23 @@ +from Plugin import PluginManager + +import re + +@PluginManager.registerTo('UiRequest') +class UiRequestPlugin: + def __init__(self, *args, **kwargs): + from Site import SiteManager + self.site_manager = SiteManager.site_manager + + super(UiRequestPlugin, self).__init__(*args, **kwargs) + + def actionSiteMedia(self, path, **kwargs): + match = re.match(r'/media/(?P
[A-Za-z0-9-]+\.[A-Za-z0-9\.-]+)(?P/.*|$)', path) + + if match: + domain = match.group('address') + address = self.site_manager.ens_resolver.resolveDomain(domain) + + if address: + path = '/media/' + address + match.group('inner_path') + + return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs) diff --git a/plugins/ENS/__init__.py b/plugins/ENS/__init__.py new file mode 100644 index 000000000..ebadd4b98 --- /dev/null +++ b/plugins/ENS/__init__.py @@ -0,0 +1,3 @@ +from . import ConfigPlugin +from . import UiRequestPlugin +from . import SiteManagerPlugin diff --git a/plugins/ENS/requirements.txt b/plugins/ENS/requirements.txt new file mode 100644 index 000000000..0984b7103 --- /dev/null +++ b/plugins/ENS/requirements.txt @@ -0,0 +1 @@ +web3>=5.0.0,<6.0.0 From 4fa558881177fc17cc84c07d50a2e20960ac1d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Sat, 28 Sep 2019 18:00:23 +0200 Subject: [PATCH 2/8] Add plugin description --- plugins/ENS/plugin_info.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 plugins/ENS/plugin_info.json diff --git a/plugins/ENS/plugin_info.json b/plugins/ENS/plugin_info.json new file mode 100644 index 000000000..cb6c8d031 --- /dev/null +++ b/plugins/ENS/plugin_info.json @@ -0,0 +1,5 @@ +{ + "name": "ENS", + "description": "Support Ethereum Name Service as domain system.", + "default": "enabled" +} From 497f0b40494bfdb72596724b88c275c2ac323a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Mon, 30 Sep 2019 21:15:07 +0200 Subject: [PATCH 3/8] Install plugin dependencies in CI --- .gitlab-ci.yml | 1 + .travis.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b62c7b0c8..4c44fc2ac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ stages: - pip install --upgrade requests>=2.22.0 - pip install --upgrade codecov coveralls flake8 mock pytest==4.6.3 pytest-cov selenium - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done script: - pip list - openssl version -a diff --git a/.travis.yml b/.travis.yml index d88210013..c9c1efa4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ before_install: # - docker run -d -v $PWD:/root/data -p 15441:15441 -p 127.0.0.1:43110:43110 zeronet install: - pip install --upgrade -r requirements.txt + - for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip install --upgrade -r ${PLUGIN}; done - pip list before_script: - openssl version -a From 6beaa0ba6f7a8ce3ff4351b2365bca7080676306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Mon, 30 Sep 2019 21:37:56 +0200 Subject: [PATCH 4/8] Install plugin dependencies in Docker and Vagrant --- Dockerfile | 24 +++++++++++------------- Vagrantfile | 26 ++++++++++++++------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index b85d44f15..bf9d74190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,26 @@ FROM alpine:3.8 -#Base settings +# Base settings ENV HOME /root +WORKDIR /root -COPY requirements.txt /root/requirements.txt +# Add ZeroNet source +COPY . /root +VOLUME /root/data -#Install ZeroNet +# Install dependencies RUN apk --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ - && pip3 install -r /root/requirements.txt \ + && pip3 install -r requirements.txt \ + && for PLUGIN in $(ls plugins/[^disabled-]*/requirements.txt); do pip3 install -r ${PLUGIN}; done \ && apk del python3-dev gcc libffi-dev musl-dev make \ && echo "ControlPort 9051" >> /etc/tor/torrc \ && echo "CookieAuthentication 1" >> /etc/tor/torrc -#Add Zeronet source -COPY . /root -VOLUME /root/data - -#Control if Tor proxy is started +# Control if Tor proxy is started ENV ENABLE_TOR false -WORKDIR /root - -#Set upstart command +# Set upstart command CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 -#Expose ports +# Expose ports EXPOSE 43110 26552 diff --git a/Vagrantfile b/Vagrantfile index 24fe0c45f..eb8ff3dd9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,41 +5,43 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - #Set box - config.vm.box = "ubuntu/trusty64" + # Set box + config.vm.box = "ubuntu/bionic64" - #Do not check fo updates + # Do not check fo updates config.vm.box_check_update = false - #Add private network + # Add private network config.vm.network "private_network", type: "dhcp" - #Redirect ports + # Redirect ports config.vm.network "forwarded_port", guest: 43110, host: 43110 config.vm.network "forwarded_port", guest: 15441, host: 15441 - #Sync folder using NFS if not windows + # Sync folder using NFS if not windows config.vm.synced_folder ".", "/vagrant", :nfs => !Vagrant::Util::Platform.windows? - #Virtal Box settings + # Virtal Box settings config.vm.provider "virtualbox" do |vb| # Don't boot with headless mode - #vb.gui = true + vb.gui = false # Set VM settings vb.customize ["modifyvm", :id, "--memory", "512"] vb.customize ["modifyvm", :id, "--cpus", 1] end - #Update system + # Update system config.vm.provision "shell", inline: "sudo apt-get update -y && sudo apt-get upgrade -y" - #Install deps + # Install dependencies config.vm.provision "shell", - inline: "sudo apt-get install msgpack-python python-gevent python-pip python-dev -y" + inline: "sudo apt-get install python3 python3-pip python3-dev gcc libffi-dev musl-dev make -y" config.vm.provision "shell", - inline: "sudo pip install msgpack --upgrade" + inline: "sudo pip3 install -r /vagrant/requirements.txt" + config.vm.provision "shell", + inline: "for PLUGIN in $(ls /vagrant/plugins/[^disabled-]*/requirements.txt); do sudo pip3 install -r ${PLUGIN}; done" end From 7f8dca5e02f99baece47bc3b52b4ff35c7f963de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Sun, 3 Nov 2019 12:06:27 +0100 Subject: [PATCH 5/8] Use Web3.py temporary fork until PR is merged --- plugins/ENS/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ENS/requirements.txt b/plugins/ENS/requirements.txt index 0984b7103..bd7a7d716 100644 --- a/plugins/ENS/requirements.txt +++ b/plugins/ENS/requirements.txt @@ -1 +1 @@ -web3>=5.0.0,<6.0.0 +web3 @ git+https://github.com/filips123/web3.py@69d28362ca8fd27e6d19e08a3e98cfb039a6ea8f From 75563c10cbca27f6f0d139c18d251dab742ac333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Sun, 3 Nov 2019 12:07:29 +0100 Subject: [PATCH 6/8] Resolve requested changes --- plugins/ENS/ENSResolver.py | 9 ++++++--- plugins/ENS/SiteManagerPlugin.py | 14 +++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/plugins/ENS/ENSResolver.py b/plugins/ENS/ENSResolver.py index 185fcc5cd..270c60580 100644 --- a/plugins/ENS/ENSResolver.py +++ b/plugins/ENS/ENSResolver.py @@ -67,11 +67,14 @@ def loadNetwork(self, network): def loadCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): if os.path.isfile(path): - try: self.cache = json.load(open(path)) - except json.decoder.JSONDecodeError: pass + try: + self.cache = json.load(open(path)) + except json.decoder.JSONDecodeError: + pass def saveCache(self, path=os.path.join(config.data_dir, 'ens_cache.json')): - json.dump(self.cache, open(path, 'w'), indent=2) + with open(path, 'w') as file: + json.dump(self.cache, file, indent=2) def isDomain(self, address): return re.match(r'(.*?)([A-Za-z0-9_-]+\.eth)$', address) diff --git a/plugins/ENS/SiteManagerPlugin.py b/plugins/ENS/SiteManagerPlugin.py index ddb9ab4f9..54611f9b3 100644 --- a/plugins/ENS/SiteManagerPlugin.py +++ b/plugins/ENS/SiteManagerPlugin.py @@ -16,35 +16,35 @@ def ens_resolver(self): 'name': 'local', 'providers': config.ens_local_providers, 'enabled': config.ens_use_local, - 'instance': None, + 'instance': None } mainnet = { 'name': 'mainnet', 'providers': config.ens_mainnet_providers, 'enabled': config.ens_use_mainnet, - 'instance': None, + 'instance': None } ropsten = { 'name': 'ropsten', 'providers': config.ens_ropsten_providers, 'enabled': config.ens_use_ropsten, - 'instance': None, + 'instance': None } rinkeby = { 'name': 'rinkeby', 'providers': config.ens_rinkeby_providers, 'enabled': config.ens_use_rinkeby, - 'instance': None, + 'instance': None } goerli = { 'name': 'goerli', 'providers': config.ens_goerli_providers, 'enabled': config.ens_use_goerli, - 'instance': None, + 'instance': None } networks = [ @@ -52,12 +52,12 @@ def ens_resolver(self): mainnet, ropsten, rinkeby, - goerli, + goerli ] self._ens_resolver = ENSResolver( site_manager=self, - networks=networks, + networks=networks ) return self._ens_resolver From f6192afe1949733f18cc9f332527f8048f39be43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Sun, 3 Nov 2019 12:07:54 +0100 Subject: [PATCH 7/8] Remove UiRequestPlugin from ENS plugin --- plugins/ENS/UiRequestPlugin.py | 23 ----------------------- plugins/ENS/__init__.py | 1 - 2 files changed, 24 deletions(-) delete mode 100644 plugins/ENS/UiRequestPlugin.py diff --git a/plugins/ENS/UiRequestPlugin.py b/plugins/ENS/UiRequestPlugin.py deleted file mode 100644 index 4eb449335..000000000 --- a/plugins/ENS/UiRequestPlugin.py +++ /dev/null @@ -1,23 +0,0 @@ -from Plugin import PluginManager - -import re - -@PluginManager.registerTo('UiRequest') -class UiRequestPlugin: - def __init__(self, *args, **kwargs): - from Site import SiteManager - self.site_manager = SiteManager.site_manager - - super(UiRequestPlugin, self).__init__(*args, **kwargs) - - def actionSiteMedia(self, path, **kwargs): - match = re.match(r'/media/(?P
[A-Za-z0-9-]+\.[A-Za-z0-9\.-]+)(?P/.*|$)', path) - - if match: - domain = match.group('address') - address = self.site_manager.ens_resolver.resolveDomain(domain) - - if address: - path = '/media/' + address + match.group('inner_path') - - return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs) diff --git a/plugins/ENS/__init__.py b/plugins/ENS/__init__.py index ebadd4b98..473988653 100644 --- a/plugins/ENS/__init__.py +++ b/plugins/ENS/__init__.py @@ -1,3 +1,2 @@ from . import ConfigPlugin -from . import UiRequestPlugin from . import SiteManagerPlugin From 83f864e78060c4b309dbef7bc7d79dfea47bc8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Mon, 3 Feb 2020 20:10:40 +0100 Subject: [PATCH 8/8] Update Web3.py to new version --- plugins/ENS/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/ENS/requirements.txt b/plugins/ENS/requirements.txt index bd7a7d716..b7c62127b 100644 --- a/plugins/ENS/requirements.txt +++ b/plugins/ENS/requirements.txt @@ -1 +1 @@ -web3 @ git+https://github.com/filips123/web3.py@69d28362ca8fd27e6d19e08a3e98cfb039a6ea8f +web3 @ git+https://github.com/filips123/web3.py@abb32516a3898ace515f2162374be387b2733403