diff --git a/hosts b/hosts index a3f7031d..aae866f3 100644 --- a/hosts +++ b/hosts @@ -1,471 +1,2 @@ -osm11.openstreetmap.fr -osm12.openstreetmap.fr -osm13.openstreetmap.fr -osm23.openstreetmap.fr -osm24.openstreetmap.fr -osm25.openstreetmap.fr -osm29.openstreetmap.fr -osm30.openstreetmap.fr -osm31.openstreetmap.fr - -[vm] -# osm101.openstreetmap.fr vm_host=osm11.openstreetmap.fr -# osm102.openstreetmap.fr vm_host=osm11.openstreetmap.fr -osm104.openstreetmap.fr vm_host=osm12.openstreetmap.fr -# osm109.openstreetmap.fr vm_host=osm11.openstreetmap.fr -osm110.openstreetmap.fr vm_host=osm11.openstreetmap.fr # backend osmose (non générique france_local_db) -# osm111.openstreetmap.fr vm_host=osm12.openstreetmap.fr -# osm112.openstreetmap.fr vm_host=osm12.openstreetmap.fr -# osm116.openstreetmap.fr vm_host=osm12.openstreetmap.fr -# osm117.openstreetmap.fr vm_host=osm11.openstreetmap.fr -taginfo.vm.openstreetmap.fr vm_host=osm41.openstreetmap.fr # osm119.openstreetmap.fr -munin.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm127.openstreetmap.fr -live.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm132.openstreetmap.fr -listes.vm.openstreetmap.fr vm_host=osm35.openstreetmap.fr # osm138.openstreetmap.fr -nextcloud.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm141.openstreetmap.fr -www.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm142.openstreetmap.fr -prev.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm146.openstreetmap.fr -sotm.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr install_apache=true # osm154.openstreetmap.fr -peertube.vm.openstreetmap.fr vm_host=osm42.openstreetmap.fr # osm155.openstreetmap.fr -petitereine.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm157.openstreetmap.fr -mail.openstreetmap.fr vm_host=osm35.openstreetmap.fr -matomo.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm160.openstreetmap.fr -bano.vm.openstreetmap.fr vm_host=osm11.openstreetmap.fr # osm162.openstreetmap.fr -educosm.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm164.openstreetmap.fr -osm165.openstreetmap.fr vm_host=osm11.openstreetmap.fr -osm166.openstreetmap.fr vm_host=osm11.openstreetmap.fr -osm167.openstreetmap.fr vm_host=osm11.openstreetmap.fr -pic4review.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm168.openstreetmap.fr -osm169.openstreetmap.fr vm_host=osm41.openstreetmap.fr -download.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm171.openstreetmap.fr -osm173.openstreetmap.fr vm_host=osm11.openstreetmap.fr install_apache=true -projetdumois.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm174.openstreetmap.fr -buildbot.osmose.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm175.openstreetmap.fr -garmin.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm176.openstreetmap.fr -tile-vect.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm177.openstreetmap.fr -osmose-jupyter.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm178.openstreetmap.fr -comcommaker.vm.openstreetmap.fr vm_host=osm11.openstreetmap.fr # osm179.openstreetmap.fr -cadastre.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm180.openstreetmap.fr -mobilizon.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm181.openstreetmap.fr -cadastre-joker.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm182.openstreetmap.fr -stable-data.vm.openstreetmap.fr vm_host=osm11.openstreetmap.fr # osm183.openstreetmap.fr -backuppc2.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm184.openstreetmap.fr -mapcontrib.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm185.openstreetmap.fr -osmose.vm.openstreetmap.fr vm_host=osm41.openstreetmap.fr # osm210.openstreetmap.fr -batimap.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm187.openstreetmap.fr -osm2pgsql-osm12.vm.openstreetmap.fr vm_host=osm12.openstreetmap.fr # osm188.openstreetmap.fr -hdm-tiles.vm.openstreetmap.fr vm_host=osm12.openstreetmap.fr # osm189.openstreetmap.fr -layers.vm.openstreetmap.fr vm_host=osm12.openstreetmap.fr # osm190.openstreetmap.fr -commerces.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm191.openstreetmap.fr -proxy-free.vm.openstreetmap.fr vm_host=osm14.openstreetmap.fr # osm192.openstreetmap.fr -enedis.vm.openstreetmap.fr vm_host=osm34.openstreetmap.fr # osm193.openstreetmap.fr -notes-heatmap.vm.openstreetmap.fr vm_host=osm43.openstreetmap.fr # osm194.openstreetmap.fr -osm2pgsql.th3.vm.openstreetmap.fr vm_host=osm33.openstreetmap.fr -renderd.th3.vm.openstreetmap.fr vm_host=osm33.openstreetmap.fr -proxy.th3.vm.openstreetmap.fr vm_host=osm33.openstreetmap.fr # osm103.openstreetmap.fr -proxy.osmose.vm.openstreetmap.fr vm_host=osm34.openstreetmap.fr # osm196.openstreetmap.fr -geovisio.vm.openstreetmap.fr vm_host=osm33.openstreetmap.fr # osm197.openstreetmap.fr -umap.vm.openstreetmap.fr vm_host=osm34.openstreetmap.fr # osm199.openstreetmap.fr -pleinair.vm.openstreetmap.fr vm_host=osm40.openstreetmap.fr # osm201.openstreetmap.fr -postgres-free.vm.openstreetmap.fr vm_host=osm12.openstreetmap.fr # osm203.openstreetmap.fr -adopte-une-commune.vm.openstreetmap.fr vm_host=osm12.openstreetmap.fr # osm204.openstreetmap.fr -renderd-cyclosm.vm.openstreetmap.fr vm_host=osm36.openstreetmap.fr # osm205.openstreetmap.fr -renderd-bzh.vm.openstreetmap.fr vm_host=osm35.openstreetmap.fr # osm206.openstreetmap.fr -osm2pgsql-cyclosm.vm.openstreetmap.fr vm_host=osm36.openstreetmap.fr # osm207.openstreetmap.fr -osm2pgsql-bzh.vm.openstreetmap.fr vm_host=osm35.openstreetmap.fr # osm208.openstreetmap.fr -localdb.osmose.vm.openstreetmap.fr vm_host=osm41.openstreetmap.fr # osm209.openstreetmap.fr -proxy.moji.vm.openstreetmap.fr vm_host=osm39.openstreetmap.fr -osm211.openstreetmap.fr vm_host=osm29.openstreetmap.fr -osm212.openstreetmap.fr vm_host=osm30.openstreetmap.fr -osm213.openstreetmap.fr vm_host=osm31.openstreetmap.fr -geoosm.vm.openstreetmap.fr vm_host=osm42.openstreetmap.fr # osm214.openstreetmap.fr -dev.download.vm.openstreetmap.fr vm_host=osm44.openstreetmap.fr # osm215.openstreetmap.fr -crm.vm.openstreetmap.fr vm_host=osm41.openstreetmap.fr # osm216.openstreetmap.fr -overpass.vm.openstreetmap.fr vm_host=osm44.openstreetmap.fr # osm217.openstreetmap.fr -osmose-aquilenet.openstreetmap.fr -osmose-hivane1.openstreetmap.fr -osmose-hivane2.openstreetmap.fr -osmose1.davintech.ca -osmose2.davintech.ca - -[cluster-free] -osm11.openstreetmap.fr -osm12.openstreetmap.fr -osm14.openstreetmap.fr - -[cluster-th3] -osm32.openstreetmap.fr -osm33.openstreetmap.fr -osm34.openstreetmap.fr -osm35.openstreetmap.fr -osm36.openstreetmap.fr - -[cluster-moji] -osm38.openstreetmap.fr -osm39.openstreetmap.fr -osm40.openstreetmap.fr -osm41.openstreetmap.fr -osm42.openstreetmap.fr -osm43.openstreetmap.fr -osm44.openstreetmap.fr - -[cluster-dc1] -osm50.openstreetmap.fr - -[proxmox:children] -cluster-ovh -cluster-free -cluster-th3 -cluster-moji -cluster-dc1 - -[proxmox] -osm29.openstreetmap.fr -osm30.openstreetmap.fr -osm31.openstreetmap.fr - -# services -[backuppc-server] -backuppc2.vm.openstreetmap.fr - -[buildbot-master] -buildbot.osmose.vm.openstreetmap.fr buildbot_site=buildbot.osmose.openstreetmap.fr buildbot_nine=yes - -[buildbot-slave] -osm110.openstreetmap.fr -osm211.openstreetmap.fr -osm212.openstreetmap.fr -osm213.openstreetmap.fr -osmose-aquilenet.openstreetmap.fr -osmose-hivane1.openstreetmap.fr -osmose-hivane2.openstreetmap.fr -osmose1.davintech.ca -osmose2.davintech.ca - -[comcommaker] -comcommaker.vm.openstreetmap.fr - -[export-cadastre] -cadastre.vm.openstreetmap.fr - -[geoosm] -osm167.openstreetmap.fr - -[layers.openstreetmap.fr] - -[letsencrypt] -osm11.openstreetmap.fr letsencrypt_hosts=osm11.openstreetmap.fr -osm13.openstreetmap.fr letsencrypt_hosts=osm13.openstreetmap.fr,a.layers.openstreetmap.fr,b.layers.openstreetmap.fr,c.layers.openstreetmap.fr -osm14.openstreetmap.fr letsencrypt_hosts=osm14.openstreetmap.fr -osm103.openstreetmap.fr letsencrypt_hosts=osm103.openstreetmap.fr,analyser.openstreetmap.fr -osm104.openstreetmap.fr letsencrypt_hosts=osm104.openstreetmap.fr,bano.openstreetmap.fr,cadastre.openstreetmap.fr,imagery.openstreetmap.fr,tms.cadastre.openstreetmap.fr -mail.openstreetmap.fr letsencrypt_hosts=mail.openstreetmap.fr,osm158.openstreetmap.fr -osm24.openstreetmap.fr letsencrypt_hosts=proxy-ign.openstreetmap.fr -osm25.openstreetmap.fr -backuppc2.vm.openstreetmap.fr letsencrypt_hosts=backuppc2.vm.openstreetmap.fr -proxy.th3.vm.openstreetmap.fr letsencrypt_certbot=yes letsencrypt_hosts=proxy.th3.vm.openstreetmap.fr -proxy.moji.vm.openstreetmap.fr letsencrypt_certbot=yes letsencrypt_hosts=proxy.moji.vm.openstreetmap.fr - -[munin-server] -munin.vm.openstreetmap.fr - -[nextcloud] -nextcloud.vm.openstreetmap.fr - -[osm2pgsql] -#osm13.openstreetmap.fr ## osm2pgsql installé hors ansible -#osm25.openstreetmap.fr ## osm2pgsql installé hors ansible -osm165.openstreetmap.fr -osm2pgsql-osm12.vm.openstreetmap.fr # nom pas ideal. osm2pgsql.free ? -osm2pgsql.th3.vm.openstreetmap.fr -osm2pgsql-cyclosm.vm.openstreetmap.fr -osm2pgsql-bzh.vm.openstreetmap.fr - -[osmose-backend] -osm110.openstreetmap.fr -osm211.openstreetmap.fr -osm212.openstreetmap.fr -osm213.openstreetmap.fr -osmose-hivane1.openstreetmap.fr -osmose-hivane2.openstreetmap.fr -osmose-aquilenet.openstreetmap.fr -osmose1.davintech.ca -osmose2.davintech.ca -localdb.osmose.vm.openstreetmap.fr - -[osmose-frontend] -osmose.vm.openstreetmap.fr osmose_frontend_site=osmose.openstreetmap.fr - -[osmbin] -download.vm.openstreetmap.fr -dev.download.vm.openstreetmap.fr - -[osmosis] -osm110.openstreetmap.fr - -[peertube] -peertube.vm.openstreetmap.fr - -[polygons] -download.vm.openstreetmap.fr -dev.download.vm.openstreetmap.fr - -[postgresql] -postgres-free.vm.openstreetmap.fr -osm2pgsql-cyclosm.vm.openstreetmap.fr -osm2pgsql-bzh.vm.openstreetmap.fr - -[taginfo] -taginfo.vm.openstreetmap.fr - -[live] -live.vm.openstreetmap.fr - -[rawedit] - -[sympa] -listes.vm.openstreetmap.fr - -[proxmox-mail-gateway] -mail.openstreetmap.fr - -[proxycache] -osm23.openstreetmap.fr -osm24.openstreetmap.fr -osm25.openstreetmap.fr -proxy.moji.vm.openstreetmap.fr -proxy.th3.vm.openstreetmap.fr -proxy.osmose.vm.openstreetmap.fr - -[proxycache:children] -cluster-free - -[overpass-api] - -[proxmox-backup] -osm32.openstreetmap.fr proxmox_backup_host=true -osm34.openstreetmap.fr proxmox_backup_target="osm50.openstreetmap.fr:rpool/backups/cluster-th3" -osm39.openstreetmap.fr proxmox_backup_exclude="101 118 144 186 209 400 999 9000" proxmox_backup_target="osm32.openstreetmap.fr:backups/cluster-moji" -osm40.openstreetmap.fr proxmox_backup_exclude="101 118 144 186 209 400 999 9000" proxmox_backup_target="osm32.openstreetmap.fr:backups/cluster-moji" -osm41.openstreetmap.fr proxmox_backup_exclude="101 118 144 186 209 400 999 9000" proxmox_backup_target="osm32.openstreetmap.fr:backups/cluster-moji" -osm42.openstreetmap.fr proxmox_backup_exclude="101 118 144 186 209 400 999 9000" proxmox_backup_target="osm32.openstreetmap.fr:backups/cluster-moji" -osm43.openstreetmap.fr proxmox_backup_exclude="101 118 144 186 209 400 999 9000" proxmox_backup_target="osm32.openstreetmap.fr:backups/cluster-moji" -osm50.openstreetmap.fr proxmox_backup_host=true - -[renderd] -osm13.openstreetmap.fr renderd_layers=true -osm166.openstreetmap.fr renderd_cyclosm=true -renderd.th3.vm.openstreetmap.fr renderd_layers=true -renderd-cyclosm.vm.openstreetmap.fr renderd_cyclosm=true - -[wireguard] -osmose.vm.openstreetmap.fr wireguard_config=server wireguard_address=10.230.0.10 -osmose-aquilenet.openstreetmap.fr wireguard_server=osmose.vm.openstreetmap.fr wireguard_address=10.230.0.2 -osmose-hivane1.openstreetmap.fr wireguard_server=osmose.vm.openstreetmap.fr wireguard_address=10.230.0.3 -osmose-hivane2.openstreetmap.fr wireguard_server=osmose.vm.openstreetmap.fr wireguard_address=10.230.0.4 - -[wordpress] -www.vm.openstreetmap.fr - -[yeswiki] -educosm.vm.openstreetmap.fr - - -# user creation - -[achetta] -sotm.vm.openstreetmap.fr - -[apavie] # Adrien P. -osm14.openstreetmap.fr -projetdumois.vm.openstreetmap.fr -postgres-free.vm.openstreetmap.fr -adopte-une-commune.vm.openstreetmap.fr - -[bristow] -osm167.openstreetmap.fr - -[cgiquello] -notes-heatmap.vm.openstreetmap.fr -petitereine.vm.openstreetmap.fr - -[cormoran] -osm103.openstreetmap.fr - -[davidbgk] -umap.vm.openstreetmap.fr - -[driss] - -[eloks] - -[etiennejr] -overpass.vm.openstreetmap.fr - -[famlam] -osm14.openstreetmap.fr -osm110.openstreetmap.fr - -[flacombe] -osm34.openstreetmap.fr -osm110.openstreetmap.fr -enedis.vm.openstreetmap.fr -localdb.osmose.vm.openstreetmap.fr - -[francois2] # François de Metz - -[fred] -cadastre-joker.vm.openstreetmap.fr -localdb.osmose.vm.openstreetmap.fr -mapcontrib.vm.openstreetmap.fr -mobilizon.vm.openstreetmap.fr -osm14.openstreetmap.fr -osm104.openstreetmap.fr -osmose.vm.openstreetmap.fr -proxy.osmose.vm.openstreetmap.fr -taginfo.vm.openstreetmap.fr -osm173.openstreetmap.fr -tile-vect.vm.openstreetmap.fr -osmose-jupyter.vm.openstreetmap.fr -download.vm.openstreetmap.fr - -[fred:children] -osmose-backend - -[gileri] -osm11.openstreetmap.fr -comcommaker.vm.openstreetmap.fr - -[guillaumeamat] -mapcontrib.vm.openstreetmap.fr - -[gpelloux] -batimap.vm.openstreetmap.fr - -[hamlet] - -[jacques] -osm103.openstreetmap.fr -www.vm.openstreetmap.fr -prev.vm.openstreetmap.fr -garmin.vm.openstreetmap.fr - -[jben] - -[jgcailton] -osm104.openstreetmap.fr - -[juliep] -sotm.vm.openstreetmap.fr - -[karl] -geoosm.vm.openstreetmap.fr -osm167.openstreetmap.fr - -[lafeychine] -geoosm.vm.openstreetmap.fr -osm167.openstreetmap.fr - -[lcombe] - -[lefuturiste] -tile-vect.vm.openstreetmap.fr - -[ljbouere] -educosm.vm.openstreetmap.fr - -[nlehuby] - -[mael] -osm160.openstreetmap.fr -renderd-bzh.vm.openstreetmap.fr - [maelito] overpass.vm.openstreetmap.fr - -[marc_marc] -osm24.openstreetmap.fr -osm103.openstreetmap.fr # old overpass api -munin.vm.openstreetmap.fr -osm138.openstreetmap.fr -bano.vm.openstreetmap.fr - -[nmoyroud] -sotm.vm.openstreetmap.fr - -[panieravide] -commerces.vm.openstreetmap.fr -geovisio.vm.openstreetmap.fr -pic4review.vm.openstreetmap.fr - -[phyks] -osm11.openstreetmap.fr -osm165.openstreetmap.fr -osm166.openstreetmap.fr -renderd-cyclosm.vm.openstreetmap.fr renderd_cyclosm=true - -[pyrog] -mobilizon.vm.openstreetmap.fr -pleinair.vm.openstreetmap.fr - -[rvoirin] - -[ssh-tunnel] -osm13.openstreetmap.fr - -[stephanec] -pleinair.vm.openstreetmap.fr - -[teslix] -osm13.openstreetmap.fr -osm25.openstreetmap.fr -osm29.openstreetmap.fr -osm30.openstreetmap.fr -osm31.openstreetmap.fr -osm165.openstreetmap.fr - -[teslix:children] -buildbot-master -osmose-backend -osmose-frontend -osm2pgsql -postgresql -renderd - -[thibtib51] -crm.vm.openstreetmap.fr - -[thomasg77] -sotm.vm.openstreetmap.fr - -[thomasl] -peertube.vm.openstreetmap.fr - -[thorieux] -sotm.vm.openstreetmap.fr - -[thrrp] # Rodolphe P-P -osm103.openstreetmap.fr - -[tykayn] -proxy.moji.vm.openstreetmap.fr -mobilizon.vm.openstreetmap.fr -sotm.vm.openstreetmap.fr - -[tyndare] -osm104.openstreetmap.fr -cadastre.vm.openstreetmap.fr - -[vdct] -osm11.openstreetmap.fr -osm12.openstreetmap.fr -osm104.openstreetmap.fr -bano.vm.openstreetmap.fr - -[ybon] -proxy.th3.vm.openstreetmap.fr -osm104.openstreetmap.fr -umap.vm.openstreetmap.fr - -[ydel] -umap.vm.openstreetmap.fr diff --git a/overpass-api.yml b/overpass-api.yml index 39b8ec0d..3ccfdb42 100644 --- a/overpass-api.yml +++ b/overpass-api.yml @@ -1,8 +1,11 @@ - name: overpass-api.yml - hosts: overpass-api + hosts: maelito + remote_user: maelito roles: - role: overpass-api + vars: + overpass_version: v0.7.62.7 + overpass_database_dir: /data/work/overpass when: - overpass_version != "skip" - - ansible_distribution_release == "stretch" become: yes diff --git a/roles/overpass-api/defaults/main.yml b/roles/overpass-api/defaults/main.yml index c245c95e..abba2931 100644 --- a/roles/overpass-api/defaults/main.yml +++ b/roles/overpass-api/defaults/main.yml @@ -2,6 +2,7 @@ overpass_user: overpass overpass_version: skip overpass_pbf_region: planet +overpass_database_dir: /data/work/overpass overpass_pbf_root_url: https://planet.osm.org/pbf overpass_replication_url: https://planet.osm.org/replication/minute overpass_replication_interval: minute diff --git a/roles/overpass-api/tasks/compile.yml b/roles/overpass-api/tasks/compile.yml index fbbce197..3347cc00 100644 --- a/roles/overpass-api/tasks/compile.yml +++ b/roles/overpass-api/tasks/compile.yml @@ -23,7 +23,6 @@ become: yes become_user: "{{ overpass_user }}" - - name: download {{ overpass_version }} release of overpass-api unarchive: src: "http://dev.overpass-api.de/releases/osm-3s_{{ overpass_version }}.tar.gz" @@ -55,6 +54,7 @@ strip: 1 become: yes become_user: "{{ overpass_user }}" + when: false # Looks to me that the diff file used to patch used to target another script and fails in the following versions. Disabling it an hoping it was fixed / not so important :) - name: install overpass-api executables shell: make install @@ -78,6 +78,8 @@ - name: install rules for areas command: cp -r {{ project_dir }}/osm-3s_{{ overpass_version }}/rules {{ overpass_database_dir }}/ + become: yes + become_user: "{{ overpass_user }}" - name: change delay to fetch diff replace: diff --git a/roles/overpass-api/tasks/main.yml b/roles/overpass-api/tasks/main.yml index 9c032e65..7b2548b6 100644 --- a/roles/overpass-api/tasks/main.yml +++ b/roles/overpass-api/tasks/main.yml @@ -1,7 +1,7 @@ - name: check ansible version assert: that: - - "ansible_version.full | version_compare('2.3', '>=')" + - "ansible_version.full is version_compare('2.3', '>=')" msg: "ansible version must be >= 2.3, please update ansible." - include_tasks: ../../../shared/project-account.yml @@ -9,7 +9,6 @@ user: "{{ overpass_user }}" user_system: "yes" - - name: set overpass_database_dir set_fact: overpass_database_dir: "{{ ssd_dir|default(work_dir) }}/database" @@ -29,7 +28,21 @@ - "{{ overpass_webroot_dir }}" - import_tasks: compile.yml -- import_tasks: set_up_database.yml +# Deactivated, it looks like the "Fast Startup" option here https://overpass-api.de/no_frills.html#startup did remove the need for this... +# ... as long as the .de instance is online +# +#- import_tasks: set_up_database.yml + +# TODO meta option could be set in general options +- name: Run "Fast Startup" scenario + tags: fast-startup + shell: "/usr/local/bin/download_clone.sh --source=https://dev.overpass-api.de/api_drolbr/ --db-dir={{ overpass_database_dir }} --meta=yes &" + become_user: "{{ overpass_user }}" + become: yes + become_method: sudo + args: + chdir: "{{ project_dir }}/osm-3s_{{ overpass_version }}" + - import_tasks: services.yml - import_tasks: web.yml - import_tasks: munin.yml diff --git a/roles/overpass-api/tasks/set_up_database.yml b/roles/overpass-api/tasks/set_up_database.yml index 17168a8f..729b88c7 100644 --- a/roles/overpass-api/tasks/set_up_database.yml +++ b/roles/overpass-api/tasks/set_up_database.yml @@ -12,6 +12,7 @@ state: directory - name: install overpass-setup.py + tags: overpass-setup template: src: templates/overpass-setup.py dest: "/usr/local/bin/" diff --git a/roles/overpass-api/templates/overpass-dispatcher.service b/roles/overpass-api/templates/overpass-dispatcher.service index ec9e04cc..7987144d 100644 --- a/roles/overpass-api/templates/overpass-dispatcher.service +++ b/roles/overpass-api/templates/overpass-dispatcher.service @@ -5,7 +5,7 @@ AssertPathExists={{ overpass_database_dir }}/replicate_id [Service] Type=simple User={{ overpass_user }} -ExecStart=/usr/local/bin/dispatcher --osm-base --db-dir={{ overpass_database_dir }} --meta +ExecStart=/usr/local/bin/dispatcher --osm-base --db-dir={{ overpass_database_dir }} --meta --allow-duplicate-queries=yes ExecStop=/usr/local/bin/dispatcher --osm-base --terminate Restart=always RestartSec=30s diff --git a/roles/overpass-api/templates/overpass-setup.py b/roles/overpass-api/templates/overpass-setup.py index 4f167d16..1cc56c47 100644 --- a/roles/overpass-api/templates/overpass-setup.py +++ b/roles/overpass-api/templates/overpass-setup.py @@ -16,6 +16,7 @@ def parse_state_txt(url): + log.info(url) resp = requests.get(url) for line in resp.text.split('\n'): @@ -23,8 +24,8 @@ def parse_state_txt(url): sequence_number = line.split('=')[1] elif line.startswith('timestamp'): timestamp = line.split('=')[1] - - return (sequence_number, datetime.strptime(timestamp[:-5], + cleaned_timestamp = timestamp.replace("\:", ":") # Looks like the datetime is not escaped in the URL https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute/state.txt + return (sequence_number, datetime.strptime(cleaned_timestamp[:-5], '%Y-%m-%dT%H:%M')) @@ -68,6 +69,9 @@ def md5(fname): obj_datetime = pbf_datetime - delta url = '{{ overpass_replication_url }}/state.txt' +log.info(url) +log.info(obj_datetime) + repl_seq, repl_datetime = parse_state_txt(url) diff_datetime = obj_datetime - repl_datetime while diff_datetime > timedelta( @@ -76,10 +80,12 @@ def md5(fname): int(repl_seq) + int(diff_datetime.total_seconds() // divider)) url = '{{ overpass_replication_url }}/{}/{}/{}.state.txt'.format( repl_seq[:-6], repl_seq[-6:-3], repl_seq[-3:]) + repl_seq, repl_datetime = parse_state_txt(url) diff_datetime = obj_datetime - repl_datetime log.info('download pbf') +log.info(url) pbf_url = '{{ overpass_pbf_root_url }}/{{ overpass_pbf_region }}-{}.osm.pbf'.format( pbf_datetime.strftime('%y%m%d')) pbf_dest = '{{ overpass_database_dir }}/{{ overpass_pbf_region }}-{}.osm.pbf'.format(