Skip to content

Valhalla fix #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
farfromrefug wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Valhalla fix #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3188a56
fix: add bbox to packages
farfromrefug Apr 20, 2020
c305386
fix: fix retrieving valhalla tiles. also much faster
farfromrefug Apr 20, 2020
353318d
chore: cleanup
farfromrefug Apr 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions scripts/build_poly_tilemasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,33 @@
import argparse
import utils.polygon2geojson as polygon2geojson
import utils.tilemask as tilemask
import functools
import re
import geojson

TILEMASK_SIZE_THRESHOLD = 512

def bbox(coord_list):
box = []
for i in (0,1):
res = sorted(coord_list, key=lambda x:x[i])
box.append((res[0][i],res[-1][i]))
return box


def read_polygon(polygon_filename):
with open(polygon_filename) as f:
return f.readlines()

def clean_polygon(polygon_data):
coordinates = polygon_data[2:][:-2]
coordinates = [re.split(r'[\s\t]+', item) for item in coordinates]
coordinates = [list(filter(None, item)) for item in coordinates]
coordinates = functools.reduce(lambda a,b: a[-1].pop(0) and a if len(a[-1]) == 1 and a[-1][0] == 'END' else a.append(['END']) or a if b[0].startswith('END') else a[-1].append(b) or a, [[[]]] + coordinates)
coordinates = [[(float(item[0]), float(item[1])) for item in coordgroup] for coordgroup in coordinates]
return coordinates


def main():
parser = argparse.ArgumentParser()
parser.add_argument(dest='input', help='Input directory for .poly files')
Expand All @@ -32,11 +56,16 @@ def main():
if len(mask) >= TILEMASK_SIZE_THRESHOLD:
break

polygon_data = read_polygon(polyFilename)
coordinates = clean_polygon(polygon_data)
poly=geojson.Polygon(coordinates)
line = bbox(list(geojson.utils.coords(poly)))
packages.append(
{
'id': packageId,
'version': 1,
'tile_mask': str(mask, 'utf8'),
'bbox': (line[0][0], line[1][0], line[0][1],line[1][1]),
'url': '',
'metainfo': { 'name_en': packageName },
'size': 0
Expand Down
182 changes: 63 additions & 119 deletions scripts/build_valhalla_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,87 +21,9 @@
# Default package version
DEFAULT_PACKAGE_VERSION = 1

# Zoom level/precision for tilemasks
TILEMASK_ZOOM = 10

# Generic projection values
VALHALLA_BOUNDS = ((-180, -90), (180, 90))
VALHALLA_TILESIZES = [4.0, 1.0, 0.25]
MERCATOR_BOUNDS = ((-6378137 * math.pi, -6378137 * math.pi), (6378137 * math.pi, 6378137 * math.pi))

class PackageTileMask(object):
def __init__(self, tileMaskStr):
self.data = self._decodeTileMask(tileMaskStr)
self.rootNode = self._buildTileNode(list(self.data), (0, 0, 0))

def contains(self, tile):
node = self._findTileNode(tile)
if node is None:
return False
return node["inside"]

def getTiles(self, maxZoom=None):
tiles = []
if self.data != []:
self._buildTiles(list(self.data), (0, 0, 0), maxZoom, tiles)
return tiles

def _decodeTileMask(self, tileMaskStr):
str = [c for c in base64.b64decode(tileMaskStr)]
data = []
for i in range(len(str) * 8):
val = (str[i // 8] >> (7 - i % 8)) & 1
data.append(val)
return data

def _buildTileNode(self, data, tile):
(zoom, x, y) = tile
subtiles = data.pop(0)
inside = data.pop(0)
node = { "tile" : tile, "inside": inside, "subtiles": [] }
if subtiles:
for dy in range(0, 2):
for dx in range(0, 2):
node["subtiles"].append(self._buildTileNode(data, (zoom + 1, x * 2 + dx, y * 2 + dy)))
return node

def _findTileNode(self, tile):
(zoom, x, y) = tile
if zoom == 0:
return self.rootNode if tile == (0, 0, 0) else None

parentNode = self._findTileNode((zoom - 1, x >> 1, y >> 1))
if parentNode:
for node in parentNode["subtiles"]:
if node["tile"] == tile:
return node
if parentNode["inside"]:
return parentNode
return None

def _buildTiles(self, data, tile, maxZoom, tiles):
(zoom, x, y) = tile
submask = data.pop(0)
inside = data.pop(0)
if inside:
tiles.append(tile)
if submask:
for dy in range(0, 2):
for dx in range(0, 2):
self._buildTiles(data, (zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles)
elif maxZoom is not None and inside:
for dy in range(0, 2):
for dx in range(0, 2):
self._buildAllTiles((zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles)

def _buildAllTiles(self, tile, maxZoom, tiles):
(zoom, x, y) = tile
if zoom > maxZoom:
return
tiles.append(tile)
for dy in range(0, 2):
for dx in range(0, 2):
self._buildAllTiles((zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles)

def valhallaTilePath(vTile):
vTileSize = VALHALLA_TILESIZES[vTile[2]]
Expand All @@ -114,34 +36,6 @@ def valhallaTilePath(vTile):
splitId = [str(vTile[2])] + splitId
return '/'.join(splitId) + '.gph'

def _calculateValhallaTiles(mTile, vZoom, epsg3857, epsg4326):
mTileSize = (MERCATOR_BOUNDS[1][0] - MERCATOR_BOUNDS[0][0]) / (1 << mTile[2])
vTileSize = VALHALLA_TILESIZES[vZoom]
mX0, mY0 = mTile[0] * mTileSize + MERCATOR_BOUNDS[0][0], mTile[1] * mTileSize + MERCATOR_BOUNDS[0][1]
mX1, mY1 = mX0 + mTileSize, mY0 + mTileSize
vX0, vY0 = pyproj.transform(epsg3857, epsg4326, mX0, mY0)
vX1, vY1 = pyproj.transform(epsg3857, epsg4326, mX1, mY1)
vTile0 = (vX0 - VALHALLA_BOUNDS[0][0]) / vTileSize, (vY0 - VALHALLA_BOUNDS[0][1]) / vTileSize
vTile1 = (vX1 - VALHALLA_BOUNDS[0][0]) / vTileSize, (vY1 - VALHALLA_BOUNDS[0][1]) / vTileSize
vTiles = []
for y in range(int(math.floor(vTile0[1])), int(math.ceil(vTile1[1]))):
for x in range(int(math.floor(vTile0[0])), int(math.ceil(vTile1[0]))):
vTiles.append((x, y, vZoom))
return vTiles

def calculateValhallaTilesFromTileMask(tileMask):
vTiles = set()
mTiles = [(x, y, zoom) for zoom, x, y in PackageTileMask(tileMask).getTiles(TILEMASK_ZOOM)]
epsg3857 = pyproj.Proj(init='EPSG:3857')
epsg4326 = pyproj.Proj(init='EPSG:4326')
for mTile in mTiles:
if mTile[2] < TILEMASK_ZOOM:
continue
for vZoom, vTileSize in enumerate(VALHALLA_TILESIZES):
for vTile in _calculateValhallaTiles(mTile, vZoom, epsg3857, epsg4326):
vTiles.add(vTile)
return sorted(list(vTiles))

def compressTile(tileData, zdict=None):
if zdict is not None:
compress = zlib.compressobj(9, zlib.DEFLATED, -15, 9, zlib.Z_DEFAULT_STRATEGY, zdict)
Expand All @@ -163,17 +57,70 @@ def loadZDict(packageId, zdictDir):
print('Warning: Could not find dictionary for package %s!' % packageId)
return None

def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=None):

valhalla_tiles = [{'level': 2, 'size': 0.25}, {'level': 1, 'size': 1.0}, {'level': 0, 'size': 4.0}]
LEVEL_BITS = 3
TILE_INDEX_BITS = 22
ID_INDEX_BITS = 21
LEVEL_MASK = (2**LEVEL_BITS) - 1
TILE_INDEX_MASK = (2**TILE_INDEX_BITS) - 1
ID_INDEX_MASK = (2**ID_INDEX_BITS) - 1
INVALID_ID = (ID_INDEX_MASK << (TILE_INDEX_BITS + LEVEL_BITS)) | (TILE_INDEX_MASK << LEVEL_BITS) | LEVEL_MASK

def get_tile_level(id):
return id & LEVEL_MASK

def get_tile_index(id):
return (id >> LEVEL_BITS) & TILE_INDEX_MASK

def get_index(id):
return (id >> (LEVEL_BITS + TILE_INDEX_BITS)) & ID_INDEX_MASK

def tiles_for_bounding_box(left, bottom, right, top):
#if this is crossing the anti meridian split it up and combine
if left > right:
east = tiles_for_bounding_box(left, bottom, 180.0, top)
west = tiles_for_bounding_box(-180.0, bottom, right, top)
return east + west
#move these so we can compute percentages
left += 180
right += 180
bottom += 90
top += 90
tiles = []
#for each size of tile
for tile_set in valhalla_tiles:
#for each column
for x in range(int(left/tile_set['size']), int(right/tile_set['size']) + 1):
#for each row
for y in range(int(bottom/tile_set['size']), int(top/tile_set['size']) + 1):
#give back the level and the tile index
# value = int(y * (360.0/tile_set['size']) + x)
# tiles.append((tile_set['level'], int(value / 1000), value - int(value / 1000)*1000))
tiles.append((x, y, tile_set['level']))
return tiles

def get_tile_id(tile_level, lat, lon):
level = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0]
width = int(360 / level['size'])
return int((lat + 90) / level['size']) * width + int((lon + 180 ) / level['size'])

def get_ll(id):
tile_level = get_tile_level(id)
tile_index = get_tile_index(id)
level = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0]
width = int(360 / level['size'])
height = int(180 / level['size'])
return int(tile_index / width) * level['size'] - 90, (tile_index % width) * level['size'] - 180


def extractTiles(packageId, bbox, outputFileName, valhallaTileDir, zdict=None):
if os.path.exists(outputFileName):
os.remove(outputFileName)

with closing(sqlite3.connect(outputFileName)) as outputDb:
outputDb.execute("PRAGMA locking_mode=EXCLUSIVE")
outputDb.execute("PRAGMA synchronous=OFF")
outputDb.execute("PRAGMA page_size=512")
outputDb.execute("PRAGMA encoding='UTF-8'")

cursor = outputDb.cursor();
cursor.execute("PRAGMA synchronous=OFF")
cursor.execute("PRAGMA page_size=512")
cursor.execute("CREATE TABLE metadata (name TEXT, value TEXT)");
cursor.execute("CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB)");
cursor.execute("INSERT INTO metadata(name, value) VALUES('name', ?)", (packageId,))
Expand All @@ -184,7 +131,7 @@ def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=Non
if zdict is not None:
cursor.execute("INSERT INTO metadata(name, value) VALUES('shared_zlib_dict', ?)", (bytes(zdict),))

vTiles = calculateValhallaTilesFromTileMask(tileMask)
vTiles = tiles_for_bounding_box(bbox[0], bbox[1], bbox[2], bbox[3])
for vTile in vTiles:
file = os.path.join(valhallaTileDir, valhallaTilePath(vTile))
if os.path.isfile(file):
Expand All @@ -195,12 +142,9 @@ def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=Non
print('Warning: File %s does not exist!' % file)

cursor.execute("CREATE UNIQUE INDEX tiles_index ON tiles (zoom_level, tile_column, tile_row)");
cursor.close()
# cursor.execute("VACUUM")
outputDb.commit()

with closing(sqlite3.connect(outputFileName)) as outputDb:
outputDb.execute("VACUUM")

def processPackage(package, outputDir, tilesDir, zdictDir=None):
outputFileName = '%s/%s.vtiles' % (outputDir, package['id'])
if os.path.exists(outputFileName):
Expand All @@ -212,7 +156,7 @@ def processPackage(package, outputDir, tilesDir, zdictDir=None):
print('Processing %s' % package['id'])
try:
zdict = loadZDict(package['id'], zdictDir)
extractTiles(package['id'], package['tile_mask'], outputFileName, tilesDir, zdict)
extractTiles(package['id'], package['bbox'], outputFileName, tilesDir, zdict)
except:
if os.path.isfile(outputFileName):
os.remove(outputFileName)
Expand Down