From 3188a567de9943e67d4b10e15e0dcb66c3137559 Mon Sep 17 00:00:00 2001 From: Martin Guillon Date: Mon, 20 Apr 2020 11:36:28 +0200 Subject: [PATCH 1/3] fix: add bbox to packages --- scripts/build_poly_tilemasks.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/build_poly_tilemasks.py b/scripts/build_poly_tilemasks.py index 0382714..8a87af5 100644 --- a/scripts/build_poly_tilemasks.py +++ b/scripts/build_poly_tilemasks.py @@ -7,9 +7,33 @@ import argparse import utils.polygon2geojson as polygon2geojson import utils.tilemask as tilemask +import functools +import re +import geojson TILEMASK_SIZE_THRESHOLD = 512 +def bbox(coord_list): + box = [] + for i in (0,1): + res = sorted(coord_list, key=lambda x:x[i]) + box.append((res[0][i],res[-1][i])) + return box + + +def read_polygon(polygon_filename): + with open(polygon_filename) as f: + return f.readlines() + +def clean_polygon(polygon_data): + coordinates = polygon_data[2:][:-2] + coordinates = [re.split(r'[\s\t]+', item) for item in coordinates] + coordinates = [list(filter(None, item)) for item in coordinates] + coordinates = functools.reduce(lambda a,b: a[-1].pop(0) and a if len(a[-1]) == 1 and a[-1][0] == 'END' else a.append(['END']) or a if b[0].startswith('END') else a[-1].append(b) or a, [[[]]] + coordinates) + coordinates = [[(float(item[0]), float(item[1])) for item in coordgroup] for coordgroup in coordinates] + return coordinates + + def main(): parser = argparse.ArgumentParser() parser.add_argument(dest='input', help='Input directory for .poly files') @@ -32,11 +56,16 @@ def main(): if len(mask) >= TILEMASK_SIZE_THRESHOLD: break + polygon_data = read_polygon(polyFilename) + coordinates = clean_polygon(polygon_data) + poly=geojson.Polygon(coordinates) + line = bbox(list(geojson.utils.coords(poly))) packages.append( { 'id': packageId, 'version': 1, 'tile_mask': str(mask, 'utf8'), + 'bbox': (line[0][0], line[1][0], line[0][1],line[1][1]), 'url': '', 'metainfo': { 'name_en': packageName }, 'size': 0 From c30538625376489252ce5171ec5b90c8251a35d5 Mon Sep 17 00:00:00 2001 From: Martin Guillon Date: Mon, 20 Apr 2020 11:36:49 +0200 Subject: [PATCH 2/3] fix: fix retrieving valhalla tiles. also much faster --- scripts/build_valhalla_packages.py | 76 +++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/scripts/build_valhalla_packages.py b/scripts/build_valhalla_packages.py index 0f76da7..4ab1b5a 100644 --- a/scripts/build_valhalla_packages.py +++ b/scripts/build_valhalla_packages.py @@ -163,17 +163,70 @@ def loadZDict(packageId, zdictDir): print('Warning: Could not find dictionary for package %s!' % packageId) return None -def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=None): + +valhalla_tiles = [{'level': 2, 'size': 0.25}, {'level': 1, 'size': 1.0}, {'level': 0, 'size': 4.0}] +LEVEL_BITS = 3 +TILE_INDEX_BITS = 22 +ID_INDEX_BITS = 21 +LEVEL_MASK = (2**LEVEL_BITS) - 1 +TILE_INDEX_MASK = (2**TILE_INDEX_BITS) - 1 +ID_INDEX_MASK = (2**ID_INDEX_BITS) - 1 +INVALID_ID = (ID_INDEX_MASK << (TILE_INDEX_BITS + LEVEL_BITS)) | (TILE_INDEX_MASK << LEVEL_BITS) | LEVEL_MASK + +def get_tile_level(id): + return id & LEVEL_MASK + +def get_tile_index(id): + return (id >> LEVEL_BITS) & TILE_INDEX_MASK + +def get_index(id): + return (id >> (LEVEL_BITS + TILE_INDEX_BITS)) & ID_INDEX_MASK + +def tiles_for_bounding_box(left, bottom, right, top): + #if this is crossing the anti meridian split it up and combine + if left > right: + east = tiles_for_bounding_box(left, bottom, 180.0, top) + west = tiles_for_bounding_box(-180.0, bottom, right, top) + return east + west + #move these so we can compute percentages + left += 180 + right += 180 + bottom += 90 + top += 90 + tiles = [] + #for each size of tile + for tile_set in valhalla_tiles: + #for each column + for x in range(int(left/tile_set['size']), int(right/tile_set['size']) + 1): + #for each row + for y in range(int(bottom/tile_set['size']), int(top/tile_set['size']) + 1): + #give back the level and the tile index + # value = int(y * (360.0/tile_set['size']) + x) + # tiles.append((tile_set['level'], int(value / 1000), value - int(value / 1000)*1000)) + tiles.append((x, y, tile_set['level'])) + return tiles + +def get_tile_id(tile_level, lat, lon): + level = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0] + width = int(360 / level['size']) + return int((lat + 90) / level['size']) * width + int((lon + 180 ) / level['size']) + +def get_ll(id): + tile_level = get_tile_level(id) + tile_index = get_tile_index(id) + level = filter(lambda x: x['level'] == tile_level, valhalla_tiles)[0] + width = int(360 / level['size']) + height = int(180 / level['size']) + return int(tile_index / width) * level['size'] - 90, (tile_index % width) * level['size'] - 180 + + +def extractTiles(packageId, bbox, outputFileName, valhallaTileDir, zdict=None): if os.path.exists(outputFileName): os.remove(outputFileName) - with closing(sqlite3.connect(outputFileName)) as outputDb: - outputDb.execute("PRAGMA locking_mode=EXCLUSIVE") - outputDb.execute("PRAGMA synchronous=OFF") - outputDb.execute("PRAGMA page_size=512") - outputDb.execute("PRAGMA encoding='UTF-8'") - cursor = outputDb.cursor(); + cursor.execute("PRAGMA synchronous=OFF") + cursor.execute("PRAGMA page_size=512") cursor.execute("CREATE TABLE metadata (name TEXT, value TEXT)"); cursor.execute("CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB)"); cursor.execute("INSERT INTO metadata(name, value) VALUES('name', ?)", (packageId,)) @@ -184,7 +237,7 @@ def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=Non if zdict is not None: cursor.execute("INSERT INTO metadata(name, value) VALUES('shared_zlib_dict', ?)", (bytes(zdict),)) - vTiles = calculateValhallaTilesFromTileMask(tileMask) + vTiles = tiles_for_bounding_box(bbox[0], bbox[1], bbox[2], bbox[3]) for vTile in vTiles: file = os.path.join(valhallaTileDir, valhallaTilePath(vTile)) if os.path.isfile(file): @@ -195,12 +248,9 @@ def extractTiles(packageId, tileMask, outputFileName, valhallaTileDir, zdict=Non print('Warning: File %s does not exist!' % file) cursor.execute("CREATE UNIQUE INDEX tiles_index ON tiles (zoom_level, tile_column, tile_row)"); - cursor.close() + cursor.execute("VACUUM") outputDb.commit() - with closing(sqlite3.connect(outputFileName)) as outputDb: - outputDb.execute("VACUUM") - def processPackage(package, outputDir, tilesDir, zdictDir=None): outputFileName = '%s/%s.vtiles' % (outputDir, package['id']) if os.path.exists(outputFileName): @@ -212,7 +262,7 @@ def processPackage(package, outputDir, tilesDir, zdictDir=None): print('Processing %s' % package['id']) try: zdict = loadZDict(package['id'], zdictDir) - extractTiles(package['id'], package['tile_mask'], outputFileName, tilesDir, zdict) + extractTiles(package['id'], package['bbox'], outputFileName, tilesDir, zdict) except: if os.path.isfile(outputFileName): os.remove(outputFileName) From 353318decd8afadd93d976671311813982d7af86 Mon Sep 17 00:00:00 2001 From: Martin Guillon Date: Mon, 20 Apr 2020 11:38:26 +0200 Subject: [PATCH 3/3] chore: cleanup --- scripts/build_valhalla_packages.py | 108 +---------------------------- 1 file changed, 1 insertion(+), 107 deletions(-) diff --git a/scripts/build_valhalla_packages.py b/scripts/build_valhalla_packages.py index 4ab1b5a..a365152 100644 --- a/scripts/build_valhalla_packages.py +++ b/scripts/build_valhalla_packages.py @@ -21,87 +21,9 @@ # Default package version DEFAULT_PACKAGE_VERSION = 1 -# Zoom level/precision for tilemasks -TILEMASK_ZOOM = 10 - # Generic projection values VALHALLA_BOUNDS = ((-180, -90), (180, 90)) VALHALLA_TILESIZES = [4.0, 1.0, 0.25] -MERCATOR_BOUNDS = ((-6378137 * math.pi, -6378137 * math.pi), (6378137 * math.pi, 6378137 * math.pi)) - -class PackageTileMask(object): - def __init__(self, tileMaskStr): - self.data = self._decodeTileMask(tileMaskStr) - self.rootNode = self._buildTileNode(list(self.data), (0, 0, 0)) - - def contains(self, tile): - node = self._findTileNode(tile) - if node is None: - return False - return node["inside"] - - def getTiles(self, maxZoom=None): - tiles = [] - if self.data != []: - self._buildTiles(list(self.data), (0, 0, 0), maxZoom, tiles) - return tiles - - def _decodeTileMask(self, tileMaskStr): - str = [c for c in base64.b64decode(tileMaskStr)] - data = [] - for i in range(len(str) * 8): - val = (str[i // 8] >> (7 - i % 8)) & 1 - data.append(val) - return data - - def _buildTileNode(self, data, tile): - (zoom, x, y) = tile - subtiles = data.pop(0) - inside = data.pop(0) - node = { "tile" : tile, "inside": inside, "subtiles": [] } - if subtiles: - for dy in range(0, 2): - for dx in range(0, 2): - node["subtiles"].append(self._buildTileNode(data, (zoom + 1, x * 2 + dx, y * 2 + dy))) - return node - - def _findTileNode(self, tile): - (zoom, x, y) = tile - if zoom == 0: - return self.rootNode if tile == (0, 0, 0) else None - - parentNode = self._findTileNode((zoom - 1, x >> 1, y >> 1)) - if parentNode: - for node in parentNode["subtiles"]: - if node["tile"] == tile: - return node - if parentNode["inside"]: - return parentNode - return None - - def _buildTiles(self, data, tile, maxZoom, tiles): - (zoom, x, y) = tile - submask = data.pop(0) - inside = data.pop(0) - if inside: - tiles.append(tile) - if submask: - for dy in range(0, 2): - for dx in range(0, 2): - self._buildTiles(data, (zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles) - elif maxZoom is not None and inside: - for dy in range(0, 2): - for dx in range(0, 2): - self._buildAllTiles((zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles) - - def _buildAllTiles(self, tile, maxZoom, tiles): - (zoom, x, y) = tile - if zoom > maxZoom: - return - tiles.append(tile) - for dy in range(0, 2): - for dx in range(0, 2): - self._buildAllTiles((zoom + 1, x * 2 + dx, y * 2 + dy), maxZoom, tiles) def valhallaTilePath(vTile): vTileSize = VALHALLA_TILESIZES[vTile[2]] @@ -114,34 +36,6 @@ def valhallaTilePath(vTile): splitId = [str(vTile[2])] + splitId return '/'.join(splitId) + '.gph' -def _calculateValhallaTiles(mTile, vZoom, epsg3857, epsg4326): - mTileSize = (MERCATOR_BOUNDS[1][0] - MERCATOR_BOUNDS[0][0]) / (1 << mTile[2]) - vTileSize = VALHALLA_TILESIZES[vZoom] - mX0, mY0 = mTile[0] * mTileSize + MERCATOR_BOUNDS[0][0], mTile[1] * mTileSize + MERCATOR_BOUNDS[0][1] - mX1, mY1 = mX0 + mTileSize, mY0 + mTileSize - vX0, vY0 = pyproj.transform(epsg3857, epsg4326, mX0, mY0) - vX1, vY1 = pyproj.transform(epsg3857, epsg4326, mX1, mY1) - vTile0 = (vX0 - VALHALLA_BOUNDS[0][0]) / vTileSize, (vY0 - VALHALLA_BOUNDS[0][1]) / vTileSize - vTile1 = (vX1 - VALHALLA_BOUNDS[0][0]) / vTileSize, (vY1 - VALHALLA_BOUNDS[0][1]) / vTileSize - vTiles = [] - for y in range(int(math.floor(vTile0[1])), int(math.ceil(vTile1[1]))): - for x in range(int(math.floor(vTile0[0])), int(math.ceil(vTile1[0]))): - vTiles.append((x, y, vZoom)) - return vTiles - -def calculateValhallaTilesFromTileMask(tileMask): - vTiles = set() - mTiles = [(x, y, zoom) for zoom, x, y in PackageTileMask(tileMask).getTiles(TILEMASK_ZOOM)] - epsg3857 = pyproj.Proj(init='EPSG:3857') - epsg4326 = pyproj.Proj(init='EPSG:4326') - for mTile in mTiles: - if mTile[2] < TILEMASK_ZOOM: - continue - for vZoom, vTileSize in enumerate(VALHALLA_TILESIZES): - for vTile in _calculateValhallaTiles(mTile, vZoom, epsg3857, epsg4326): - vTiles.add(vTile) - return sorted(list(vTiles)) - def compressTile(tileData, zdict=None): if zdict is not None: compress = zlib.compressobj(9, zlib.DEFLATED, -15, 9, zlib.Z_DEFAULT_STRATEGY, zdict) @@ -248,7 +142,7 @@ def extractTiles(packageId, bbox, outputFileName, valhallaTileDir, zdict=None): print('Warning: File %s does not exist!' % file) cursor.execute("CREATE UNIQUE INDEX tiles_index ON tiles (zoom_level, tile_column, tile_row)"); - cursor.execute("VACUUM") + # cursor.execute("VACUUM") outputDb.commit() def processPackage(package, outputDir, tilesDir, zdictDir=None):