marstr/orthographic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial step toward all-in-memory management, osmxml is now completely in-RAM driven. Remaining parts to follow. Initial changes to all other elements toward preparation-then-render approach.

Browse Source
This commit is contained in:
Marcus Str. 2024-11-12 20:25:46 +01:00
parent 925f7bee27
commit fc88947768
6 changed files with 237 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
functions.py
View File

@ -91,3 +91,20 @@ def in_circle(center_x, center_y, radius, x, y):
def meters_per_pixel(lngwidth): def meters_per_pixel(lngwidth):
mpx = lngwidth / mstr_photores mpx = lngwidth / mstr_photores
return mpx return mpx
# Construct an X-Plane compatible folder name for latitude and longitude
def xplane_latlng_folder(numbers):
fstr = ""
if numbers[0] >= 0: fstr = "+"
if numbers[0] < 0: fstr = "-"
if abs(numbers[0]) < 10: fstr = fstr + "0" + str(numbers[0])
if abs(numbers[0]) >= 10 and numbers[0] <= 90: fstr = fstr + str(numbers[0])
if numbers[1] >= 0: fstr = fstr + "+"
if numbers[1] < 0: fstr = fstr + "-"
if abs(numbers[1]) < 10: fstr = fstr + "00" + str(numbers[1])
if abs(numbers[1]) >= 10 and numbers[0] <= 99: fstr = fstr + "0" + str(numbers[1])
if abs(numbers[1]) >= 100 : fstr = fstr + str(numbers[1])
return fstr

20
maskgen.py
View File

@ -75,17 +75,18 @@ class mstr_maskgen:
# Builds the required mask # Builds the required mask
def _build_mask(self): def _build_mask(self, xml, is_prep=False):
# Generate empty image # Generate empty image
imgsize = 2048 imgsize = 2048
mask_img = Image.new("RGBA", (imgsize, imgsize)) mask_img = Image.new("RGBA", (imgsize, imgsize))
tilexml = mstr_datafolder + "_cache/tile.xml" #tilexml = mstr_datafolder + "_cache/tile_" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + ".xml"
xml = mstr_osmxml(0,0) #tilexml = mstr_datafolder + "_cache/tile.xml"
#xml = mstr_osmxml(0,0)
fstr = str(self._box[0]) + "-" + str(self._box[1]) + "_" + str(self._box[2]) + "-" + str(self._box[3]) fstr = str(self._box[0]) + "-" + str(self._box[1]) + "_" + str(self._box[2]) + "-" + str(self._box[3])
nds = xml.acquire_nodes(tilexml) nds = xml.acquire_nodes()
way = xml.acquire_waypoint_data(tilexml) way = xml.acquire_waypoint_data()
rls = xml.acquire_relations(tilexml) rls = xml.acquire_relations()
mstr_msg("maskgen", "Building mask for " + str(self._box[0]) + "-" + str(self._box[1]) + ", " + str(self._box[2]) + "-" + str(self._box[3]) + ", for " + self._tag + ": " + self._value ) mstr_msg("maskgen", "Building mask for " + str(self._box[0]) + "-" + str(self._box[1]) + ", " + str(self._box[2]) + "-" + str(self._box[3]) + ", for " + self._tag + ": " + self._value )
@ -205,11 +206,14 @@ class mstr_maskgen:
imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve") imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve")
# Save image # Save image
mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png") if is_prep == False:
mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png")
if is_prep == True:
return mask_img
# If this is a building, we need to render the shadow here, as we only know the height # If this is a building, we need to render the shadow here, as we only know the height
# of the building in this loop. # of the building in this loop.
if mstr_shadow_enabled == True: if mstr_shadow_enabled == True and is_prep == False:
if self._tag == "building": if self._tag == "building":
mpp = meters_per_pixel(self._tile_width) * mstr_zl_18 mpp = meters_per_pixel(self._tile_width) * mstr_zl_18
pix_per_floor = mstr_shadow_floor_h / mpp pix_per_floor = mstr_shadow_floor_h / mpp

12
og.py
View File

@ -31,12 +31,14 @@ print(" ")
cli = False cli = False
pbf = False pbf = False
prep = False
if len(sys.argv) == 3:
cli = True
if len(sys.argv) == 4: if len(sys.argv) == 4:
pbf = True cli = True
if sys.argv[3] == "true": prep = True
#if len(sys.argv) == 4:
# pbf = True
# Only if we find enough arguments, proceed. # Only if we find enough arguments, proceed.
if cli == True: if cli == True:
@ -46,7 +48,7 @@ if cli == True:
mstr_msg("_main", "Beginning tile generation process.") mstr_msg("_main", "Beginning tile generation process.")
# Create the class and init values # Create the class and init values
og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd()) og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd(), prep)
og._buildTile() og._buildTile()

302
orthographic.py
View File

@ -19,6 +19,7 @@ from osmxml import *
from maskgen import * from maskgen import *
from layergen import * from layergen import *
from photogen import * from photogen import *
from tileprep import *
from xp_scenery import * from xp_scenery import *
@ -26,13 +27,14 @@ from xp_scenery import *
class mstr_orthographic: class mstr_orthographic:
# Constructor of class. Takes longitude and latitude. # Constructor of class. Takes longitude and latitude.
def __init__(self, lat, lng, outfolder, pwd): def __init__(self, lat, lng, outfolder, pwd, prep=False):
self._lat = lat self._lat = lat
self._long = lng self._long = lng
self._output = outfolder self._output = outfolder
self._pwd = pwd self._pwd = pwd
self._vstep = self._findVerticalStepping() self._vstep = self._findVerticalStepping()
self._latlngfld = self.latlng_folder([lat,lng]) self._latlngfld = self.latlng_folder([lat,lng])
self._prep = prep
mstr_msg("orthographic", "Initiated with LAT: " + str(lat) + ", LNG: " + str(lng)) mstr_msg("orthographic", "Initiated with LAT: " + str(lat) + ", LNG: " + str(lng))
@ -197,7 +199,7 @@ class mstr_orthographic:
mstr_msg("orthographic", "[X-Plane] created tile normal maps folder") mstr_msg("orthographic", "[X-Plane] created tile normal maps folder")
if not os.path.exists(self._output + "/z_orthographic/normals/" + self._latlngfld): if not os.path.exists(self._output + "/z_orthographic/normals/" + self._latlngfld):
os.makedirs(self._output + "/z_orthographic/normals/" + self._latlngfld) os.makedirs(self._output + "/z_orthographic/normals/" + self._latlngfld)
# The tile is constructed of many smaller parts. We walk through the # The tile is constructed of many smaller parts. We walk through the
# smallest possible, from which the bigger ones are later built. # smallest possible, from which the bigger ones are later built.
bb_lat = self._lat bb_lat = self._lat
@ -206,7 +208,7 @@ class mstr_orthographic:
bb_lng_edge = self._long+mstr_zl_18 bb_lng_edge = self._long+mstr_zl_18
cur_tile_x = 1 cur_tile_x = 1
cur_tile_y = 1 cur_tile_y = 1
osmxml = mstr_osmxml(0,0) #osmxml = mstr_osmxml(0,0)
mstr_msg("orthographic", "Set initial coordinates and bounding box for OSM acquisition") mstr_msg("orthographic", "Set initial coordinates and bounding box for OSM acquisition")
# The highest encountered tile numbers # The highest encountered tile numbers
@ -231,147 +233,215 @@ class mstr_orthographic:
bb_lat = self._lat bb_lat = self._lat
bb_lng = self._long bb_lng = self._long
# We will now prepare the graphic tile generation. We do this by only generating
# the masks and determine which sources to use in the actual images.
# Previously, I downloaded all XML files in one go - but to ease the # Previously, I downloaded all XML files in one go - but to ease the
# stress on OSM servers and my server, we will do acquire the data # stress on OSM servers and my server, we will do acquire the data
# only for the current processed part of the tile. # only for the current processed part of the tile.
for lat_grid in range(1, maxlatlng[0]+1): if self._prep == True:
for lng_grid in range(1, maxlatlng[1]+1): for lat_grid in range(1, maxlatlng[0]+1):
# Adjust bounding box for lng_grid in range(1, maxlatlng[1]+1):
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) # Adjust bounding box
mstr_msg("orthographic", "Adjusted bounding box for XML object") osmxml = mstr_osmxml()
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
# Determine what to do... maybe work was interrupted osmxml.acquire_osm(lat_grid, lng_grid)
if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False: mstr_msg("orthographic", "Adjusted bounding box for XML object")
# Let the user know
mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
# Get the data
osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
# Check for work to be done # Check for work to be done
layers = self.determineLayerWork() layers = self.determineLayerWork(osmxml)
# We need to walk through the array of layers,
# in their z-order.
# For each layer, we will generate the mask, the layer image
# itself, and finally, compose the ortho photo.
mstr_msg("orthographic", "Beginning generation of layers")
curlyr = 1 curlyr = 1
for layer in layers: for layer in layers:
# Let the user know if layer[2] == False and layer[0] != "building":
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers))) # Let the user know
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
mask = mg._build_mask(osmxml, is_prep=True) # We need an object here
tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False)
tp._prepareTile()
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
if layer[0] == "building":
mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
mg._build_mask()
# Generate the layer
lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
lg.set_max_latlng_tile(maxlatlng)
lg.set_latlng_folder(self._latlngfld)
#lg.open_db()
lg.open_tile_info()
lg.genlayer()
curlyr = curlyr+1 curlyr = curlyr+1
mstr_msg("orthographic", "All layers created")
# We should have all layers now. # Adjust longitude coordinates
# Snap a photo with our satellite :) cur_tile_x = cur_tile_x+1
mstr_msg("orthographic", "Generating ortho photo") bb_lng = bb_lng + mstr_zl_18
pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x, maxlatlng[0], maxlatlng[1]) bb_lng_edge = bb_lng_edge + mstr_zl_18
pg.genphoto() mstr_msg("orthographic", "Adjustment of longitude performed")
mstr_msg("orthographic", " -- Ortho photo generated -- ") # Adjust peak longitude tile number
print("") if cur_tile_x > top_lng:
print("") top_lng = cur_tile_x
# Adjust longitude coordinates # Adjust latitude and all other values when we get here
cur_tile_x = cur_tile_x+1 cur_tile_y = cur_tile_y+1
bb_lng = bb_lng + mstr_zl_18 cur_tile_x = 1
bb_lng_edge = bb_lng_edge + mstr_zl_18 bb_lng = self._long
mstr_msg("orthographic", "Adjustment of longitude performed") bb_lng_edge = self._long + mstr_zl_18
# Adjust peak longitude tile number bb_lat = bb_lat + self._vstep
if cur_tile_x > top_lng: bb_lat_edge = bb_lat_edge + self._vstep
top_lng = cur_tile_x mstr_msg("orthographic", "Adjustment of latitude performed")
# Adjust peak latitude number
if cur_tile_y > top_lat:
top_lat = cur_tile_y
# Clear out cache # Need to differentiate
if mstr_clear_cache == True: if self._prep == False:
ch = glob.glob(mstr_datafolder + "_cache/*") # The tile is constructed of many smaller parts. We walk through the
for f in ch: # smallest possible, from which the bigger ones are later built.
if os_platform == "nt": bb_lat = self._lat
if self._isFileAccessibleWin(f) == True:
os.remove(f)
if os_platform == "posix":
if self._isFileAccessiblePosix(f) == True:
os.remove(f)
mstr_msg("orthographic", "Cleared cache")
# Adjust latitude and all other values when we get here
cur_tile_y = cur_tile_y+1
cur_tile_x = 1
bb_lng = self._long bb_lng = self._long
bb_lng_edge = self._long + mstr_zl_18 bb_lat_edge = self._lat+self._vstep
bb_lat = bb_lat + self._vstep bb_lng_edge = self._long+mstr_zl_18
bb_lat_edge = bb_lat_edge + self._vstep cur_tile_x = 1
mstr_msg("orthographic", "Adjustment of latitude performed") cur_tile_y = 1
# Adjust peak latitude number osmxml = mstr_osmxml(0,0)
if cur_tile_y > top_lat:
top_lat = cur_tile_y # Previously, I downloaded all XML files in one go - but to ease the
# stress on OSM servers and my server, we will do acquire the data
mstr_msg("orthographic", "Generation of all tiles completed!") # only for the current processed part of the tile.
for lat_grid in range(1, maxlatlng[0]+1):
for lng_grid in range(1, maxlatlng[1]+1):
# Adjust bounding box
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
mstr_msg("orthographic", "Adjusted bounding box for XML object")
# Determine what to do... maybe work was interrupted
if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False:
# Let the user know
mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
# Get the data
#osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
#mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
# Check for work to be done
layers = self.determineLayerWork()
# We need to walk through the array of layers,
# in their z-order.
# For each layer, we will generate the mask, the layer image
# itself, and finally, compose the ortho photo.
mstr_msg("orthographic", "Beginning generation of layers")
curlyr = 1
for layer in layers:
# Let the user know
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
if layer[0] == "building":
mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
mg._build_mask()
# Generate the layer
lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
lg.set_max_latlng_tile(maxlatlng)
lg.set_latlng_folder(self._latlngfld)
#lg.open_db()
lg.open_tile_info()
lg.genlayer()
curlyr = curlyr+1
mstr_msg("orthographic", "All layers created")
# We should have all layers now.
# Snap a photo with our satellite :)
mstr_msg("orthographic", "Generating ortho photo")
pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x, maxlatlng[0], maxlatlng[1])
pg.genphoto()
mstr_msg("orthographic", " -- Ortho photo generated -- ")
print("")
print("")
# Adjust longitude coordinates
cur_tile_x = cur_tile_x+1
bb_lng = bb_lng + mstr_zl_18
bb_lng_edge = bb_lng_edge + mstr_zl_18
mstr_msg("orthographic", "Adjustment of longitude performed")
# Adjust peak longitude tile number
if cur_tile_x > top_lng:
top_lng = cur_tile_x
# Clear out cache
"""
if mstr_clear_cache == True:
ch = glob.glob(mstr_datafolder + "_cache/*")
for f in ch:
if os_platform == "nt":
if self._isFileAccessibleWin(f) == True:
os.remove(f)
if os_platform == "posix":
if self._isFileAccessiblePosix(f) == True:
os.remove(f)
mstr_msg("orthographic", "Cleared cache")
"""
# Adjust latitude and all other values when we get here
cur_tile_y = cur_tile_y+1
cur_tile_x = 1
bb_lng = self._long
bb_lng_edge = self._long + mstr_zl_18
bb_lat = bb_lat + self._vstep
bb_lat_edge = bb_lat_edge + self._vstep
mstr_msg("orthographic", "Adjustment of latitude performed")
# Adjust peak latitude number
if cur_tile_y > top_lat:
top_lat = cur_tile_y
mstr_msg("orthographic", "Generation of all tiles completed!")
# Complete scenery # Complete scenery
if mstr_xp_genscenery == True: if mstr_xp_genscenery == True:
scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld) scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld)
scn.acquire_elevation_data() scn.acquire_elevation_data()
scn.acquire_xes_data() scn.acquire_xes_data()
scn.build_mesh_script() scn.build_mesh_script()
scn.build_mesh() scn.build_mesh()
scn.build_ter_files() scn.build_ter_files()
mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed") mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed")
mstr_msg("orthographic", "Final step completed.") mstr_msg("orthographic", "Final step completed.")
mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld) mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld)
print("") print("")
mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus") mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
print("") print("")
# Let's leave this out for the moment # Let's leave this out for the moment
""" """
mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles") mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles")
tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng) tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng)
tg.genTiles() tg.genTiles()
mstr_msg("orthographic", "Final step completed.") mstr_msg("orthographic", "Final step completed.")
print("") print("")
mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng) mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng)
print("") print("")
print("") print("")
mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus") mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
print("") print("")
""" """
# Checks which layers need to be generated, and what kind of layer it is # Checks which layers need to be generated, and what kind of layer it is
def determineLayerWork(self): def determineLayerWork(self, xmlobj):
mstr_msg("orthographic", "Checking for work to be performed") mstr_msg("orthographic", "Checking for work to be performed")
layers = [] layers = []
tilexml = mstr_datafolder + "_cache/tile.xml" #tilexml = mstr_datafolder + "_cache/tile.xml"
xml = mstr_osmxml(0,0) #xml = mstr_osmxml(0,0)
way = xml.acquire_waypoint_data(tilexml) way = xmlobj.acquire_waypoint_data()
rls = xml.acquire_relations(tilexml) rls = xmlobj.acquire_relations()
for l in mstr_ortho_layers: for l in mstr_ortho_layers:
# Check if there is anything to render # Check if there is anything to render

6
osmxml.py
View File

@ -54,11 +54,11 @@ class mstr_osmxml:
# Acquire XMLs in chunks, then store them # Acquire XMLs in chunks, then store them
def acquire_osm(self, v, h, asobject=False): def acquire_osm(self, v, h):
mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng)) mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng))
# We will use our self-hosted API for this. # We will use our self-hosted API for this.
while os.path.isfile(self._xmlfn) == False: while self._xmlcontent == "":
data = { data = {
"bbox": { "bbox": {
"lat": str(self._lat), "lat": str(self._lat),
@ -86,7 +86,7 @@ class mstr_osmxml:
sleep(1) sleep(1)
# Store the content in memory # Store the content in memory
self._xmldata = xml.dom.minidom.parse(self._xmlcontent) self._xmldata = xml.dom.minidom.parseString(self._xmlcontent)
self._xmlcontent = "" # Clear self._xmlcontent = "" # Clear

13
tileprep.py
View File

@ -43,6 +43,16 @@ class mstr_tileprep:
self._is_completion = is_completion self._is_completion = is_completion
def _findCorrectTextureFolder(self):
srcfld = []
for lyr in mstr_ortho_layers:
if lyr[0] == self._tag and lyr[1] == self._value:
srcfld.append(lyr[2])
srcfld.append(lyr[3])
break
return srcfld
# Prepare the tile accordingly # Prepare the tile accordingly
def _prepareTile(self): def _prepareTile(self):
# Load the mask pixels # Load the mask pixels
@ -117,7 +127,8 @@ class mstr_tileprep:
# If there was nothing in the info still, we need to select some source # If there was nothing in the info still, we need to select some source
if self._source == -1: if self._source == -1:
tx = mstr_datafolder + "textures/" + self._tag + "/" + self._value + "/brd/b*.png" srcfld = self._findCorrectTextureFolder()
tx = mstr_datafolder + "textures/" + srcfld[0] + "/" + srcfld[1] + "/brd/b*.png"
lst = glob.glob(tx) lst = glob.glob(tx)
if len(lst) == 1: self._source = 1 if len(lst) == 1: self._source = 1
if len(lst) >= 2: self._source = randrange(1, len(lst)+1) if len(lst) >= 2: self._source = randrange(1, len(lst)+1)