Milestone commit - switch to mostly in-memory management, and functioning multi-processing for ortho generation

This commit is contained in:
Marcus Str. 2024-11-16 15:20:00 +01:00
parent fc88947768
commit 88bcdf4a7a
8 changed files with 461 additions and 546 deletions

View File

@ -148,7 +148,6 @@ mstr_ortho_layers = [
("highway", "tertiary", 20),
("highway", "unclassified", 17),
("highway", "living_street", 12),
("waterway", "river", 10),
("waterway", "stream", 2),
("amenity", "parking", "amenity", "parking"),
("amenity", "school", "amenity", "school"),
@ -162,12 +161,8 @@ mstr_ortho_layers = [
("natural", "sand", "natural", "sand"),
("natural", "desert", "natural", "desert"),
# Z-Order 3
("natural", "water", "natural", "water"),
("natural", "bay", "natural", "beach"),
("natural", "beach", "natural", "beach"),
("water", "lake", "natural", "water"),
("water", "pond", "natural", "water"),
("water", "river", "natural", "water"),
("leisure", "swimming_pool", "natural", "water"),
("highway", "pedestrian", 4),
# Z-Order 4
@ -195,6 +190,11 @@ mstr_ortho_layers = [
("building", "public", "building", "public"),
("building", "commercial", "building", "commercial"),
("building", "yes", "building", "common"),
("water", "lake", "natural", "water"),
("water", "pond", "natural", "water"),
("water", "river", "natural", "water"),
("natural", "water", "natural", "water"),
("waterway", "river", 10),
("place", "sea", "natural", "sea"),
("place", "ocean", "natural", "sea")
]
@ -229,10 +229,10 @@ mstr_mask_blur = [
("landuse", "farmland", 10),
("landuse", "farmyard", 10),
# Z-Order 2
("landuse", "forest", 8),
("leisure", "nature_reserve", 8),
("natural", "wood", 8),
("natural", "tree_row", 8),
("landuse", "forest", 10),
("leisure", "nature_reserve", 10),
("natural", "wood", 10),
("natural", "tree_row", 10),
("landuse", "military", 15),
# Z-Order 3
("natural", "bare_rock", 25),

View File

@ -114,27 +114,63 @@ class mstr_layergen:
return layer_final
# Find the source to use pre-determined in phase one
def findLayerSource(self):
# The source number
src = -1
# The already existing source data
srcfile = mstr_datafolder + "z_orthographic/data/" + self._latlngfld + "/" + str(self._lat_number) + "_" + str(self._lng_number)
# Let's open the file and find our entry
with open(srcfile) as file:
for line in file:
linedata = line.split(" ")
if linedata[2] == self._tag and linedata[3] == self._value:
src = int(linedata[4])
break
# Should we encounter a -1 at this point, we can choose something
# It means it touches no border as it was not found in the file
if src == -1:
root_folder = mstr_datafolder + "textures/"
for s in mstr_ortho_layers:
if s[0] == self._tag and s[1] == self._value:
fld_main = len(s)-2
fld_sub = len(s)-1
root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
brd = glob.glob(root_folder + "/brd/b*.png")
src = randrange(1, len(brd)+1)
return src
# This generates the layer from the defined mask
def genlayer(self):
def genlayer(self, mask, xml):
mstr_msg("layergen", "Layer to be generated: " + str(self._latitude) + "-" + str(self._lat_number) + ":" + str(self._longitude) + "-" + str(self._lng_number) + " -- tag: " + self._tag + " - value: " + self._value )
# Before we generate the layer, let's check for airports in this chunk
mstr_msg("layergen", "Checking for airport/s with ICAO code")
osmxml = mstr_osmxml(0,0)
icao = osmxml.find_icao_codes(mstr_datafolder + "_cache/tile.xml")
mstr_msg("layergen", "Found " + str(len(icao)) + " airport/s")
icao = None
if xml != None:
icao = xml.find_icao_codes()
mstr_msg("layergen", "Found " + str(len(icao)) + " airport/s")
# Runway surface, if any other than concrete/asphalt
rw_surface = ""
# If we find an airport, make a note ...
if len(icao) >= 1:
#for i in icao:
# ... but only, if this airport is not already noted
#iccheck = self._tiledb.perform_query("SELECT * FROM airports WHERE icao='" + i +"';")
#if len(iccheck) == 0:
#self._tiledb.insert_icao(i, self._lat_number, self._lng_number, self._latitude, self._longitude)
# mstr_msg("layergen", "Airport/s noted in data file")
rw_surface = osmxml.find_runway_surface(mstr_datafolder + "_cache/tile.xml")
if icao != None:
if len(icao) >= 1 and self._is_completion == False:
#for i in icao:
# ... but only, if this airport is not already noted
#iccheck = self._tiledb.perform_query("SELECT * FROM airports WHERE icao='" + i +"';")
#if len(iccheck) == 0:
#self._tiledb.insert_icao(i, self._lat_number, self._lng_number, self._latitude, self._longitude)
# mstr_msg("layergen", "Airport/s noted in data file")
rw_surface = xml.find_runway_surface()
# The image for the layer itself
layer = Image.new("RGBA", (self._imgsize, self._imgsize))
@ -153,176 +189,7 @@ class mstr_layergen:
root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
# Determine which sources to use.
# First, we need to check for adjacent tile information. We then either
# need to use the source of any adjacent tile, or we can choose freely.
src = -1
# Find our adjacent tiles
adjtiles = findAdjacentTilesTo(self._lat_number, self._lng_number)
mstr_msg("layergen", "Performing adjacency check")
# Walk through each tile and see what we can find in relation to this
# tile in the center
# Since we already know the order in adjtiles, we can do this real easy
if self._is_completion == False:
at = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top
ar = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right
ab = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom
al = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left
#at = self._tiledb.get_adjacency_for_source(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top
#ar = self._tiledb.get_adjacency_for_source(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right
#ab = self._tiledb.get_adjacency_for_source(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom
#al = self._tiledb.get_adjacency_for_source(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left
if self._is_completion == True:
at = self._tileinfo.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top
ar = self._tileinfo.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right
ab = self._tileinfo.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom
al = self._tileinfo.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left
#at = self._tiledb.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top
#ar = self._tiledb.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right
#ab = self._tiledb.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom
#al = self._tiledb.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left
if len(at) == 4:
self._tag = at[0]
self._value = at[1]
if len(ar) == 4:
self._tag = ar[0]
self._value = ar[1]
if len(ab) == 4:
self._tag = ab[0]
self._value = ab[1]
if len(al) == 4:
self._tag = al[0]
self._value = al[1]
root_folder = mstr_datafolder + "textures/"
for s in mstr_ortho_layers:
if s[0] == self._tag and s[1] == self._value:
fld_main = len(s)-2
fld_sub = len(s)-1
root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
# We are south to the top tile.
if len(at) == 2 and src == -1:
if "b" in at[1]: src = at[0]
# We are west to the right tile.
if len(ar) == 2 and src == -1:
if "l" in ar[1]: src = ar[0]
# We are north to the bottom tile.
if len(ab) == 2 and src == -1:
if "t" in ab[1]: src = ab[0]
# We are east to the left tile.
if len(al) == 2 and src == -1:
if "r" in al[1]: src = al[0]
# Should we be at the border of the degree for latitude and longitude, we need to perform
# additional checks
is_deg_brd_t = False
is_deg_brd_r = False
is_deg_brd_b = False
is_deg_brd_l = False
if self._lat_number == 1: is_deg_brd_b = True
if self._lat_number == self._maxlat: is_deg_brd_t = True
if self._lng_number == 1: is_deg_brd_l = True
if self._lng_number == self._maxlng: is_deg_brd_r = True
# Adjacent latitude and longitude tiles
deg_tiles = []
deg_tiles.append( ( self._latitude+1, self._longitude ) ) # Top
deg_tiles.append( ( self._latitude, self._longitude+1 ) ) # Right
deg_tiles.append( ( self._latitude-1, self._longitude ) ) # Bottom
deg_tiles.append( ( self._latitude, self._longitude-1 ) ) # Left
# Perform degree border checks
# - and make sure we do not run into errors - this drove me crazy in testing
atd = []
ard = []
abd = []
ald = []
if is_deg_brd_t == True:
if self._is_completion == False:
atd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], self._tag, self._value, 1, self._lng_number, "b") # Top
#atd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, self._tag, self._value) # Top
if self._is_completion == True:
atd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, "b")
#atd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number) # Top
if is_deg_brd_r == True:
if self._is_completion == False:
ard = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._tag, self._value, self._lat_number, 1, "l") # Right
#ard = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, self._tag, self._value) # Right
if self._is_completion == True:
ard = self._tileinfo.get_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, "l")
#ard = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1) # Right
if is_deg_brd_b == True:
#maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude-1, self._longitude)
if self._is_completion == False:
abd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "t")
#abd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number, self._tag, self._value) # Bottom
if self._is_completion == True:
abd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "t")
#abd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number) # Bottom
if is_deg_brd_l == True:
#maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude, self._longitude-1)
if self._is_completion == False:
ald = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "r")
#ald = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1], self._tag, self._value) # Left
if self._is_completion == True:
ald = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "r")
#ald = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1]) # Left
if (is_deg_brd_t == True or is_deg_brd_r == True or is_deg_brd_b == True or is_deg_brd_l == True):
if src == -1 and self._is_completion == True:
if len(atd) == 4:
self._tag = atd[0]
self._value = atd[1]
if len(ard) == 4:
self._tag = ard[0]
self._value = ard[1]
if len(abd) == 4:
self._tag = abd[0]
self._value = abd[1]
if len(ald) == 4:
self._tag = ald[0]
self._value = ald[1]
root_folder = mstr_datafolder + "textures/"
for s in mstr_ortho_layers:
if s[0] == self._tag and s[1] == self._value:
fld_main = len(s)-2
fld_sub = len(s)-1
root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
# Should we get here and one of the degree border checks turns out true,
# we need to make sure that we select the source we found. This should
# enable seamless tiling... around the entire planet
if is_deg_brd_t == True and len(at) == 0 and src == -1:
if len(atd) == 2:
if "b" in atd[3]: src = int(atd[2])
if is_deg_brd_r == True and len(ar) == 0 and src == -1:
if len(ard) == 2:
if "l" in ard[3]: src = int(ard[2])
if is_deg_brd_b == True and len(ab) == 0 and src == -1:
if len(abd) == 2:
if "t" in abd[3]: src = int(abd[2])
if is_deg_brd_l == True and len(al) == 0 and src == -1:
if len(ald) == 2:
if "r" in ald[3]: src = int(ald[2])
mstr_msg("layergen", "Adjacency check completed")
brd = glob.glob(root_folder + "/brd/b*.png")
# If the adjacency check returned nothing (src is still -1),
# then pick something
if src == -1:
if len(brd) == 1: src=1
if len(brd) >= 2:
src = randrange(1, len(brd)+1)
src = self.findLayerSource()
ptc = glob.glob(root_folder + "/ptc/b" + str(src) + "_p*.png")
@ -333,25 +200,27 @@ class mstr_layergen:
ptc_src.append(Image.open(p))
mstr_msg("layergen", "Layer sources selected")
"""
# OK! Load the mask
if self._is_completion == False:
osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
if self._is_completion == True:
osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" )
mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" )
"""
# Generate an edge mask from the original
osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
osm_edge = mask.filter(ImageFilter.FIND_EDGES)
osm_edge = osm_edge.filter(ImageFilter.MaxFilter)
mstr_msg("layergen", "Edge mask generated")
# This adds some natural looking shapes to these types of features
if self._value == "forest" or self._value == "nature_reserve":
epx = osm_edge.load()
imgd = ImageDraw.Draw(osm_mask)
imgd = ImageDraw.Draw(mask)
# Walk through a grid of 200x200 - on the edge image
for y in range(0, osm_mask.height, int(osm_mask.height/200)):
for x in range(0, osm_mask.width, int(osm_mask.width/200)):
# Walk through a grid of 100x100 - on the edge image
for y in range(0, mask.height, int(mask.height/100)):
for x in range(0, mask.width, int(mask.width/100)):
px = epx[x,y]
if px[3] == 255:
rx = randrange(24,60)
@ -363,7 +232,7 @@ class mstr_layergen:
psy = randrange(y-11, y+11)
# Do some magic - but not on edges
if x > 0 and x < osm_mask.width and y > 0 and y < osm_mask.height:
if x > 0 and x < mask.width and y > 0 and y < mask.height:
if f != 5:
imgd.ellipse((psx-int(rx/2), psy-int(ry/2), psx+rx, psy+ry), fill="black")
if f == 3 or f == 7:
@ -372,14 +241,14 @@ class mstr_layergen:
# We need to change the image in certain conditions
if self._value == "hedge" and self._tag == "barrier":
osm_mask = osm_edge
mask = osm_edge
# From here on in we will need to perform some adjustments on the masks, depending
# on what they are.
for i in mstr_mask_blur:
if i[0] == self._tag and i[1] == self._value:
if self._tag != "place" and (self._value != "sea" or self._value != "ocean"):
osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
break
# Begin producing a largely random image
@ -414,7 +283,7 @@ class mstr_layergen:
ly = randrange( self._imgsize - img.height )
layer.alpha_composite( img, (lx, ly) )
if self._is_completion == True:
mp = osm_mask.load()
mp = mask.load()
edn = self.xplane_latlng_folder(self.find_earthnavdata_number())
idx = 0
for r in mstr_completion_colors:
@ -483,7 +352,7 @@ class mstr_layergen:
# Generate the layer from the mask.
layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
layer_pix = layer.load()
mask_pix = osm_mask.load()
mask_pix = mask.load()
layer_comp_pix = layer_comp.load()
for y in range(self._imgsize):
for x in range(self._imgsize):
@ -505,9 +374,10 @@ class mstr_layergen:
# Here we want to make sure that the generated image fits well with others, so
# let's do that.
mstr_msg("layergen", "Generating adjacent fades")
self.generate_adjacent_fades()
adjfade = self.generate_adjacent_fades(mask)
# Determine if there are any fades, and if so, fade those in first before we save the layer
"""
fade_fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value
if os.path.isfile(fade_fn + "_fade_top.png") == True:
fade = Image.open(fade_fn + "_fade_top.png")
@ -521,6 +391,9 @@ class mstr_layergen:
if os.path.isfile(fade_fn + "_fade_left.png") == True:
fade = Image.open(fade_fn + "_fade_left.png")
layer_comp.alpha_composite(fade)
"""
layer_comp.alpha_composite(adjfade)
mstr_msg("layergen", "Adjacent fading completed")
@ -551,10 +424,10 @@ class mstr_layergen:
layer_comp_pix[x,y] = (lpix[0], lpix[1], lpix[2], 255-rpix[3])
# Store layer
if self._is_completion == False:
layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
if self._is_completion == True:
layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" )
#if self._is_completion == False:
# layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
#if self._is_completion == True:
# layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" )
#layer_final.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
mstr_msg("layergen", "Layer image finalized and saved.")
@ -570,7 +443,7 @@ class mstr_layergen:
break
if nm == True:
nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
nrm.build_normalmap()
nrm.build_normalmap(layer_comp)
# Let's try our hand at pseudo shadows
@ -581,7 +454,7 @@ class mstr_layergen:
if self._tag == sh[0] and self._value == sh[1]:
mstr_msg("layergen", "Generating shadow for layer")
shadow_pix = shadow.load()
mask_pix = osm_mask.load()
mask_pix = mask.load()
shf = 1
while shf < mstr_shadow_shift:
for y in range(self._imgsize):
@ -602,73 +475,12 @@ class mstr_layergen:
shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1.5))
shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png")
#shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png")
shadow.alpha_composite(layer_comp)
layer_comp = shadow
mstr_msg("layergen", "Shadow layer completed")
# Check if pixels touch the borders of the image, and if so -
# make a not of that in the database.
at=False
ar=False
ab=False
al=False
layer_pix = layer_comp.load() # <- Just to be safe
# Top scan
for i in range(0, self._imgsize-1):
p = layer_pix[i,0]
if p[3] > 0:
at=True
break
# Right scan
for i in range(0, self._imgsize-1):
p = layer_pix[self._imgsize-1,i]
if p[3] > 0:
ar=True
break
# Bottom scan
for i in range(0, self._imgsize-1):
p = layer_pix[i,self._imgsize-1]
if p[3] > 0:
ab=True
break
# Left scan
for i in range(0, self._imgsize-1):
p = layer_pix[1,i]
if p[3] > 0:
al=True
break
# Construct DB String
adjstr = ""
if at==True: adjstr = adjstr + "t"
if ar==True: adjstr = adjstr + "r"
if ab==True: adjstr = adjstr + "b"
if al==True: adjstr = adjstr + "l"
# Store into DB - but only if there is something to store
if adjstr != "":
if self._is_completion == False:
r = self._tileinfo.get_adjacency_for_tag_and_value(self._lat_number, self._lng_number, self._tag, self._value)
#r = self._tiledb.get_adjacency_for_source(self._lat_number, self._lng_number, self._tag, self._value)
if len(r) == 0:
self._tileinfo.add_adjacency_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr)
mstr_msg("layergen", "Adjacency info stored in database")
if self._is_completion == True:
r = self._tileinfo.get_adjacency_for_completion(self._lat_number, self._lng_number)
#r = self._tiledb.get_adjacency_for_completion(self._lat_number, self._lng_number)
if len(r) == 0:
self._tileinfo.add_completion_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr)
mstr_msg("layergen", "Adjacency info for completion stored in database")
#self._tiledb.commit_query()
#self._tiledb.close_db()
# Create a water mask we need to remove from the DDS later
if (self._tag == "natural" and self._value == "water") or (self._tag == "water" and self._value == "lake") or (self._tag == "water" and self._value == "pond") or (self._tag == "water" and self._value == "river") or (self._tag == "leisure" and self._value == "swimming_pool"):
mstr_msg("layergen", "Generating inland water mask")
@ -681,8 +493,12 @@ class mstr_layergen:
if l[3] > 65:
b = 255 - l[3]
inl_pix[x,y] = (255,0,255,255)
inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
#inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
layer_comp = inl_mask
mstr_msg("layergen", "Inland water mask generated and saved")
# Return the completed image
return layer_comp
# ---------------------------------------------------------------------------------------
@ -694,10 +510,10 @@ class mstr_layergen:
if self._isline == True or self._tag == "building":
# We will need the mask in question
osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
#mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
# Generate an edge mask from the original
osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
osm_edge = mask.filter(ImageFilter.FIND_EDGES)
osm_edge = osm_edge.filter(ImageFilter.MaxFilter)
mstr_msg("layergen", "Edge mask generated")
@ -706,7 +522,7 @@ class mstr_layergen:
if self._tag != "building":
for i in mstr_mask_blur:
if i[0] == self._tag and i[1] == self._value:
osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
break
@ -715,7 +531,7 @@ class mstr_layergen:
# This time we have no source material - instead we will fill the
# mask with a color that is appropriate for this street type.
layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
mask_pix = osm_mask.load()
mask_pix = mask.load()
edge_pix = osm_edge.load()
layer_comp_pix = layer_comp.load()
@ -812,10 +628,21 @@ class mstr_layergen:
# We will do some super magic here to let houses look more realistic
if self._tag == "building":
details = Image.new("RGBA", (self._imgsize, self._imgsize))
tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
trees = Image.new("RGBA", (self._imgsize, self._imgsize))
roof_details = Image.new("RGBA", (self._imgsize, self._imgsize))
shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
if mstr_shadow_enabled == True:
fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_building-" + self._value + "_layer_shadow.png"
if os.path.isfile(fn):
shadow = Image.open(fn)
vls = [ "detached", "hotel", "farm", "semidetached_house", "apartments", "civic", "house", "school", "kindergarten", "yes" ]
if self._value in vls:
# Generate a new image
details = Image.new("RGBA", (self._imgsize, self._imgsize))
details_pix = details.load()
layer_pix = layer_comp.load()
for y in range(self._imgsize):
@ -836,7 +663,6 @@ class mstr_layergen:
details_pix[shf_x2, shf_y2] = (187-d2, 179-d2, 176-d2, aa)
# Image for roof details
roof_details = Image.new("RGBA", (self._imgsize, self._imgsize))
roof_det_pix = roof_details.load()
for y in range(self._imgsize):
for x in range(self._imgsize):
@ -851,15 +677,14 @@ class mstr_layergen:
a = randrange(1, 151)
nc = (mstr_building_detail_colors[0][0]-d, mstr_building_detail_colors[0][1]-d, mstr_building_detail_colors[0][2]-d, a)
roof_det_pix[x,y] = nc
layer_comp.alpha_composite(roof_details)
# Let's see how it works with this method
details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png")
#details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png")
#layer_comp.alpha_composite(details)
# Add some random trees
div = int(self._imgsize/200)
trees = Image.new("RGBA", (self._imgsize, self._imgsize))
for y in range(0, self._imgsize, div):
for x in range(0, self._imgsize, div):
if x > 0 and x < self._imgsize and y > 0 and y < self._imgsize:
@ -885,9 +710,9 @@ class mstr_layergen:
if shf_x > self._imgsize - tree.width: shf_x = self._imgsize - tree.width - 1
if shf_y > self._imgsize - tree.height: shf_y = self._imgsize - tree.height - 1
trees.alpha_composite(tree, (shf_x, shf_y))
if mstr_shadow_enabled == True:
tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
tree_pix = trees.load()
shadow_pix = tree_shadow.load()
for y in range(self._imgsize):
@ -902,18 +727,34 @@ class mstr_layergen:
tree_shadow.alpha_composite(trees)
# Save this separately, so that we can blur buildings, but not the trees
fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png"
#fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png"
if os.path.isfile(fn) == True:
extrees = Image.open(fn)
extrees.alpha_composite(tree_shadow)
extrees.save(fn)
else:
tree_shadow.save(fn)
#if os.path.isfile(fn) == True:
# extrees = Image.open(fn)
# extrees.alpha_composite(tree_shadow)
# extrees.save(fn)
#else:
# tree_shadow.save(fn)
#layer_comp.alpha_composite(tree_shadow)
#tree_shadow.alpha_composite(layer_comp)
#layer_comp = tree_shadow
# Let's try this one on for size
bld_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
details = details.filter(ImageFilter.GaussianBlur(radius=1))
bld_comp.alpha_composite(details)
bld_comp.alpha_composite(tree_shadow)
bld_comp.alpha_composite(trees)
shd_p = shadow.load()
for y in range(self._imgsize):
for x in range(self._imgsize):
c = shd_p[x,y]
if c[3] > 0:
s = (0,0,0,120-(randrange(0,21)))
shd_p[x,y] = s
shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1))
bld_comp.alpha_composite(shadow)
layer_comp = layer_comp.filter(ImageFilter.GaussianBlur(radius=1.1))
bld_comp.alpha_composite(layer_comp)
layer_comp = bld_comp
layer_comp.alpha_composite(roof_details)
mstr_msg("layergen", "Layer image generated")
@ -922,7 +763,7 @@ class mstr_layergen:
# Some funnies with shadows
if self._tag == "building" and (self._value == "detached" or self._value == "semidetached_house" or self._value == "apartments" or self._value == "civic" or self._value == "house" or self._value == "terrace"):
mask_pix = osm_mask.load()
mask_pix = mask.load()
roofshadow = Image.new("RGBA", (self._imgsize, self._imgsize))
roofpix = roofshadow.load()
# Generate a pseudo shifted roof shadow
@ -937,7 +778,7 @@ class mstr_layergen:
# Now apply the shift where necessary
roofpix = roofshadow.load()
mask_pix = osm_mask.load()
mask_pix = mask.load()
layer_comp_pix = layer_comp.load()
for y in range(self._imgsize):
for x in range(self._imgsize):
@ -1014,7 +855,7 @@ class mstr_layergen:
# Highways and runways of any kind get some special treatment
if (self._tag == "highway" and self._value == "motorway") or (self._tag == "highway" and self._value == "primary") or (self._tag == "highway" and self._value == "secondary") or (self._tag == "highway" and self._value == "tertiary") or (self._tag == "aeroway" and self._value == "runway"):
# We will now add some white lines for coolness
osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
osm_edge = mask.filter(ImageFilter.FIND_EDGES)
mask_pix = osm_edge.load()
layer_comp_pix = layer_comp.load()
for y in range(self._imgsize):
@ -1026,7 +867,7 @@ class mstr_layergen:
layer_comp_pix[x, y] = ( w,w,w,a[3] )
if self._tag == "highway" and self._value == "residential":
osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
osm_edge = mask.filter(ImageFilter.FIND_EDGES)
mask_pix = osm_edge.load()
layer_comp_pix = layer_comp.load()
for y in range(self._imgsize):
@ -1052,7 +893,7 @@ class mstr_layergen:
if l[3] > 65:
b = 255 - l[3]
inl_pix[x,y] = (255,0,255,255)
inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
#inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
mstr_msg("layergen", "Inland water mask generated and saved")
@ -1062,7 +903,7 @@ class mstr_layergen:
# Store layer
layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
#layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
mstr_msg("layergen", "Layer image finalized and saved.")
@ -1077,13 +918,17 @@ class mstr_layergen:
break
if nm == True:
nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
nrm.build_normalmap()
nrm.build_normalmap(layer_comp)
# Return image
return layer_comp
# Should we find more than one source, the first one found will take precedence.
# For the others, we will need to generate fading images, so that the final layer
# image works with other tiles
def generate_adjacent_fades(self):
def generate_adjacent_fades(self, mask):
adj_sources = self.find_all_adjacent_sources()
precedence = -1
@ -1105,11 +950,11 @@ class mstr_layergen:
# Generate required images
# Basically a shortened version of the main layergen call
adj_image = Image.new("RGBA", (self._imgsize, self._imgsize))
for s in range(0, 4):
if adj_sources[s] != precedence and adj_sources[s] != -1:
src = adj_sources[s]
adj_image = Image.new("RGBA", (self._imgsize, self._imgsize))
adj_pix = adj_image.load()
# Root folder
@ -1122,15 +967,15 @@ class mstr_layergen:
for p in ptc:
ptc_src.append(Image.open(p))
osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
#mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
#lyr_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
for i in mstr_mask_blur:
if i[0] == self._tag and i[1] == self._value:
if self._tag != "place" and (self._value != "sea" or self._value != "ocean"):
osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
break
mask_pix = osm_mask.load()
mask_pix = mask.load()
# Begin producing a largely random image
samples = 250 # <- We need this in a moment
@ -1167,7 +1012,7 @@ class mstr_layergen:
adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
else:
adj_pix[x,y] = (0,0,0,0)
adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png")
#adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png")
if s == 1:
for y in range(self._imgsize):
@ -1178,7 +1023,7 @@ class mstr_layergen:
adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
else:
adj_pix[x,y] = (0,0,0,0)
adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png")
#adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png")
if s == 2:
for y in range(self._imgsize):
@ -1189,7 +1034,7 @@ class mstr_layergen:
adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
else:
adj_pix[x,y] = (0,0,0,0)
adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png")
#adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png")
if s == 3:
for y in range(self._imgsize):
@ -1200,7 +1045,11 @@ class mstr_layergen:
adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
else:
adj_pix[x,y] = (0,0,0,0)
adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png")
#adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png")
# Return the image
return adj_image
def find_all_adjacent_sources(self):

View File

@ -205,12 +205,10 @@ class mstr_maskgen:
break
imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve")
# Save image
if is_prep == False:
mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png")
if is_prep == True:
return mask_img
# If this is a building, we need to render the shadow here, as we only know the height
# of the building in this loop.
if mstr_shadow_enabled == True and is_prep == False:
@ -265,3 +263,6 @@ class mstr_maskgen:
# Inform
mstr_msg("maskgen", "Mask built.")
# Return the image
return mask_img

7
og.py
View File

@ -49,7 +49,12 @@ if cli == True:
# Create the class and init values
og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd(), prep)
og._buildTile()
if prep == True:
og._prepareTile()
if prep == False:
og._generateOrthos_mt(int(sys.argv[3]))
# Only if we find enough arguments, proceed.

View File

@ -13,6 +13,8 @@
import math
import os
import glob
import threading
from multiprocessing import Process
from defines import *
from log import *
from osmxml import *
@ -147,8 +149,140 @@ class mstr_orthographic:
top_lat = cur_tile_y
# Builds and processes the tile with everything required, in one call.
def _buildTile(self):
# Start the multi-threaded build of all orthos
# amtsmt = AmountSimultaneous - so how many orthos you want to
# generate at the same time. You may need to fine tune this value
# so that you don't overload your machine.
def _generateOrthos_mt(self, amtsmt):
# Need to know maximum values first
bb_lat = self._lat
bb_lng = self._long
bb_lat_edge = self._lat+self._vstep
bb_lng_edge = self._long+mstr_zl_18
mlat = 1
mlng = 1
while bb_lat < self._lat + 1:
bb_lat = bb_lat + self._vstep
mlat = mlat+1
while bb_lng < self._long + 1:
bb_lng = bb_lng + mstr_zl_18
mlng = mlng+1
mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng))
maxlatlng = [ mlat, mlng ]
procs = []
for p in range(1, amtsmt+1):
proc = Process(target=self._buildOrtho, args=[1, p, amtsmt])
procs.append(proc)
proc.start()
mstr_msg("orthographic", "Ortho threads started")
# Starts a threading loop to build orthos, with the defined starting point in
# the lat-lng grid. You will also need to provide the horizontal stepping so
# that the thread keeps running.
def _buildOrtho(self, v, h, step):
# Starting point
grid_lat = v
grid_lng = h
# The tile is constructed of many smaller parts. We walk through the
# smallest possible, from which the bigger ones are later built.
bb_lat = self._lat
bb_lng = self._long
bb_lat_edge = self._lat+self._vstep
bb_lng_edge = self._long+mstr_zl_18
# We need to know the highest possible latitude and longitude tile numbers,
# in case we render at the edge
mlat = 1
mlng = 1
while bb_lat < self._lat + 1:
bb_lat = bb_lat + self._vstep
mlat = mlat+1
while bb_lng < self._long + 1:
bb_lng = bb_lng + mstr_zl_18
mlng = mlng+1
mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng))
maxlatlng = [ mlat, mlng ]
while grid_lat <= maxlatlng[0]:
# Reset these two
bb_lat = self._lat + ((grid_lat-1)*self._vstep)
bb_lng = self._long + ((grid_lng-1)*mstr_zl_18)
bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep
bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18
osmxml = mstr_osmxml()
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
osmxml.acquire_osm(grid_lat, grid_lng)
# Let the user know
mstr_msg("orthographic", "Generating orthophoto " + str(grid_lat) + "-" + str(grid_lng))
# Get the data
#osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
#mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
# Check for work to be done
layers = self.determineLayerWork(osmxml)
# We need to walk through the array of layers,
# in their z-order.
# For each layer, we will generate the mask, the layer image
# itself, and finally, compose the ortho photo.
mstr_msg("orthographic", "Beginning generation of layers")
# In here we store the layers
photolayers = []
# The masks are handed to layergen in sequence. The layers are then
# in turn handed to photogen.
curlyr = 1
for layer in layers:
# Let the user know
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Generate the mask
mg = mstr_maskgen( [self._lat, grid_lat, self._long, grid_lng], self._vstep, layer[0], layer[1], layer[2])
if layer[0] == "building":
mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
mg.set_latlng_numbers(self._lat, grid_lat, self._long, grid_lng)
mask = mg._build_mask(osmxml)
# Generate the layer
lg = mstr_layergen(layer[0], layer[1], self._lat, grid_lat, self._long, grid_lng, layer[2])
lg.set_max_latlng_tile(maxlatlng)
lg.set_latlng_folder(self._latlngfld)
#lg.open_db()
lg.open_tile_info()
photolayers.append(lg.genlayer(mask, osmxml))
curlyr = curlyr+1
mstr_msg("orthographic", "All layers created")
# We should have all layers now.
# Snap a photo with our satellite :)
mstr_msg("orthographic", "Generating ortho photo")
pg = mstr_photogen(self._lat, self._long, grid_lat, grid_lng, maxlatlng[0], maxlatlng[1])
pg.genphoto(photolayers)
mstr_msg("orthographic", " -- Ortho photo generated -- ")
print("")
print("")
# Perform adjustment of grid position
n_lng = grid_lng + step
if n_lng > maxlatlng[1]:
np = n_lng - maxlatlng[1]
grid_lng = np
grid_lat = grid_lat+1
else:
grid_lng = n_lng
# Prepares the entire tile
def _prepareTile(self):
mstr_msg("orthographic", "Beginning construction of tile")
# We need to know which platform we are on
@ -239,194 +373,52 @@ class mstr_orthographic:
# Previously, I downloaded all XML files in one go - but to ease the
# stress on OSM servers and my server, we will do acquire the data
# only for the current processed part of the tile.
if self._prep == True:
for lat_grid in range(1, maxlatlng[0]+1):
for lng_grid in range(1, maxlatlng[1]+1):
# Adjust bounding box
osmxml = mstr_osmxml()
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
osmxml.acquire_osm(lat_grid, lng_grid)
mstr_msg("orthographic", "Adjusted bounding box for XML object")
for lat_grid in range(1, maxlatlng[0]+1):
for lng_grid in range(1, maxlatlng[1]+1):
# Adjust bounding box
osmxml = mstr_osmxml()
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
osmxml.acquire_osm(lat_grid, lng_grid)
mstr_msg("orthographic", "Adjusted bounding box for XML object")
# Check for work to be done
layers = self.determineLayerWork(osmxml)
curlyr = 1
for layer in layers:
if layer[2] == False and layer[0] != "building":
# Let the user know
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
mask = mg._build_mask(osmxml, is_prep=True) # We need an object here
tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False)
tp._prepareTile()
curlyr = curlyr+1
# Adjust longitude coordinates
cur_tile_x = cur_tile_x+1
bb_lng = bb_lng + mstr_zl_18
bb_lng_edge = bb_lng_edge + mstr_zl_18
mstr_msg("orthographic", "Adjustment of longitude performed")
# Adjust peak longitude tile number
if cur_tile_x > top_lng:
top_lng = cur_tile_x
# Adjust latitude and all other values when we get here
cur_tile_y = cur_tile_y+1
cur_tile_x = 1
bb_lng = self._long
bb_lng_edge = self._long + mstr_zl_18
bb_lat = bb_lat + self._vstep
bb_lat_edge = bb_lat_edge + self._vstep
mstr_msg("orthographic", "Adjustment of latitude performed")
# Adjust peak latitude number
if cur_tile_y > top_lat:
top_lat = cur_tile_y
# Need to differentiate
if self._prep == False:
# The tile is constructed of many smaller parts. We walk through the
# smallest possible, from which the bigger ones are later built.
bb_lat = self._lat
bb_lng = self._long
bb_lat_edge = self._lat+self._vstep
bb_lng_edge = self._long+mstr_zl_18
cur_tile_x = 1
cur_tile_y = 1
osmxml = mstr_osmxml(0,0)
# Previously, I downloaded all XML files in one go - but to ease the
# stress on OSM servers and my server, we will do acquire the data
# only for the current processed part of the tile.
for lat_grid in range(1, maxlatlng[0]+1):
for lng_grid in range(1, maxlatlng[1]+1):
# Adjust bounding box
osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
mstr_msg("orthographic", "Adjusted bounding box for XML object")
# Determine what to do... maybe work was interrupted
if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False:
# Check for work to be done
layers = self.determineLayerWork(osmxml)
curlyr = 1
for layer in layers:
if layer[2] == False and layer[0] != "building":
# Let the user know
mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Get the data
#osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
#mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
mask = mg._build_mask(osmxml, is_prep=True) # We need an object here
# Check for work to be done
layers = self.determineLayerWork()
tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False)
tp._prepareTile()
# We need to walk through the array of layers,
# in their z-order.
# For each layer, we will generate the mask, the layer image
# itself, and finally, compose the ortho photo.
mstr_msg("orthographic", "Beginning generation of layers")
curlyr = curlyr+1
curlyr = 1
for layer in layers:
# Let the user know
mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
# Generate the mask
mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
if layer[0] == "building":
mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
mg._build_mask()
# Generate the layer
lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
lg.set_max_latlng_tile(maxlatlng)
lg.set_latlng_folder(self._latlngfld)
#lg.open_db()
lg.open_tile_info()
lg.genlayer()
curlyr = curlyr+1
mstr_msg("orthographic", "All layers created")
# We should have all layers now.
# Snap a photo with our satellite :)
mstr_msg("orthographic", "Generating ortho photo")
pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x, maxlatlng[0], maxlatlng[1])
pg.genphoto()
mstr_msg("orthographic", " -- Ortho photo generated -- ")
print("")
print("")
# Adjust longitude coordinates
cur_tile_x = cur_tile_x+1
bb_lng = bb_lng + mstr_zl_18
bb_lng_edge = bb_lng_edge + mstr_zl_18
mstr_msg("orthographic", "Adjustment of longitude performed")
# Adjust peak longitude tile number
if cur_tile_x > top_lng:
top_lng = cur_tile_x
# Clear out cache
"""
if mstr_clear_cache == True:
ch = glob.glob(mstr_datafolder + "_cache/*")
for f in ch:
if os_platform == "nt":
if self._isFileAccessibleWin(f) == True:
os.remove(f)
if os_platform == "posix":
if self._isFileAccessiblePosix(f) == True:
os.remove(f)
mstr_msg("orthographic", "Cleared cache")
"""
# Adjust latitude and all other values when we get here
cur_tile_y = cur_tile_y+1
cur_tile_x = 1
bb_lng = self._long
bb_lng_edge = self._long + mstr_zl_18
bb_lat = bb_lat + self._vstep
bb_lat_edge = bb_lat_edge + self._vstep
mstr_msg("orthographic", "Adjustment of latitude performed")
# Adjust peak latitude number
if cur_tile_y > top_lat:
top_lat = cur_tile_y
mstr_msg("orthographic", "Generation of all tiles completed!")
# Complete scenery
if mstr_xp_genscenery == True:
scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld)
scn.acquire_elevation_data()
scn.acquire_xes_data()
scn.build_mesh_script()
scn.build_mesh()
scn.build_ter_files()
mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed")
mstr_msg("orthographic", "Final step completed.")
mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld)
print("")
mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
print("")
# Let's leave this out for the moment
"""
mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles")
tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng)
tg.genTiles()
mstr_msg("orthographic", "Final step completed.")
print("")
mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng)
print("")
print("")
mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
print("")
"""
# Adjust longitude coordinates
cur_tile_x = cur_tile_x+1
bb_lng = bb_lng + mstr_zl_18
bb_lng_edge = bb_lng_edge + mstr_zl_18
mstr_msg("orthographic", "Adjustment of longitude performed")
# Adjust peak longitude tile number
if cur_tile_x > top_lng:
top_lng = cur_tile_x
# Adjust latitude and all other values when we get here
cur_tile_y = cur_tile_y+1
cur_tile_x = 1
bb_lng = self._long
bb_lng_edge = self._long + mstr_zl_18
bb_lat = bb_lat + self._vstep
bb_lat_edge = bb_lat_edge + self._vstep
mstr_msg("orthographic", "Adjustment of latitude performed")
# Adjust peak latitude number
if cur_tile_y > top_lat:
top_lat = cur_tile_y

View File

@ -4,6 +4,7 @@ from PIL import Image, ImageFilter, ImageEnhance
from defines import *
from layergen import *
from log import *
from functions import *
# -------------------------------------------------------------------
# ORTHOGRAPHIC
@ -38,13 +39,14 @@ class mstr_photogen:
# This puts it all together. Bonus: AND saves it.
def genphoto(self):
def genphoto(self, layers):
# Template for the file name which is always the same
root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_"
#root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_"
# First, we walk through all layers and blend them on top of each other, in order
mstr_msg("photogen", "Merging layers")
"""
# Note if we added building shadows
bldg_shadow_added = False
@ -86,8 +88,11 @@ class mstr_photogen:
#bldg_final.alpha_composite(bldg_shadow)
bldg_final.alpha_composite(tree_main)
bldg_final.alpha_composite(bldg_main)
"""
for l in mstr_ortho_layers:
for l in layers:
self._tile.alpha_composite(l)
"""
if os.path.isfile(root_filename + l[0] + "-" + l[1] + "_layer.png"):
# Need to divert in case we have shadows
if mstr_shadow_enabled == True:
@ -117,9 +122,10 @@ class mstr_photogen:
layer = layer.filter(ImageFilter.GaussianBlur(radius=0.2))
# Converge the layer with this image
self._tile.alpha_composite(layer)
"""
# Drop the buildings on top
self._tile.alpha_composite(bldg_final)
#self._tile.alpha_composite(bldg_final)
# When we have run through this loop, we will end up with a sandwiched
@ -136,29 +142,68 @@ class mstr_photogen:
# If this check comes back as true, we need to perform
# aforementioned fix:
if emptyspace == True:
# Choose a suitable layer type
tag = "landuse"
value = "meadow"
mstr_msg("photogen", "Patching empty space")
self.buildCompletionMask()
mask = self.buildCompletionMask()
# Generate the layer as if it were part of the OSM data
lg = mstr_layergen(tag, value, self._lat, self._ty, self._lng, self._tx, False, is_completion=True)
lg.set_max_latlng_tile(self._maxlatlng)
lg.set_latlng_folder(self._latlngfld)
#lg.open_db()
lg.open_tile_info()
lg.genlayer()
# Load the mask
mask_px = mask.load()
# Load the image
completion = Image.open(root_filename + "tile-completion_layer.png")
cmpl = Image.new("RGBA", (self._imgsize, self._imgsize))
cmp_px = cmpl.load()
edn = self.find_earthnavdata_number()
edns = self.latlng_folder(edn)
idx = 0
for r in mstr_completion_colors:
if r[0] == edns:
break
else:
idx = idx+1
for y in range(self._imgsize):
for x in range(self._imgsize):
p = mask_px[x,y]
if p[3] > 0:
cidx = randrange(0, len(mstr_completion_colors[idx][1])-1)
clr = mstr_completion_colors[idx][1][cidx]
cmp_px[x,y] = (clr[0], clr[1], clr[2], 255)
# Some features
patches = glob.glob(mstr_datafolder + "textures/tile/completion/*.png")
# Pick an amount of features to add
patch_amt = randrange(1, 7)
# Add those somewhere
for p in range(1, patch_amt+1):
# Load some patch
ptc = Image.open(mstr_datafolder + "textures/tile/completion/p" + str(randrange(1, len(patches)+1)) + ".png")
# Rotate it
ptc = ptc.rotate(randrange(0, 360), expand=True)
# Adjust alpha on this image
ptc_p = ptc.load()
for y in range(ptc.height):
for x in range(ptc.width):
c = ptc_p[x,y]
if c[3] > 0:
na = c[3] - 160
if na < 0: na = 0
nc = (c[0], c[1], c[2], na)
ptc_p[x,y] = nc
# Find a location INSIDE the image!
px = randrange(1, randrange(self._imgsize - ptc.width - 1))
py = randrange(1, randrange(self._imgsize - ptc.height - 1))
# Add it to the completion image
cmpl.alpha_composite(ptc)
# Merge the images
completion.alpha_composite(self._tile)
cmpl.alpha_composite(self._tile)
# Make this the real one
self._tile = completion
self._tile = cmpl
# There may be some tiles that have a larger sea or even an ocean in them - these need to be
@ -262,8 +307,9 @@ class mstr_photogen:
# We do not apply any blur or other effects here - we only want the
# exact pixel positions.
mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" )
#mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" )
mstr_msg("photogen", "Generated and saved empty space mask")
return mask
# Construct a folder name for latitude and longitude
@ -281,3 +327,13 @@ class mstr_photogen:
if abs(numbers[1]) >= 100 : fstr = fstr + str(numbers[1])
return fstr
# Find the next "by-ten" numbers for the current latitude and longitude
def find_earthnavdata_number(self):
earthnavdata = []
lat = abs(int(self._lat / 10) * 10)
lng = abs(int(self._lng / 10) * 10)
earthnavdata.append(lat)
earthnavdata.append(lng)
return earthnavdata

View File

@ -6,6 +6,16 @@ While tools like Ortho4XP or AutoOrtho can provide next-level realism into X-Pla
Currently, zoom level 18 is implemented.
Orthographic | Real World
[img]comp_1_1.jpg[/img]
[img]comp_2_11.jpg[/img]
[img]comp_2_23.jpg[/img]
[section]Wait, just what are ortho photos?[/section]
Aerial photos taken with a satellite, of varying resolution, altitude, and detail. When you switch to satellite in Google or Bing Maps - what you see is a collage of ortho photos, depending on how close or far away you are zoomed in.
@ -70,8 +80,7 @@ Apart from that I am aware that the code is most likely not the best and can be
[section]Requirements[/section]
- Current Python version (3.10 and up)
- Python modules: Pillow (formerly PIL), wand, numpy
- SQLite must be available to Python. On Windows it is by default, you may need to install the sqlite3 developer libraries on your distribution, then compile Python to automatically avail of this built-in module. On my Endeavour OS install, SQLite was already built-in to the Python package.
- Python modules: Pillow (formerly PIL), requests, numpy
[section]Configuration[/section]
@ -82,11 +91,7 @@ You will find various configuration options, each of which is documented what it
In this file you can also define how large you want the final products to be - you can choose between 4k resolution (2048px) or 16k resolution (4096). The choice you make here should be based on 1) how much detail you want, and 2) how much VRAM your GPU has. The larger a texture, the more VRAM is needed, and the more processing is required by the GPU. I personally go with 2048, as I have a RTX2060, so my VRAM is limited. When at altitudes of 10,000 feet and above, you will most definitely not notice the difference between 4096 and 2048.
NOTE! 4096 not yet implemented, but planned.
Change the mstr_photores variable to 4096 if you want the maximum possible.
Just a note: 4096 also uses 4x more hard drive space. 4k uses about 2MB per image, where as 16k uses about 8-10MB per image. This may not seem much for one image, but keep in mind we are talking about quite a considerable number of images. To get an idea - if you have it, look into any folder of an Ortho4XP tile in your X-Plane folder, and check the size of the "textures" folder.
NOTE! 4096 not yet implemented.
Also in defines.py, you will find the single layers, along with their corresponding mask blurring values. It is my strong recommendation NOT to change these values, as I have taken a lot of time to fine-tune these values.
@ -95,35 +100,38 @@ Also in defines.py, you will find the single layers, along with their correspond
Very simple.
[codebox]python og.py LATITUDE LONGITUDE[/codebox]
First step:
[codebox]python og.py LATITUDE LONGITUDE true[/codebox]
So for example
[codebox]python main.py 51 7[codebox]
[codebox]python main.py 51 7 true[codebox]
This will generate a complete dataset containing which sources to use in which quadrant of the image grid.
Then:
[codebox]python og.py LATITUDE LONGITUDE NUMBER_OF_SIMULTANEOUS_IMAGES_TO_PRODUCE[/codebox]
So for example
[codebox]python main.py 51 7 32[codebox]
This will generate 32 "orthos" at once. Which number you can use highly depends on the performance of your hardware.
This does everything for you.
ATTENTION! This process will take a considerable amount of time due to the work being involved. Please keep this in mind, especially if you seek to do this for an entire country.
For X-Plane scenery, you will need to do this once step 2 is done:
[codebox]python main.py 51 7 xpscenery[codebox]
This will generate the terrain mesh and associate the textures with the mesh.
I am considering hosting the finished tiles on my server, so that you only need to download the finished photos / tile packs.
[section]Examples[/section]
Dortmund, Germany
[REAL WORLD]
[RESULT]
Leverkusen, Germany
[REAL WORLD]
[RESULT]
[section]Improving the code[/section]
As I have hosted this on my private but publicly accessible git repository, it should be clear that I am making the code available to everyone. Of course, you are free to improve (I am sure the code needs some optimisation). If you changed or improved something, and you publish it (no matter where), you MUST adhere to the license this software ships with, which is OSL 3.0.

View File

@ -86,6 +86,10 @@ class mstr_xp_normalmap:
mstr_msg("xp_normalmap", "[X-Plane] Beginning normal map generation")
# No specularity, no reflectivity - but standard color
# Blue (reflectivity) and alpha (specularity) need to be 1 - but can be adjusted as needed
# Resize original
image = image.resize((int(mstr_photores/4), int(mstr_photores/4)), Image.Resampling.BILINEAR)
nmp = Image.new("RGBA", (image.width, image.height), (128,128,1,1))
org = image.load()
nmp_pix = nmp.load()
@ -137,13 +141,13 @@ class mstr_xp_normalmap:
# The funnction to call. Blends with the existing map, or creates a new one
def build_normalmap(self):
def build_normalmap(self, layer):
mstr_msg("xp_normalmap", "[X-Plane] Building normal map")
# The layer image
lyr = self.load_layer()
#lyr = self.load_layer()
# Make the normal map for the layer
nrm = self.generate_normal_map_for_layer(lyr)
nrm = self.generate_normal_map_for_layer(layer)
# Normal map final file name
nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._tv) + "_" + str(self._th) + ".png"