From 88bcdf4a7a60d4213b336db77c3b64a41d7e6f62 Mon Sep 17 00:00:00 2001 From: "Marcus Str." Date: Sat, 16 Nov 2024 15:20:00 +0100 Subject: [PATCH] Milestone commit - switch to mostly in-memory management, and functioning multi-processing for ortho generation --- defines.py | 18 +- layergen.py | 459 ++++++++++++++++-------------------------------- maskgen.py | 7 +- og.py | 7 +- orthographic.py | 360 +++++++++++++++++++------------------ photogen.py | 96 +++++++--- repoinfo | 46 +++-- xp_normalmap.py | 10 +- 8 files changed, 459 insertions(+), 544 deletions(-) diff --git a/defines.py b/defines.py index e9553ca..dea842c 100644 --- a/defines.py +++ b/defines.py @@ -148,7 +148,6 @@ mstr_ortho_layers = [ ("highway", "tertiary", 20), ("highway", "unclassified", 17), ("highway", "living_street", 12), - ("waterway", "river", 10), ("waterway", "stream", 2), ("amenity", "parking", "amenity", "parking"), ("amenity", "school", "amenity", "school"), @@ -162,12 +161,8 @@ mstr_ortho_layers = [ ("natural", "sand", "natural", "sand"), ("natural", "desert", "natural", "desert"), # Z-Order 3 - ("natural", "water", "natural", "water"), ("natural", "bay", "natural", "beach"), ("natural", "beach", "natural", "beach"), - ("water", "lake", "natural", "water"), - ("water", "pond", "natural", "water"), - ("water", "river", "natural", "water"), ("leisure", "swimming_pool", "natural", "water"), ("highway", "pedestrian", 4), # Z-Order 4 @@ -195,6 +190,11 @@ mstr_ortho_layers = [ ("building", "public", "building", "public"), ("building", "commercial", "building", "commercial"), ("building", "yes", "building", "common"), + ("water", "lake", "natural", "water"), + ("water", "pond", "natural", "water"), + ("water", "river", "natural", "water"), + ("natural", "water", "natural", "water"), + ("waterway", "river", 10), ("place", "sea", "natural", "sea"), ("place", "ocean", "natural", "sea") ] @@ -229,10 +229,10 @@ mstr_mask_blur = [ ("landuse", "farmland", 10), ("landuse", "farmyard", 10), # Z-Order 2 - ("landuse", "forest", 8), - ("leisure", "nature_reserve", 8), - ("natural", "wood", 8), - ("natural", "tree_row", 8), + ("landuse", "forest", 10), + ("leisure", "nature_reserve", 10), + ("natural", "wood", 10), + ("natural", "tree_row", 10), ("landuse", "military", 15), # Z-Order 3 ("natural", "bare_rock", 25), diff --git a/layergen.py b/layergen.py index 6c02f9b..21520ce 100644 --- a/layergen.py +++ b/layergen.py @@ -114,27 +114,63 @@ class mstr_layergen: return layer_final + # Find the source to use pre-determined in phase one + def findLayerSource(self): + # The source number + src = -1 + + # The already existing source data + srcfile = mstr_datafolder + "z_orthographic/data/" + self._latlngfld + "/" + str(self._lat_number) + "_" + str(self._lng_number) + + # Let's open the file and find our entry + with open(srcfile) as file: + for line in file: + linedata = line.split(" ") + if linedata[2] == self._tag and linedata[3] == self._value: + src = int(linedata[4]) + break + + # Should we encounter a -1 at this point, we can choose something + # It means it touches no border as it was not found in the file + if src == -1: + root_folder = mstr_datafolder + "textures/" + for s in mstr_ortho_layers: + if s[0] == self._tag and s[1] == self._value: + fld_main = len(s)-2 + fld_sub = len(s)-1 + root_folder = root_folder + s[fld_main] + "/" + s[fld_sub] + + brd = glob.glob(root_folder + "/brd/b*.png") + src = randrange(1, len(brd)+1) + + return src + + + # This generates the layer from the defined mask - def genlayer(self): + def genlayer(self, mask, xml): mstr_msg("layergen", "Layer to be generated: " + str(self._latitude) + "-" + str(self._lat_number) + ":" + str(self._longitude) + "-" + str(self._lng_number) + " -- tag: " + self._tag + " - value: " + self._value ) # Before we generate the layer, let's check for airports in this chunk mstr_msg("layergen", "Checking for airport/s with ICAO code") - osmxml = mstr_osmxml(0,0) - icao = osmxml.find_icao_codes(mstr_datafolder + "_cache/tile.xml") - mstr_msg("layergen", "Found " + str(len(icao)) + " airport/s") + + icao = None + if xml != None: + icao = xml.find_icao_codes() + mstr_msg("layergen", "Found " + str(len(icao)) + " airport/s") # Runway surface, if any other than concrete/asphalt rw_surface = "" # If we find an airport, make a note ... - if len(icao) >= 1: - #for i in icao: - # ... but only, if this airport is not already noted - #iccheck = self._tiledb.perform_query("SELECT * FROM airports WHERE icao='" + i +"';") - #if len(iccheck) == 0: - #self._tiledb.insert_icao(i, self._lat_number, self._lng_number, self._latitude, self._longitude) - # mstr_msg("layergen", "Airport/s noted in data file") - rw_surface = osmxml.find_runway_surface(mstr_datafolder + "_cache/tile.xml") + if icao != None: + if len(icao) >= 1 and self._is_completion == False: + #for i in icao: + # ... but only, if this airport is not already noted + #iccheck = self._tiledb.perform_query("SELECT * FROM airports WHERE icao='" + i +"';") + #if len(iccheck) == 0: + #self._tiledb.insert_icao(i, self._lat_number, self._lng_number, self._latitude, self._longitude) + # mstr_msg("layergen", "Airport/s noted in data file") + rw_surface = xml.find_runway_surface() # The image for the layer itself layer = Image.new("RGBA", (self._imgsize, self._imgsize)) @@ -153,176 +189,7 @@ class mstr_layergen: root_folder = root_folder + s[fld_main] + "/" + s[fld_sub] # Determine which sources to use. - # First, we need to check for adjacent tile information. We then either - # need to use the source of any adjacent tile, or we can choose freely. - src = -1 - - # Find our adjacent tiles - adjtiles = findAdjacentTilesTo(self._lat_number, self._lng_number) - - mstr_msg("layergen", "Performing adjacency check") - # Walk through each tile and see what we can find in relation to this - # tile in the center - # Since we already know the order in adjtiles, we can do this real easy - if self._is_completion == False: - at = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top - ar = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right - ab = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom - al = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left - #at = self._tiledb.get_adjacency_for_source(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top - #ar = self._tiledb.get_adjacency_for_source(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right - #ab = self._tiledb.get_adjacency_for_source(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom - #al = self._tiledb.get_adjacency_for_source(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left - if self._is_completion == True: - at = self._tileinfo.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top - ar = self._tileinfo.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right - ab = self._tileinfo.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom - al = self._tileinfo.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left - #at = self._tiledb.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top - #ar = self._tiledb.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right - #ab = self._tiledb.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom - #al = self._tiledb.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left - - if len(at) == 4: - self._tag = at[0] - self._value = at[1] - if len(ar) == 4: - self._tag = ar[0] - self._value = ar[1] - if len(ab) == 4: - self._tag = ab[0] - self._value = ab[1] - if len(al) == 4: - self._tag = al[0] - self._value = al[1] - - root_folder = mstr_datafolder + "textures/" - for s in mstr_ortho_layers: - if s[0] == self._tag and s[1] == self._value: - fld_main = len(s)-2 - fld_sub = len(s)-1 - root_folder = root_folder + s[fld_main] + "/" + s[fld_sub] - - # We are south to the top tile. - if len(at) == 2 and src == -1: - if "b" in at[1]: src = at[0] - # We are west to the right tile. - if len(ar) == 2 and src == -1: - if "l" in ar[1]: src = ar[0] - # We are north to the bottom tile. - if len(ab) == 2 and src == -1: - if "t" in ab[1]: src = ab[0] - # We are east to the left tile. - if len(al) == 2 and src == -1: - if "r" in al[1]: src = al[0] - - # Should we be at the border of the degree for latitude and longitude, we need to perform - # additional checks - is_deg_brd_t = False - is_deg_brd_r = False - is_deg_brd_b = False - is_deg_brd_l = False - if self._lat_number == 1: is_deg_brd_b = True - if self._lat_number == self._maxlat: is_deg_brd_t = True - if self._lng_number == 1: is_deg_brd_l = True - if self._lng_number == self._maxlng: is_deg_brd_r = True - - # Adjacent latitude and longitude tiles - deg_tiles = [] - deg_tiles.append( ( self._latitude+1, self._longitude ) ) # Top - deg_tiles.append( ( self._latitude, self._longitude+1 ) ) # Right - deg_tiles.append( ( self._latitude-1, self._longitude ) ) # Bottom - deg_tiles.append( ( self._latitude, self._longitude-1 ) ) # Left - - # Perform degree border checks - # - and make sure we do not run into errors - this drove me crazy in testing - atd = [] - ard = [] - abd = [] - ald = [] - if is_deg_brd_t == True: - if self._is_completion == False: - atd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], self._tag, self._value, 1, self._lng_number, "b") # Top - #atd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, self._tag, self._value) # Top - if self._is_completion == True: - atd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, "b") - #atd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number) # Top - - if is_deg_brd_r == True: - if self._is_completion == False: - ard = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._tag, self._value, self._lat_number, 1, "l") # Right - #ard = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, self._tag, self._value) # Right - if self._is_completion == True: - ard = self._tileinfo.get_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, "l") - #ard = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1) # Right - - if is_deg_brd_b == True: - #maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude-1, self._longitude) - if self._is_completion == False: - abd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "t") - #abd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number, self._tag, self._value) # Bottom - if self._is_completion == True: - abd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "t") - #abd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number) # Bottom - - if is_deg_brd_l == True: - #maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude, self._longitude-1) - if self._is_completion == False: - ald = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "r") - #ald = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1], self._tag, self._value) # Left - if self._is_completion == True: - ald = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "r") - #ald = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1]) # Left - - if (is_deg_brd_t == True or is_deg_brd_r == True or is_deg_brd_b == True or is_deg_brd_l == True): - if src == -1 and self._is_completion == True: - if len(atd) == 4: - self._tag = atd[0] - self._value = atd[1] - if len(ard) == 4: - self._tag = ard[0] - self._value = ard[1] - if len(abd) == 4: - self._tag = abd[0] - self._value = abd[1] - if len(ald) == 4: - self._tag = ald[0] - self._value = ald[1] - - root_folder = mstr_datafolder + "textures/" - for s in mstr_ortho_layers: - if s[0] == self._tag and s[1] == self._value: - fld_main = len(s)-2 - fld_sub = len(s)-1 - root_folder = root_folder + s[fld_main] + "/" + s[fld_sub] - - - # Should we get here and one of the degree border checks turns out true, - # we need to make sure that we select the source we found. This should - # enable seamless tiling... around the entire planet - if is_deg_brd_t == True and len(at) == 0 and src == -1: - if len(atd) == 2: - if "b" in atd[3]: src = int(atd[2]) - if is_deg_brd_r == True and len(ar) == 0 and src == -1: - if len(ard) == 2: - if "l" in ard[3]: src = int(ard[2]) - if is_deg_brd_b == True and len(ab) == 0 and src == -1: - if len(abd) == 2: - if "t" in abd[3]: src = int(abd[2]) - if is_deg_brd_l == True and len(al) == 0 and src == -1: - if len(ald) == 2: - if "r" in ald[3]: src = int(ald[2]) - - mstr_msg("layergen", "Adjacency check completed") - - brd = glob.glob(root_folder + "/brd/b*.png") - - # If the adjacency check returned nothing (src is still -1), - # then pick something - if src == -1: - if len(brd) == 1: src=1 - if len(brd) >= 2: - src = randrange(1, len(brd)+1) + src = self.findLayerSource() ptc = glob.glob(root_folder + "/ptc/b" + str(src) + "_p*.png") @@ -333,25 +200,27 @@ class mstr_layergen: ptc_src.append(Image.open(p)) mstr_msg("layergen", "Layer sources selected") + """ # OK! Load the mask if self._is_completion == False: - osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) + mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) if self._is_completion == True: - osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" ) + mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" ) + """ # Generate an edge mask from the original - osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES) + osm_edge = mask.filter(ImageFilter.FIND_EDGES) osm_edge = osm_edge.filter(ImageFilter.MaxFilter) mstr_msg("layergen", "Edge mask generated") # This adds some natural looking shapes to these types of features if self._value == "forest" or self._value == "nature_reserve": epx = osm_edge.load() - imgd = ImageDraw.Draw(osm_mask) + imgd = ImageDraw.Draw(mask) - # Walk through a grid of 200x200 - on the edge image - for y in range(0, osm_mask.height, int(osm_mask.height/200)): - for x in range(0, osm_mask.width, int(osm_mask.width/200)): + # Walk through a grid of 100x100 - on the edge image + for y in range(0, mask.height, int(mask.height/100)): + for x in range(0, mask.width, int(mask.width/100)): px = epx[x,y] if px[3] == 255: rx = randrange(24,60) @@ -363,7 +232,7 @@ class mstr_layergen: psy = randrange(y-11, y+11) # Do some magic - but not on edges - if x > 0 and x < osm_mask.width and y > 0 and y < osm_mask.height: + if x > 0 and x < mask.width and y > 0 and y < mask.height: if f != 5: imgd.ellipse((psx-int(rx/2), psy-int(ry/2), psx+rx, psy+ry), fill="black") if f == 3 or f == 7: @@ -372,14 +241,14 @@ class mstr_layergen: # We need to change the image in certain conditions if self._value == "hedge" and self._tag == "barrier": - osm_mask = osm_edge + mask = osm_edge # From here on in we will need to perform some adjustments on the masks, depending # on what they are. for i in mstr_mask_blur: if i[0] == self._tag and i[1] == self._value: if self._tag != "place" and (self._value != "sea" or self._value != "ocean"): - osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2])) + mask = mask.filter(ImageFilter.BoxBlur(radius=i[2])) break # Begin producing a largely random image @@ -414,7 +283,7 @@ class mstr_layergen: ly = randrange( self._imgsize - img.height ) layer.alpha_composite( img, (lx, ly) ) if self._is_completion == True: - mp = osm_mask.load() + mp = mask.load() edn = self.xplane_latlng_folder(self.find_earthnavdata_number()) idx = 0 for r in mstr_completion_colors: @@ -483,7 +352,7 @@ class mstr_layergen: # Generate the layer from the mask. layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize)) layer_pix = layer.load() - mask_pix = osm_mask.load() + mask_pix = mask.load() layer_comp_pix = layer_comp.load() for y in range(self._imgsize): for x in range(self._imgsize): @@ -505,9 +374,10 @@ class mstr_layergen: # Here we want to make sure that the generated image fits well with others, so # let's do that. mstr_msg("layergen", "Generating adjacent fades") - self.generate_adjacent_fades() + adjfade = self.generate_adjacent_fades(mask) # Determine if there are any fades, and if so, fade those in first before we save the layer + """ fade_fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value if os.path.isfile(fade_fn + "_fade_top.png") == True: fade = Image.open(fade_fn + "_fade_top.png") @@ -521,6 +391,9 @@ class mstr_layergen: if os.path.isfile(fade_fn + "_fade_left.png") == True: fade = Image.open(fade_fn + "_fade_left.png") layer_comp.alpha_composite(fade) + """ + + layer_comp.alpha_composite(adjfade) mstr_msg("layergen", "Adjacent fading completed") @@ -551,10 +424,10 @@ class mstr_layergen: layer_comp_pix[x,y] = (lpix[0], lpix[1], lpix[2], 255-rpix[3]) # Store layer - if self._is_completion == False: - layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) - if self._is_completion == True: - layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" ) + #if self._is_completion == False: + # layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) + #if self._is_completion == True: + # layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" ) #layer_final.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) mstr_msg("layergen", "Layer image finalized and saved.") @@ -570,7 +443,7 @@ class mstr_layergen: break if nm == True: nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld) - nrm.build_normalmap() + nrm.build_normalmap(layer_comp) # Let's try our hand at pseudo shadows @@ -581,7 +454,7 @@ class mstr_layergen: if self._tag == sh[0] and self._value == sh[1]: mstr_msg("layergen", "Generating shadow for layer") shadow_pix = shadow.load() - mask_pix = osm_mask.load() + mask_pix = mask.load() shf = 1 while shf < mstr_shadow_shift: for y in range(self._imgsize): @@ -602,73 +475,12 @@ class mstr_layergen: shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1.5)) - shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png") + #shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png") + shadow.alpha_composite(layer_comp) + layer_comp = shadow mstr_msg("layergen", "Shadow layer completed") - - # Check if pixels touch the borders of the image, and if so - - # make a not of that in the database. - at=False - ar=False - ab=False - al=False - layer_pix = layer_comp.load() # <- Just to be safe - - # Top scan - for i in range(0, self._imgsize-1): - p = layer_pix[i,0] - if p[3] > 0: - at=True - break - - # Right scan - for i in range(0, self._imgsize-1): - p = layer_pix[self._imgsize-1,i] - if p[3] > 0: - ar=True - break - - # Bottom scan - for i in range(0, self._imgsize-1): - p = layer_pix[i,self._imgsize-1] - if p[3] > 0: - ab=True - break - - # Left scan - for i in range(0, self._imgsize-1): - p = layer_pix[1,i] - if p[3] > 0: - al=True - break - - # Construct DB String - adjstr = "" - if at==True: adjstr = adjstr + "t" - if ar==True: adjstr = adjstr + "r" - if ab==True: adjstr = adjstr + "b" - if al==True: adjstr = adjstr + "l" - - # Store into DB - but only if there is something to store - if adjstr != "": - if self._is_completion == False: - r = self._tileinfo.get_adjacency_for_tag_and_value(self._lat_number, self._lng_number, self._tag, self._value) - #r = self._tiledb.get_adjacency_for_source(self._lat_number, self._lng_number, self._tag, self._value) - if len(r) == 0: - self._tileinfo.add_adjacency_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr) - mstr_msg("layergen", "Adjacency info stored in database") - - if self._is_completion == True: - r = self._tileinfo.get_adjacency_for_completion(self._lat_number, self._lng_number) - #r = self._tiledb.get_adjacency_for_completion(self._lat_number, self._lng_number) - if len(r) == 0: - self._tileinfo.add_completion_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr) - mstr_msg("layergen", "Adjacency info for completion stored in database") - - #self._tiledb.commit_query() - #self._tiledb.close_db() - # Create a water mask we need to remove from the DDS later if (self._tag == "natural" and self._value == "water") or (self._tag == "water" and self._value == "lake") or (self._tag == "water" and self._value == "pond") or (self._tag == "water" and self._value == "river") or (self._tag == "leisure" and self._value == "swimming_pool"): mstr_msg("layergen", "Generating inland water mask") @@ -681,8 +493,12 @@ class mstr_layergen: if l[3] > 65: b = 255 - l[3] inl_pix[x,y] = (255,0,255,255) - inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png") + #inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png") + layer_comp = inl_mask mstr_msg("layergen", "Inland water mask generated and saved") + + # Return the completed image + return layer_comp # --------------------------------------------------------------------------------------- @@ -694,10 +510,10 @@ class mstr_layergen: if self._isline == True or self._tag == "building": # We will need the mask in question - osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) + #mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) # Generate an edge mask from the original - osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES) + osm_edge = mask.filter(ImageFilter.FIND_EDGES) osm_edge = osm_edge.filter(ImageFilter.MaxFilter) mstr_msg("layergen", "Edge mask generated") @@ -706,7 +522,7 @@ class mstr_layergen: if self._tag != "building": for i in mstr_mask_blur: if i[0] == self._tag and i[1] == self._value: - osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2])) + mask = mask.filter(ImageFilter.BoxBlur(radius=i[2])) break @@ -715,7 +531,7 @@ class mstr_layergen: # This time we have no source material - instead we will fill the # mask with a color that is appropriate for this street type. layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize)) - mask_pix = osm_mask.load() + mask_pix = mask.load() edge_pix = osm_edge.load() layer_comp_pix = layer_comp.load() @@ -812,10 +628,21 @@ class mstr_layergen: # We will do some super magic here to let houses look more realistic if self._tag == "building": + + details = Image.new("RGBA", (self._imgsize, self._imgsize)) + tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize)) + trees = Image.new("RGBA", (self._imgsize, self._imgsize)) + roof_details = Image.new("RGBA", (self._imgsize, self._imgsize)) + shadow = Image.new("RGBA", (self._imgsize, self._imgsize)) + + if mstr_shadow_enabled == True: + fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_building-" + self._value + "_layer_shadow.png" + if os.path.isfile(fn): + shadow = Image.open(fn) + vls = [ "detached", "hotel", "farm", "semidetached_house", "apartments", "civic", "house", "school", "kindergarten", "yes" ] if self._value in vls: # Generate a new image - details = Image.new("RGBA", (self._imgsize, self._imgsize)) details_pix = details.load() layer_pix = layer_comp.load() for y in range(self._imgsize): @@ -836,7 +663,6 @@ class mstr_layergen: details_pix[shf_x2, shf_y2] = (187-d2, 179-d2, 176-d2, aa) # Image for roof details - roof_details = Image.new("RGBA", (self._imgsize, self._imgsize)) roof_det_pix = roof_details.load() for y in range(self._imgsize): for x in range(self._imgsize): @@ -851,15 +677,14 @@ class mstr_layergen: a = randrange(1, 151) nc = (mstr_building_detail_colors[0][0]-d, mstr_building_detail_colors[0][1]-d, mstr_building_detail_colors[0][2]-d, a) roof_det_pix[x,y] = nc - layer_comp.alpha_composite(roof_details) # Let's see how it works with this method - details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png") + #details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png") + #layer_comp.alpha_composite(details) # Add some random trees div = int(self._imgsize/200) - trees = Image.new("RGBA", (self._imgsize, self._imgsize)) for y in range(0, self._imgsize, div): for x in range(0, self._imgsize, div): if x > 0 and x < self._imgsize and y > 0 and y < self._imgsize: @@ -885,9 +710,9 @@ class mstr_layergen: if shf_x > self._imgsize - tree.width: shf_x = self._imgsize - tree.width - 1 if shf_y > self._imgsize - tree.height: shf_y = self._imgsize - tree.height - 1 trees.alpha_composite(tree, (shf_x, shf_y)) - + + if mstr_shadow_enabled == True: - tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize)) tree_pix = trees.load() shadow_pix = tree_shadow.load() for y in range(self._imgsize): @@ -902,18 +727,34 @@ class mstr_layergen: tree_shadow.alpha_composite(trees) # Save this separately, so that we can blur buildings, but not the trees - fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png" - - if os.path.isfile(fn) == True: - extrees = Image.open(fn) - extrees.alpha_composite(tree_shadow) - extrees.save(fn) - else: - tree_shadow.save(fn) - - #layer_comp.alpha_composite(tree_shadow) - #tree_shadow.alpha_composite(layer_comp) - #layer_comp = tree_shadow + #fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png" + + #if os.path.isfile(fn) == True: + # extrees = Image.open(fn) + # extrees.alpha_composite(tree_shadow) + # extrees.save(fn) + #else: + # tree_shadow.save(fn) + + # Let's try this one on for size + bld_comp = Image.new("RGBA", (self._imgsize, self._imgsize)) + details = details.filter(ImageFilter.GaussianBlur(radius=1)) + bld_comp.alpha_composite(details) + bld_comp.alpha_composite(tree_shadow) + bld_comp.alpha_composite(trees) + shd_p = shadow.load() + for y in range(self._imgsize): + for x in range(self._imgsize): + c = shd_p[x,y] + if c[3] > 0: + s = (0,0,0,120-(randrange(0,21))) + shd_p[x,y] = s + shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1)) + bld_comp.alpha_composite(shadow) + layer_comp = layer_comp.filter(ImageFilter.GaussianBlur(radius=1.1)) + bld_comp.alpha_composite(layer_comp) + layer_comp = bld_comp + layer_comp.alpha_composite(roof_details) mstr_msg("layergen", "Layer image generated") @@ -922,7 +763,7 @@ class mstr_layergen: # Some funnies with shadows if self._tag == "building" and (self._value == "detached" or self._value == "semidetached_house" or self._value == "apartments" or self._value == "civic" or self._value == "house" or self._value == "terrace"): - mask_pix = osm_mask.load() + mask_pix = mask.load() roofshadow = Image.new("RGBA", (self._imgsize, self._imgsize)) roofpix = roofshadow.load() # Generate a pseudo shifted roof shadow @@ -937,7 +778,7 @@ class mstr_layergen: # Now apply the shift where necessary roofpix = roofshadow.load() - mask_pix = osm_mask.load() + mask_pix = mask.load() layer_comp_pix = layer_comp.load() for y in range(self._imgsize): for x in range(self._imgsize): @@ -1014,7 +855,7 @@ class mstr_layergen: # Highways and runways of any kind get some special treatment if (self._tag == "highway" and self._value == "motorway") or (self._tag == "highway" and self._value == "primary") or (self._tag == "highway" and self._value == "secondary") or (self._tag == "highway" and self._value == "tertiary") or (self._tag == "aeroway" and self._value == "runway"): # We will now add some white lines for coolness - osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES) + osm_edge = mask.filter(ImageFilter.FIND_EDGES) mask_pix = osm_edge.load() layer_comp_pix = layer_comp.load() for y in range(self._imgsize): @@ -1026,7 +867,7 @@ class mstr_layergen: layer_comp_pix[x, y] = ( w,w,w,a[3] ) if self._tag == "highway" and self._value == "residential": - osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES) + osm_edge = mask.filter(ImageFilter.FIND_EDGES) mask_pix = osm_edge.load() layer_comp_pix = layer_comp.load() for y in range(self._imgsize): @@ -1052,7 +893,7 @@ class mstr_layergen: if l[3] > 65: b = 255 - l[3] inl_pix[x,y] = (255,0,255,255) - inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png") + #inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png") mstr_msg("layergen", "Inland water mask generated and saved") @@ -1062,7 +903,7 @@ class mstr_layergen: # Store layer - layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) + #layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) mstr_msg("layergen", "Layer image finalized and saved.") @@ -1077,13 +918,17 @@ class mstr_layergen: break if nm == True: nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld) - nrm.build_normalmap() + nrm.build_normalmap(layer_comp) + + + # Return image + return layer_comp # Should we find more than one source, the first one found will take precedence. # For the others, we will need to generate fading images, so that the final layer # image works with other tiles - def generate_adjacent_fades(self): + def generate_adjacent_fades(self, mask): adj_sources = self.find_all_adjacent_sources() precedence = -1 @@ -1105,11 +950,11 @@ class mstr_layergen: # Generate required images # Basically a shortened version of the main layergen call + adj_image = Image.new("RGBA", (self._imgsize, self._imgsize)) for s in range(0, 4): if adj_sources[s] != precedence and adj_sources[s] != -1: src = adj_sources[s] - adj_image = Image.new("RGBA", (self._imgsize, self._imgsize)) adj_pix = adj_image.load() # Root folder @@ -1122,15 +967,15 @@ class mstr_layergen: for p in ptc: ptc_src.append(Image.open(p)) - osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) + #mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" ) #lyr_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" ) for i in mstr_mask_blur: if i[0] == self._tag and i[1] == self._value: if self._tag != "place" and (self._value != "sea" or self._value != "ocean"): - osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2])) + mask = mask.filter(ImageFilter.BoxBlur(radius=i[2])) break - mask_pix = osm_mask.load() + mask_pix = mask.load() # Begin producing a largely random image samples = 250 # <- We need this in a moment @@ -1167,7 +1012,7 @@ class mstr_layergen: adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3]) else: adj_pix[x,y] = (0,0,0,0) - adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png") + #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png") if s == 1: for y in range(self._imgsize): @@ -1178,7 +1023,7 @@ class mstr_layergen: adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3]) else: adj_pix[x,y] = (0,0,0,0) - adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png") + #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png") if s == 2: for y in range(self._imgsize): @@ -1189,7 +1034,7 @@ class mstr_layergen: adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3]) else: adj_pix[x,y] = (0,0,0,0) - adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png") + #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png") if s == 3: for y in range(self._imgsize): @@ -1200,7 +1045,11 @@ class mstr_layergen: adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3]) else: adj_pix[x,y] = (0,0,0,0) - adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png") + #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png") + + # Return the image + return adj_image + def find_all_adjacent_sources(self): diff --git a/maskgen.py b/maskgen.py index d60f866..519e811 100644 --- a/maskgen.py +++ b/maskgen.py @@ -205,12 +205,10 @@ class mstr_maskgen: break imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve") - # Save image - if is_prep == False: - mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png") if is_prep == True: return mask_img + # If this is a building, we need to render the shadow here, as we only know the height # of the building in this loop. if mstr_shadow_enabled == True and is_prep == False: @@ -265,3 +263,6 @@ class mstr_maskgen: # Inform mstr_msg("maskgen", "Mask built.") + + # Return the image + return mask_img diff --git a/og.py b/og.py index 57a95f4..01f7d24 100644 --- a/og.py +++ b/og.py @@ -49,7 +49,12 @@ if cli == True: # Create the class and init values og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd(), prep) - og._buildTile() + + if prep == True: + og._prepareTile() + + if prep == False: + og._generateOrthos_mt(int(sys.argv[3])) # Only if we find enough arguments, proceed. diff --git a/orthographic.py b/orthographic.py index 9cccd68..f316ac6 100644 --- a/orthographic.py +++ b/orthographic.py @@ -13,6 +13,8 @@ import math import os import glob +import threading +from multiprocessing import Process from defines import * from log import * from osmxml import * @@ -147,8 +149,140 @@ class mstr_orthographic: top_lat = cur_tile_y - # Builds and processes the tile with everything required, in one call. - def _buildTile(self): + # Start the multi-threaded build of all orthos + # amtsmt = AmountSimultaneous - so how many orthos you want to + # generate at the same time. You may need to fine tune this value + # so that you don't overload your machine. + def _generateOrthos_mt(self, amtsmt): + # Need to know maximum values first + bb_lat = self._lat + bb_lng = self._long + bb_lat_edge = self._lat+self._vstep + bb_lng_edge = self._long+mstr_zl_18 + mlat = 1 + mlng = 1 + while bb_lat < self._lat + 1: + bb_lat = bb_lat + self._vstep + mlat = mlat+1 + while bb_lng < self._long + 1: + bb_lng = bb_lng + mstr_zl_18 + mlng = mlng+1 + mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) + maxlatlng = [ mlat, mlng ] + + procs = [] + for p in range(1, amtsmt+1): + proc = Process(target=self._buildOrtho, args=[1, p, amtsmt]) + procs.append(proc) + proc.start() + mstr_msg("orthographic", "Ortho threads started") + + + # Starts a threading loop to build orthos, with the defined starting point in + # the lat-lng grid. You will also need to provide the horizontal stepping so + # that the thread keeps running. + def _buildOrtho(self, v, h, step): + + # Starting point + grid_lat = v + grid_lng = h + + # The tile is constructed of many smaller parts. We walk through the + # smallest possible, from which the bigger ones are later built. + bb_lat = self._lat + bb_lng = self._long + bb_lat_edge = self._lat+self._vstep + bb_lng_edge = self._long+mstr_zl_18 + + # We need to know the highest possible latitude and longitude tile numbers, + # in case we render at the edge + mlat = 1 + mlng = 1 + while bb_lat < self._lat + 1: + bb_lat = bb_lat + self._vstep + mlat = mlat+1 + while bb_lng < self._long + 1: + bb_lng = bb_lng + mstr_zl_18 + mlng = mlng+1 + mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) + maxlatlng = [ mlat, mlng ] + + while grid_lat <= maxlatlng[0]: + # Reset these two + bb_lat = self._lat + ((grid_lat-1)*self._vstep) + bb_lng = self._long + ((grid_lng-1)*mstr_zl_18) + bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep + bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18 + + osmxml = mstr_osmxml() + osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) + osmxml.acquire_osm(grid_lat, grid_lng) + + # Let the user know + mstr_msg("orthographic", "Generating orthophoto " + str(grid_lat) + "-" + str(grid_lng)) + + # Get the data + #osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info + #mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository") + + # Check for work to be done + layers = self.determineLayerWork(osmxml) + + # We need to walk through the array of layers, + # in their z-order. + # For each layer, we will generate the mask, the layer image + # itself, and finally, compose the ortho photo. + mstr_msg("orthographic", "Beginning generation of layers") + + # In here we store the layers + photolayers = [] + + # The masks are handed to layergen in sequence. The layers are then + # in turn handed to photogen. + + curlyr = 1 + for layer in layers: + # Let the user know + mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers))) + + # Generate the mask + mg = mstr_maskgen( [self._lat, grid_lat, self._long, grid_lng], self._vstep, layer[0], layer[1], layer[2]) + if layer[0] == "building": + mg.set_tile_width(self._findWidthOfLongitude(bb_lat)) + mg.set_latlng_numbers(self._lat, grid_lat, self._long, grid_lng) + mask = mg._build_mask(osmxml) + + # Generate the layer + lg = mstr_layergen(layer[0], layer[1], self._lat, grid_lat, self._long, grid_lng, layer[2]) + lg.set_max_latlng_tile(maxlatlng) + lg.set_latlng_folder(self._latlngfld) + #lg.open_db() + lg.open_tile_info() + photolayers.append(lg.genlayer(mask, osmxml)) + curlyr = curlyr+1 + mstr_msg("orthographic", "All layers created") + + # We should have all layers now. + # Snap a photo with our satellite :) + mstr_msg("orthographic", "Generating ortho photo") + pg = mstr_photogen(self._lat, self._long, grid_lat, grid_lng, maxlatlng[0], maxlatlng[1]) + pg.genphoto(photolayers) + mstr_msg("orthographic", " -- Ortho photo generated -- ") + print("") + print("") + + # Perform adjustment of grid position + n_lng = grid_lng + step + if n_lng > maxlatlng[1]: + np = n_lng - maxlatlng[1] + grid_lng = np + grid_lat = grid_lat+1 + else: + grid_lng = n_lng + + + # Prepares the entire tile + def _prepareTile(self): mstr_msg("orthographic", "Beginning construction of tile") # We need to know which platform we are on @@ -239,194 +373,52 @@ class mstr_orthographic: # Previously, I downloaded all XML files in one go - but to ease the # stress on OSM servers and my server, we will do acquire the data # only for the current processed part of the tile. - if self._prep == True: - for lat_grid in range(1, maxlatlng[0]+1): - for lng_grid in range(1, maxlatlng[1]+1): - # Adjust bounding box - osmxml = mstr_osmxml() - osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) - osmxml.acquire_osm(lat_grid, lng_grid) - mstr_msg("orthographic", "Adjusted bounding box for XML object") - - # Check for work to be done - layers = self.determineLayerWork(osmxml) - - curlyr = 1 - for layer in layers: - if layer[2] == False and layer[0] != "building": - # Let the user know - mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers))) - - # Generate the mask - mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2]) - mask = mg._build_mask(osmxml, is_prep=True) # We need an object here - - tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False) - tp._prepareTile() - - curlyr = curlyr+1 - - # Adjust longitude coordinates - cur_tile_x = cur_tile_x+1 - bb_lng = bb_lng + mstr_zl_18 - bb_lng_edge = bb_lng_edge + mstr_zl_18 - mstr_msg("orthographic", "Adjustment of longitude performed") - # Adjust peak longitude tile number - if cur_tile_x > top_lng: - top_lng = cur_tile_x - - # Adjust latitude and all other values when we get here - cur_tile_y = cur_tile_y+1 - cur_tile_x = 1 - bb_lng = self._long - bb_lng_edge = self._long + mstr_zl_18 - bb_lat = bb_lat + self._vstep - bb_lat_edge = bb_lat_edge + self._vstep - mstr_msg("orthographic", "Adjustment of latitude performed") - # Adjust peak latitude number - if cur_tile_y > top_lat: - top_lat = cur_tile_y - - # Need to differentiate - if self._prep == False: - # The tile is constructed of many smaller parts. We walk through the - # smallest possible, from which the bigger ones are later built. - bb_lat = self._lat - bb_lng = self._long - bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 - cur_tile_x = 1 - cur_tile_y = 1 - osmxml = mstr_osmxml(0,0) - - # Previously, I downloaded all XML files in one go - but to ease the - # stress on OSM servers and my server, we will do acquire the data - # only for the current processed part of the tile. - for lat_grid in range(1, maxlatlng[0]+1): - for lng_grid in range(1, maxlatlng[1]+1): - # Adjust bounding box - osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) - mstr_msg("orthographic", "Adjusted bounding box for XML object") + for lat_grid in range(1, maxlatlng[0]+1): + for lng_grid in range(1, maxlatlng[1]+1): + # Adjust bounding box + osmxml = mstr_osmxml() + osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) + osmxml.acquire_osm(lat_grid, lng_grid) + mstr_msg("orthographic", "Adjusted bounding box for XML object") - # Determine what to do... maybe work was interrupted - if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False: + # Check for work to be done + layers = self.determineLayerWork(osmxml) + curlyr = 1 + for layer in layers: + if layer[2] == False and layer[0] != "building": # Let the user know - mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x)) - - # Get the data - #osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info - #mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository") - - # Check for work to be done - layers = self.determineLayerWork() - - # We need to walk through the array of layers, - # in their z-order. - # For each layer, we will generate the mask, the layer image - # itself, and finally, compose the ortho photo. - mstr_msg("orthographic", "Beginning generation of layers") - - curlyr = 1 - for layer in layers: - # Let the user know - mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers))) - - # Generate the mask - mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2]) - if layer[0] == "building": - mg.set_tile_width(self._findWidthOfLongitude(bb_lat)) - mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid) - mg._build_mask() - - # Generate the layer - lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2]) - lg.set_max_latlng_tile(maxlatlng) - lg.set_latlng_folder(self._latlngfld) - #lg.open_db() - lg.open_tile_info() - lg.genlayer() - curlyr = curlyr+1 - mstr_msg("orthographic", "All layers created") - - # We should have all layers now. - # Snap a photo with our satellite :) - mstr_msg("orthographic", "Generating ortho photo") - pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x, maxlatlng[0], maxlatlng[1]) - pg.genphoto() - mstr_msg("orthographic", " -- Ortho photo generated -- ") - print("") - print("") - - # Adjust longitude coordinates - cur_tile_x = cur_tile_x+1 - bb_lng = bb_lng + mstr_zl_18 - bb_lng_edge = bb_lng_edge + mstr_zl_18 - mstr_msg("orthographic", "Adjustment of longitude performed") - # Adjust peak longitude tile number - if cur_tile_x > top_lng: - top_lng = cur_tile_x - - # Clear out cache - """ - if mstr_clear_cache == True: - ch = glob.glob(mstr_datafolder + "_cache/*") - for f in ch: - if os_platform == "nt": - if self._isFileAccessibleWin(f) == True: - os.remove(f) - if os_platform == "posix": - if self._isFileAccessiblePosix(f) == True: - os.remove(f) - mstr_msg("orthographic", "Cleared cache") - """ - + mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers))) - # Adjust latitude and all other values when we get here - cur_tile_y = cur_tile_y+1 - cur_tile_x = 1 - bb_lng = self._long - bb_lng_edge = self._long + mstr_zl_18 - bb_lat = bb_lat + self._vstep - bb_lat_edge = bb_lat_edge + self._vstep - mstr_msg("orthographic", "Adjustment of latitude performed") - # Adjust peak latitude number - if cur_tile_y > top_lat: - top_lat = cur_tile_y - - mstr_msg("orthographic", "Generation of all tiles completed!") - - - # Complete scenery - if mstr_xp_genscenery == True: - scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld) - scn.acquire_elevation_data() - scn.acquire_xes_data() - scn.build_mesh_script() - scn.build_mesh() - scn.build_ter_files() - mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed") - - mstr_msg("orthographic", "Final step completed.") - mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld) - print("") - mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus") - print("") + # Generate the mask + mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2]) + mask = mg._build_mask(osmxml, is_prep=True) # We need an object here - # Let's leave this out for the moment - """ - mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles") - tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng) - tg.genTiles() - mstr_msg("orthographic", "Final step completed.") - print("") - mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng) - print("") - print("") - mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus") - print("") - """ + tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False) + tp._prepareTile() + + curlyr = curlyr+1 + # Adjust longitude coordinates + cur_tile_x = cur_tile_x+1 + bb_lng = bb_lng + mstr_zl_18 + bb_lng_edge = bb_lng_edge + mstr_zl_18 + mstr_msg("orthographic", "Adjustment of longitude performed") + # Adjust peak longitude tile number + if cur_tile_x > top_lng: + top_lng = cur_tile_x + + # Adjust latitude and all other values when we get here + cur_tile_y = cur_tile_y+1 + cur_tile_x = 1 + bb_lng = self._long + bb_lng_edge = self._long + mstr_zl_18 + bb_lat = bb_lat + self._vstep + bb_lat_edge = bb_lat_edge + self._vstep + mstr_msg("orthographic", "Adjustment of latitude performed") + # Adjust peak latitude number + if cur_tile_y > top_lat: + top_lat = cur_tile_y diff --git a/photogen.py b/photogen.py index 033bef0..3aad51d 100644 --- a/photogen.py +++ b/photogen.py @@ -4,6 +4,7 @@ from PIL import Image, ImageFilter, ImageEnhance from defines import * from layergen import * from log import * +from functions import * # ------------------------------------------------------------------- # ORTHOGRAPHIC @@ -38,13 +39,14 @@ class mstr_photogen: # This puts it all together. Bonus: AND saves it. - def genphoto(self): + def genphoto(self, layers): # Template for the file name which is always the same - root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_" + #root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_" # First, we walk through all layers and blend them on top of each other, in order mstr_msg("photogen", "Merging layers") + """ # Note if we added building shadows bldg_shadow_added = False @@ -86,8 +88,11 @@ class mstr_photogen: #bldg_final.alpha_composite(bldg_shadow) bldg_final.alpha_composite(tree_main) bldg_final.alpha_composite(bldg_main) + """ - for l in mstr_ortho_layers: + for l in layers: + self._tile.alpha_composite(l) + """ if os.path.isfile(root_filename + l[0] + "-" + l[1] + "_layer.png"): # Need to divert in case we have shadows if mstr_shadow_enabled == True: @@ -117,9 +122,10 @@ class mstr_photogen: layer = layer.filter(ImageFilter.GaussianBlur(radius=0.2)) # Converge the layer with this image self._tile.alpha_composite(layer) + """ # Drop the buildings on top - self._tile.alpha_composite(bldg_final) + #self._tile.alpha_composite(bldg_final) # When we have run through this loop, we will end up with a sandwiched @@ -136,29 +142,68 @@ class mstr_photogen: # If this check comes back as true, we need to perform # aforementioned fix: if emptyspace == True: - # Choose a suitable layer type - tag = "landuse" - value = "meadow" mstr_msg("photogen", "Patching empty space") - self.buildCompletionMask() + mask = self.buildCompletionMask() + + # Load the mask + mask_px = mask.load() - # Generate the layer as if it were part of the OSM data - lg = mstr_layergen(tag, value, self._lat, self._ty, self._lng, self._tx, False, is_completion=True) - lg.set_max_latlng_tile(self._maxlatlng) - lg.set_latlng_folder(self._latlngfld) - #lg.open_db() - lg.open_tile_info() - lg.genlayer() + cmpl = Image.new("RGBA", (self._imgsize, self._imgsize)) + cmp_px = cmpl.load() - # Load the image - completion = Image.open(root_filename + "tile-completion_layer.png") + edn = self.find_earthnavdata_number() + edns = self.latlng_folder(edn) + idx = 0 + for r in mstr_completion_colors: + if r[0] == edns: + break + else: + idx = idx+1 + + for y in range(self._imgsize): + for x in range(self._imgsize): + p = mask_px[x,y] + if p[3] > 0: + cidx = randrange(0, len(mstr_completion_colors[idx][1])-1) + clr = mstr_completion_colors[idx][1][cidx] + cmp_px[x,y] = (clr[0], clr[1], clr[2], 255) + + # Some features + patches = glob.glob(mstr_datafolder + "textures/tile/completion/*.png") + + # Pick an amount of features to add + patch_amt = randrange(1, 7) + + # Add those somewhere + for p in range(1, patch_amt+1): + # Load some patch + ptc = Image.open(mstr_datafolder + "textures/tile/completion/p" + str(randrange(1, len(patches)+1)) + ".png") + # Rotate it + ptc = ptc.rotate(randrange(0, 360), expand=True) + # Adjust alpha on this image + ptc_p = ptc.load() + for y in range(ptc.height): + for x in range(ptc.width): + c = ptc_p[x,y] + if c[3] > 0: + na = c[3] - 160 + if na < 0: na = 0 + nc = (c[0], c[1], c[2], na) + ptc_p[x,y] = nc + + # Find a location INSIDE the image! + px = randrange(1, randrange(self._imgsize - ptc.width - 1)) + py = randrange(1, randrange(self._imgsize - ptc.height - 1)) + + # Add it to the completion image + cmpl.alpha_composite(ptc) # Merge the images - completion.alpha_composite(self._tile) + cmpl.alpha_composite(self._tile) # Make this the real one - self._tile = completion + self._tile = cmpl # There may be some tiles that have a larger sea or even an ocean in them - these need to be @@ -262,8 +307,9 @@ class mstr_photogen: # We do not apply any blur or other effects here - we only want the # exact pixel positions. - mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" ) + #mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" ) mstr_msg("photogen", "Generated and saved empty space mask") + return mask # Construct a folder name for latitude and longitude @@ -281,3 +327,13 @@ class mstr_photogen: if abs(numbers[1]) >= 100 : fstr = fstr + str(numbers[1]) return fstr + + + # Find the next "by-ten" numbers for the current latitude and longitude + def find_earthnavdata_number(self): + earthnavdata = [] + lat = abs(int(self._lat / 10) * 10) + lng = abs(int(self._lng / 10) * 10) + earthnavdata.append(lat) + earthnavdata.append(lng) + return earthnavdata \ No newline at end of file diff --git a/repoinfo b/repoinfo index d0e4842..5c7559f 100644 --- a/repoinfo +++ b/repoinfo @@ -6,6 +6,16 @@ While tools like Ortho4XP or AutoOrtho can provide next-level realism into X-Pla Currently, zoom level 18 is implemented. + +Orthographic | Real World + +[img]comp_1_1.jpg[/img] + +[img]comp_2_11.jpg[/img] + +[img]comp_2_23.jpg[/img] + + [section]Wait, just what are ortho photos?[/section] Aerial photos taken with a satellite, of varying resolution, altitude, and detail. When you switch to satellite in Google or Bing Maps - what you see is a collage of ortho photos, depending on how close or far away you are zoomed in. @@ -70,8 +80,7 @@ Apart from that I am aware that the code is most likely not the best and can be [section]Requirements[/section] - Current Python version (3.10 and up) -- Python modules: Pillow (formerly PIL), wand, numpy -- SQLite must be available to Python. On Windows it is by default, you may need to install the sqlite3 developer libraries on your distribution, then compile Python to automatically avail of this built-in module. On my Endeavour OS install, SQLite was already built-in to the Python package. +- Python modules: Pillow (formerly PIL), requests, numpy [section]Configuration[/section] @@ -82,11 +91,7 @@ You will find various configuration options, each of which is documented what it In this file you can also define how large you want the final products to be - you can choose between 4k resolution (2048px) or 16k resolution (4096). The choice you make here should be based on 1) how much detail you want, and 2) how much VRAM your GPU has. The larger a texture, the more VRAM is needed, and the more processing is required by the GPU. I personally go with 2048, as I have a RTX2060, so my VRAM is limited. When at altitudes of 10,000 feet and above, you will most definitely not notice the difference between 4096 and 2048. -NOTE! 4096 not yet implemented, but planned. - -Change the mstr_photores variable to 4096 if you want the maximum possible. - -Just a note: 4096 also uses 4x more hard drive space. 4k uses about 2MB per image, where as 16k uses about 8-10MB per image. This may not seem much for one image, but keep in mind we are talking about quite a considerable number of images. To get an idea - if you have it, look into any folder of an Ortho4XP tile in your X-Plane folder, and check the size of the "textures" folder. +NOTE! 4096 not yet implemented. Also in defines.py, you will find the single layers, along with their corresponding mask blurring values. It is my strong recommendation NOT to change these values, as I have taken a lot of time to fine-tune these values. @@ -95,33 +100,36 @@ Also in defines.py, you will find the single layers, along with their correspond Very simple. -[codebox]python og.py LATITUDE LONGITUDE[/codebox] +First step: +[codebox]python og.py LATITUDE LONGITUDE true[/codebox] So for example -[codebox]python main.py 51 7[codebox] +[codebox]python main.py 51 7 true[codebox] -This does everything for you. +This will generate a complete dataset containing which sources to use in which quadrant of the image grid. -ATTENTION! This process will take a considerable amount of time due to the work being involved. Please keep this in mind, especially if you seek to do this for an entire country. +Then: -I am considering hosting the finished tiles on my server, so that you only need to download the finished photos / tile packs. +[codebox]python og.py LATITUDE LONGITUDE NUMBER_OF_SIMULTANEOUS_IMAGES_TO_PRODUCE[/codebox] +So for example -[section]Examples[/section] +[codebox]python main.py 51 7 32[codebox] -Dortmund, Germany +This will generate 32 "orthos" at once. Which number you can use highly depends on the performance of your hardware. -[REAL WORLD] +This does everything for you. -[RESULT] +ATTENTION! This process will take a considerable amount of time due to the work being involved. Please keep this in mind, especially if you seek to do this for an entire country. +For X-Plane scenery, you will need to do this once step 2 is done: -Leverkusen, Germany +[codebox]python main.py 51 7 xpscenery[codebox] -[REAL WORLD] +This will generate the terrain mesh and associate the textures with the mesh. -[RESULT] +I am considering hosting the finished tiles on my server, so that you only need to download the finished photos / tile packs. [section]Improving the code[/section] diff --git a/xp_normalmap.py b/xp_normalmap.py index 059aa8d..8a0551b 100644 --- a/xp_normalmap.py +++ b/xp_normalmap.py @@ -86,6 +86,10 @@ class mstr_xp_normalmap: mstr_msg("xp_normalmap", "[X-Plane] Beginning normal map generation") # No specularity, no reflectivity - but standard color # Blue (reflectivity) and alpha (specularity) need to be 1 - but can be adjusted as needed + + # Resize original + image = image.resize((int(mstr_photores/4), int(mstr_photores/4)), Image.Resampling.BILINEAR) + nmp = Image.new("RGBA", (image.width, image.height), (128,128,1,1)) org = image.load() nmp_pix = nmp.load() @@ -137,13 +141,13 @@ class mstr_xp_normalmap: # The funnction to call. Blends with the existing map, or creates a new one - def build_normalmap(self): + def build_normalmap(self, layer): mstr_msg("xp_normalmap", "[X-Plane] Building normal map") # The layer image - lyr = self.load_layer() + #lyr = self.load_layer() # Make the normal map for the layer - nrm = self.generate_normal_map_for_layer(lyr) + nrm = self.generate_normal_map_for_layer(layer) # Normal map final file name nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._tv) + "_" + str(self._th) + ".png" -- 2.30.2