Milestone commit - switch to mostly in-memory management, and functioning multi-processing for ortho generation

defines.py

@@ -148,7 +148,6 @@ mstr_ortho_layers = [
     ("highway", "tertiary", 20),
     ("highway", "unclassified", 17),
     ("highway", "living_street", 12),
-    ("waterway", "river", 10),
     ("waterway", "stream", 2),
     ("amenity", "parking", "amenity", "parking"),
     ("amenity", "school", "amenity", "school"),
@@ -162,12 +161,8 @@ mstr_ortho_layers = [
     ("natural", "sand", "natural", "sand"),
     ("natural", "desert", "natural", "desert"),
     # Z-Order 3
-    ("natural", "water", "natural", "water"),
     ("natural", "bay", "natural", "beach"),
     ("natural", "beach", "natural", "beach"),
-    ("water", "lake", "natural", "water"),
-    ("water", "pond", "natural", "water"),
-    ("water", "river", "natural", "water"),
     ("leisure", "swimming_pool", "natural", "water"),
     ("highway", "pedestrian", 4),
     # Z-Order 4
@@ -195,6 +190,11 @@ mstr_ortho_layers = [
     ("building", "public", "building", "public"),
     ("building", "commercial", "building", "commercial"),
     ("building", "yes", "building", "common"),
+    ("water", "lake", "natural", "water"),
+    ("water", "pond", "natural", "water"),
+    ("water", "river", "natural", "water"),
+    ("natural", "water", "natural", "water"),
+    ("waterway", "river", 10),
     ("place", "sea", "natural", "sea"),
     ("place", "ocean", "natural", "sea")
 ]
@@ -229,10 +229,10 @@ mstr_mask_blur = [
     ("landuse", "farmland", 10),
     ("landuse", "farmyard", 10),
     # Z-Order 2
-    ("landuse", "forest", 8),
+    ("landuse", "forest", 10),
-    ("leisure", "nature_reserve", 8),
+    ("leisure", "nature_reserve", 10),
-    ("natural", "wood", 8),
+    ("natural", "wood", 10),
-    ("natural", "tree_row", 8),
+    ("natural", "tree_row", 10),
     ("landuse", "military", 15),
     # Z-Order 3
     ("natural", "bare_rock", 25),

layergen.py

@@ -114,27 +114,63 @@ class mstr_layergen:
         return layer_final
 
 
+    # Find the source to use pre-determined in phase one
+    def findLayerSource(self):
+        # The source number
+        src = -1
+
+        # The already existing source data
+        srcfile = mstr_datafolder + "z_orthographic/data/" + self._latlngfld + "/" + str(self._lat_number) + "_" + str(self._lng_number)
+
+        # Let's open the file and find our entry
+        with open(srcfile) as file:
+            for line in file:
+                linedata = line.split(" ")
+                if linedata[2] == self._tag and linedata[3] == self._value:
+                    src = int(linedata[4])
+                    break
+        
+        # Should we encounter a -1 at this point, we can choose something
+        # It means it touches no border as it was not found in the file
+        if src == -1:
+            root_folder = mstr_datafolder + "textures/"
+            for s in mstr_ortho_layers:
+                if s[0] == self._tag and s[1] == self._value:
+                    fld_main = len(s)-2
+                    fld_sub  = len(s)-1
+                    root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
+            
+            brd = glob.glob(root_folder + "/brd/b*.png")
+            src = randrange(1, len(brd)+1)
+
+        return src
+
+
+
     # This generates the layer from the defined mask
-    def genlayer(self):
+    def genlayer(self, mask, xml):
 
         mstr_msg("layergen", "Layer to be generated: " + str(self._latitude) + "-" + str(self._lat_number) + ":" + str(self._longitude) + "-" + str(self._lng_number) + " -- tag: " + self._tag + " - value: " + self._value )
 
         # Before we generate the layer, let's check for airports in this chunk
         mstr_msg("layergen", "Checking for airport/s with ICAO code")
-        osmxml = mstr_osmxml(0,0)
+        
-        icao = osmxml.find_icao_codes(mstr_datafolder + "_cache/tile.xml")
+        icao = None
+        if xml != None:
+            icao = xml.find_icao_codes()
             mstr_msg("layergen", "Found " + str(len(icao)) + " airport/s")
         # Runway surface, if any other than concrete/asphalt
         rw_surface = ""
         # If we find an airport, make a note ...
-        if len(icao) >= 1:
+        if icao != None:
+            if len(icao) >= 1 and self._is_completion == False:
                 #for i in icao:
                     # ... but only, if this airport is not already noted
                     #iccheck = self._tiledb.perform_query("SELECT * FROM airports WHERE icao='" + i +"';")
                     #if len(iccheck) == 0:
                         #self._tiledb.insert_icao(i, self._lat_number, self._lng_number, self._latitude, self._longitude)
                     #    mstr_msg("layergen", "Airport/s noted in data file")
-            rw_surface = osmxml.find_runway_surface(mstr_datafolder + "_cache/tile.xml")
+                rw_surface = xml.find_runway_surface()
 
         # The image for the layer itself
         layer = Image.new("RGBA", (self._imgsize, self._imgsize))
@@ -153,176 +189,7 @@ class mstr_layergen:
                     root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
 
             # Determine which sources to use.
-            # First, we need to check for adjacent tile information. We then either
+            src = self.findLayerSource()
-            # need to use the source of any adjacent tile, or we can choose freely.
-            src = -1
-            
-            # Find our adjacent tiles
-            adjtiles = findAdjacentTilesTo(self._lat_number, self._lng_number)
-
-            mstr_msg("layergen", "Performing adjacency check")
-            # Walk through each tile and see what we can find in relation to this
-            # tile in the center
-            # Since we already know the order in adjtiles, we can do this real easy
-            if self._is_completion == False:
-                at = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top
-                ar = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right
-                ab = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom
-                al = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left
-                #at = self._tiledb.get_adjacency_for_source(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top
-                #ar = self._tiledb.get_adjacency_for_source(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right
-                #ab = self._tiledb.get_adjacency_for_source(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom
-                #al = self._tiledb.get_adjacency_for_source(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left
-            if self._is_completion == True:
-                at = self._tileinfo.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top
-                ar = self._tileinfo.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right
-                ab = self._tileinfo.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom
-                al = self._tileinfo.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left
-                #at = self._tiledb.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top
-                #ar = self._tiledb.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right
-                #ab = self._tiledb.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom
-                #al = self._tiledb.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left
-
-                if len(at) == 4:
-                    self._tag   = at[0]
-                    self._value = at[1]
-                if len(ar) == 4:
-                    self._tag   = ar[0]
-                    self._value = ar[1]
-                if len(ab) == 4:
-                    self._tag   = ab[0]
-                    self._value = ab[1]
-                if len(al) == 4:
-                    self._tag   = al[0]
-                    self._value = al[1]
-
-                root_folder = mstr_datafolder + "textures/"
-                for s in mstr_ortho_layers:
-                    if s[0] == self._tag and s[1] == self._value:
-                        fld_main = len(s)-2
-                        fld_sub  = len(s)-1
-                        root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
-
-            # We are south to the top tile.
-            if len(at) == 2 and src == -1:
-                if "b" in at[1]: src = at[0]
-            # We are west to the right tile.
-            if len(ar) == 2 and src == -1:
-                if "l" in ar[1]: src = ar[0]
-            # We are north to the bottom tile.
-            if len(ab) == 2 and src == -1:
-                if "t" in ab[1]: src = ab[0]
-            # We are east to the left tile.
-            if len(al) == 2 and src == -1:
-                if "r" in al[1]: src = al[0]
-
-            # Should we be at the border of the degree for latitude and longitude, we need to perform
-            # additional checks
-            is_deg_brd_t = False
-            is_deg_brd_r = False
-            is_deg_brd_b = False
-            is_deg_brd_l = False
-            if self._lat_number == 1: is_deg_brd_b = True
-            if self._lat_number == self._maxlat: is_deg_brd_t = True
-            if self._lng_number == 1: is_deg_brd_l = True
-            if self._lng_number == self._maxlng: is_deg_brd_r = True
-
-            # Adjacent latitude and longitude tiles
-            deg_tiles = []
-            deg_tiles.append( ( self._latitude+1, self._longitude ) ) # Top
-            deg_tiles.append( ( self._latitude, self._longitude+1 ) ) # Right
-            deg_tiles.append( ( self._latitude-1, self._longitude ) ) # Bottom
-            deg_tiles.append( ( self._latitude, self._longitude-1 ) ) # Left
-
-            # Perform degree border checks
-            # - and make sure we do not run into errors - this drove me crazy in testing
-            atd = []
-            ard = []
-            abd = []
-            ald = []
-            if is_deg_brd_t == True:
-                if self._is_completion == False:
-                    atd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], self._tag, self._value, 1, self._lng_number, "b") # Top
-                    #atd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, self._tag, self._value) # Top
-                if self._is_completion == True:
-                    atd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number, "b")
-                    #atd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[0][0], deg_tiles[0][1], 1, self._lng_number) # Top
-            
-            if is_deg_brd_r == True:
-                if self._is_completion == False:
-                    ard = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._tag, self._value, self._lat_number, 1, "l") # Right
-                    #ard = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, self._tag, self._value) # Right
-                if self._is_completion == True:
-                    ard = self._tileinfo.get_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1, "l")
-                    #ard = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[1][0], deg_tiles[1][1], self._lat_number, 1) # Right
-            
-            if is_deg_brd_b == True:
-                #maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude-1, self._longitude)
-                if self._is_completion == False:
-                    abd = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "t")
-                    #abd = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number, self._tag, self._value) # Bottom
-                if self._is_completion == True:
-                    abd = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "t")
-                    #abd = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], maxlatlng[0], self._lng_number) # Bottom
-            
-            if is_deg_brd_l == True:
-                #maxlatlng = self._tiledb.get_highest_latlong_from_tile(self._latitude, self._longitude-1)
-                if self._is_completion == False:
-                    ald = self._tileinfo.get_adjacency_for_tag_and_value_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._tag, self._value, self._lat_number, self._lng_number, "r")
-                    #ald = self._tiledb.get_adjacency_for_source_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1], self._tag, self._value) # Left
-                if self._is_completion == True:
-                    ald = self._tileinfo.get_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, self._lng_number, "r")
-                    #ald = self._tiledb.get_adjacency_for_completion_in_lat_lng(deg_tiles[2][0], deg_tiles[2][1], self._lat_number, maxlatlng[1]) # Left
-
-            if (is_deg_brd_t == True or is_deg_brd_r == True or is_deg_brd_b == True or is_deg_brd_l == True):
-                if src == -1 and self._is_completion == True:
-                    if len(atd) == 4:
-                        self._tag   = atd[0]
-                        self._value = atd[1]
-                    if len(ard) == 4:
-                        self._tag   = ard[0]
-                        self._value = ard[1]
-                    if len(abd) == 4:
-                        self._tag   = abd[0]
-                        self._value = abd[1]
-                    if len(ald) == 4:
-                        self._tag   = ald[0]
-                        self._value = ald[1]
-
-                    root_folder = mstr_datafolder + "textures/"
-                    for s in mstr_ortho_layers:
-                        if s[0] == self._tag and s[1] == self._value:
-                            fld_main = len(s)-2
-                            fld_sub  = len(s)-1
-                            root_folder = root_folder + s[fld_main] + "/" + s[fld_sub]
-
-
-            # Should we get here and one of the degree border checks turns out true,
-            # we need to make sure that we select the source we found. This should
-            # enable seamless tiling... around the entire planet
-            if is_deg_brd_t == True and len(at) == 0 and src == -1:
-                if len(atd) == 2:
-                    if "b" in atd[3]: src = int(atd[2])
-            if is_deg_brd_r == True and len(ar) == 0 and src == -1:
-                if len(ard) == 2:
-                    if "l" in ard[3]: src = int(ard[2])
-            if is_deg_brd_b == True and len(ab) == 0 and src == -1:
-                if len(abd) == 2:
-                    if "t" in abd[3]: src = int(abd[2])
-            if is_deg_brd_l == True and len(al) == 0 and src == -1:
-                if len(ald) == 2:
-                    if "r" in ald[3]: src = int(ald[2])
-            
-            mstr_msg("layergen", "Adjacency check completed")
-
-            brd = glob.glob(root_folder + "/brd/b*.png")
-
-            # If the adjacency check returned nothing (src is still -1),
-            # then pick something
-            if src == -1:
-                if len(brd) == 1: src=1
-                if len(brd) >= 2:
-                    src = randrange(1, len(brd)+1)
             
             ptc = glob.glob(root_folder + "/ptc/b" + str(src) + "_p*.png")
             
@@ -333,25 +200,27 @@ class mstr_layergen:
                 ptc_src.append(Image.open(p))
             mstr_msg("layergen", "Layer sources selected")
 
+            """
             # OK! Load the mask
             if self._is_completion == False:
-                osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
+                mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
             if self._is_completion == True:
-                osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" )
+                mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion.png" )
+            """
             
             # Generate an edge mask from the original
-            osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
+            osm_edge = mask.filter(ImageFilter.FIND_EDGES)
             osm_edge = osm_edge.filter(ImageFilter.MaxFilter)
             mstr_msg("layergen", "Edge mask generated")
 
             # This adds some natural looking shapes to these types of features
             if self._value == "forest" or self._value == "nature_reserve":
                 epx = osm_edge.load()
-                imgd = ImageDraw.Draw(osm_mask)
+                imgd = ImageDraw.Draw(mask)
 
-                # Walk through a grid of 200x200 - on the edge image
+                # Walk through a grid of 100x100 - on the edge image
-                for y in range(0, osm_mask.height, int(osm_mask.height/200)):
+                for y in range(0, mask.height, int(mask.height/100)):
-                    for x in range(0, osm_mask.width, int(osm_mask.width/200)):
+                    for x in range(0, mask.width, int(mask.width/100)):
                         px = epx[x,y]
                         if px[3] == 255:
                             rx = randrange(24,60)
@@ -363,7 +232,7 @@ class mstr_layergen:
                             psy = randrange(y-11, y+11)
                             
                             # Do some magic - but not on edges
-                            if x > 0 and x < osm_mask.width and y > 0 and y < osm_mask.height:
+                            if x > 0 and x < mask.width and y > 0 and y < mask.height:
                                 if f != 5:
                                     imgd.ellipse((psx-int(rx/2), psy-int(ry/2), psx+rx, psy+ry), fill="black")
                                 if f == 3 or f == 7:
@@ -372,14 +241,14 @@ class mstr_layergen:
 
             # We need to change the image in certain conditions
             if self._value == "hedge" and self._tag == "barrier":
-                osm_mask = osm_edge
+                mask = osm_edge
 
             # From here on in we will need to perform some adjustments on the masks, depending
             # on what they are.
             for i in mstr_mask_blur:
                 if i[0] == self._tag and i[1] == self._value:
                     if self._tag != "place" and (self._value != "sea" or self._value != "ocean"): 
-                        osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
+                        mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
                         break
             
             # Begin producing a largely random image
@@ -414,7 +283,7 @@ class mstr_layergen:
                         ly = randrange( self._imgsize - img.height )
                         layer.alpha_composite( img, (lx, ly) )
                 if self._is_completion == True:
-                    mp = osm_mask.load()
+                    mp = mask.load()
                     edn = self.xplane_latlng_folder(self.find_earthnavdata_number())
                     idx = 0
                     for r in mstr_completion_colors:
@@ -483,7 +352,7 @@ class mstr_layergen:
             # Generate the layer from the mask.
             layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
             layer_pix = layer.load()
-            mask_pix = osm_mask.load()
+            mask_pix = mask.load()
             layer_comp_pix = layer_comp.load()
             for y in range(self._imgsize):
                 for x in range(self._imgsize):
@@ -505,9 +374,10 @@ class mstr_layergen:
             # Here we want to make sure that the generated image fits well with others, so
             # let's do that.
             mstr_msg("layergen", "Generating adjacent fades")
-            self.generate_adjacent_fades()
+            adjfade = self.generate_adjacent_fades(mask)
 
             # Determine if there are any fades, and if so, fade those in first before we save the layer
+            """
             fade_fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value
             if os.path.isfile(fade_fn + "_fade_top.png") == True:
                 fade = Image.open(fade_fn + "_fade_top.png")
@@ -521,6 +391,9 @@ class mstr_layergen:
             if os.path.isfile(fade_fn + "_fade_left.png") == True:
                 fade = Image.open(fade_fn + "_fade_left.png")
                 layer_comp.alpha_composite(fade)
+            """
+
+            layer_comp.alpha_composite(adjfade)
             mstr_msg("layergen", "Adjacent fading completed")
 
 
@@ -551,10 +424,10 @@ class mstr_layergen:
                                 layer_comp_pix[x,y] = (lpix[0], lpix[1], lpix[2], 255-rpix[3])
 
             # Store layer
-            if self._is_completion == False:
+            #if self._is_completion == False:
-                layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
+            #    layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
-            if self._is_completion == True:
+            #if self._is_completion == True:
-                layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" )
+            #   layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_tile-completion_layer.png" )
             #layer_final.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
             mstr_msg("layergen", "Layer image finalized and saved.")
 
@@ -570,7 +443,7 @@ class mstr_layergen:
                             break
                     if nm == True:
                         nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
-                        nrm.build_normalmap()
+                        nrm.build_normalmap(layer_comp)
 
 
             # Let's try our hand at pseudo shadows
@@ -581,7 +454,7 @@ class mstr_layergen:
                         if self._tag == sh[0] and self._value == sh[1]:
                             mstr_msg("layergen", "Generating shadow for layer")
                             shadow_pix = shadow.load()
-                            mask_pix = osm_mask.load()
+                            mask_pix = mask.load()
                             shf = 1
                             while shf < mstr_shadow_shift:
                                 for y in range(self._imgsize):
@@ -602,73 +475,12 @@ class mstr_layergen:
 
                             shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1.5))
 
-                            shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png")
+                            #shadow.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_shadow.png")
+                            shadow.alpha_composite(layer_comp)
+                            layer_comp = shadow
                             mstr_msg("layergen", "Shadow layer completed")
 
 
-
-            # Check if pixels touch the borders of the image, and if so -
-            # make a not of that in the database.
-            at=False
-            ar=False
-            ab=False
-            al=False
-            layer_pix = layer_comp.load() # <- Just to be safe
-
-            # Top scan
-            for i in range(0, self._imgsize-1):
-                p = layer_pix[i,0]
-                if p[3] > 0:
-                    at=True
-                    break
-           
-            # Right scan
-            for i in range(0, self._imgsize-1):
-                p = layer_pix[self._imgsize-1,i]
-                if p[3] > 0:
-                    ar=True
-                    break
-            
-            # Bottom scan
-            for i in range(0, self._imgsize-1):
-                p = layer_pix[i,self._imgsize-1]
-                if p[3] > 0:
-                    ab=True
-                    break
-            
-            # Left scan
-            for i in range(0, self._imgsize-1):
-                p = layer_pix[1,i]
-                if p[3] > 0:
-                    al=True
-                    break
-            
-            # Construct DB String
-            adjstr = ""
-            if at==True: adjstr = adjstr + "t"
-            if ar==True: adjstr = adjstr + "r"
-            if ab==True: adjstr = adjstr + "b"
-            if al==True: adjstr = adjstr + "l"
-
-            # Store into DB - but only if there is something to store
-            if adjstr != "":
-                if self._is_completion == False:
-                    r = self._tileinfo.get_adjacency_for_tag_and_value(self._lat_number, self._lng_number, self._tag, self._value)
-                    #r = self._tiledb.get_adjacency_for_source(self._lat_number, self._lng_number, self._tag, self._value)
-                    if len(r) == 0:
-                        self._tileinfo.add_adjacency_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr)
-                        mstr_msg("layergen", "Adjacency info stored in database")
-
-                if self._is_completion == True:
-                    r = self._tileinfo.get_adjacency_for_completion(self._lat_number, self._lng_number)
-                    #r = self._tiledb.get_adjacency_for_completion(self._lat_number, self._lng_number)
-                    if len(r) == 0:
-                        self._tileinfo.add_completion_data(self._lat_number, self._lng_number, self._tag, self._value, src, adjstr)
-                        mstr_msg("layergen", "Adjacency info for completion stored in database")
-
-                #self._tiledb.commit_query()
-                #self._tiledb.close_db()
-
             # Create a water mask we need to remove from the DDS later
             if (self._tag == "natural" and self._value == "water") or (self._tag == "water" and self._value == "lake") or (self._tag == "water" and self._value == "pond") or (self._tag == "water" and self._value == "river") or (self._tag == "leisure" and self._value == "swimming_pool"):
                 mstr_msg("layergen", "Generating inland water mask")
@@ -681,9 +493,13 @@ class mstr_layergen:
                         if l[3] > 65:
                             b = 255 - l[3]
                             inl_pix[x,y] = (255,0,255,255)
-                inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
+                #inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
+                layer_comp = inl_mask
                 mstr_msg("layergen", "Inland water mask generated and saved")
 
+            # Return the completed image
+            return layer_comp
+        
 
         # ---------------------------------------------------------------------------------------
         # ---------------------------------------------------------------------------------------
@@ -694,10 +510,10 @@ class mstr_layergen:
         if self._isline == True or self._tag == "building":
 
             # We will need the mask in question
-            osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
+            #mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
 
             # Generate an edge mask from the original
-            osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
+            osm_edge = mask.filter(ImageFilter.FIND_EDGES)
             osm_edge = osm_edge.filter(ImageFilter.MaxFilter)
             mstr_msg("layergen", "Edge mask generated")
 
@@ -706,7 +522,7 @@ class mstr_layergen:
             if self._tag != "building":
                 for i in mstr_mask_blur:
                     if i[0] == self._tag and i[1] == self._value:
-                        osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
+                        mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
                         break
             
             
@@ -715,7 +531,7 @@ class mstr_layergen:
             # This time we have no source material - instead we will fill the
             # mask with a color that is appropriate for this street type.
             layer_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
-            mask_pix = osm_mask.load()
+            mask_pix = mask.load()
             edge_pix = osm_edge.load()
             layer_comp_pix = layer_comp.load()
 
@@ -812,10 +628,21 @@ class mstr_layergen:
 
             # We will do some super magic here to let houses look more realistic
             if self._tag == "building":
+                
+                details = Image.new("RGBA", (self._imgsize, self._imgsize))
+                tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
+                trees = Image.new("RGBA", (self._imgsize, self._imgsize))
+                roof_details = Image.new("RGBA", (self._imgsize, self._imgsize))
+                shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
+
+                if mstr_shadow_enabled == True:
+                    fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_building-" + self._value + "_layer_shadow.png"
+                    if os.path.isfile(fn):
+                        shadow = Image.open(fn)
+
                 vls = [ "detached", "hotel", "farm", "semidetached_house", "apartments", "civic", "house", "school", "kindergarten", "yes" ]
                 if self._value in vls:
                     # Generate a new image
-                    details = Image.new("RGBA", (self._imgsize, self._imgsize))
                     details_pix = details.load()
                     layer_pix = layer_comp.load()
                     for y in range(self._imgsize):
@@ -836,7 +663,6 @@ class mstr_layergen:
                                     details_pix[shf_x2, shf_y2] = (187-d2, 179-d2, 176-d2, aa)
 
                     # Image for roof details
-                    roof_details = Image.new("RGBA", (self._imgsize, self._imgsize))
                     roof_det_pix = roof_details.load()
                     for y in range(self._imgsize):
                         for x in range(self._imgsize):
@@ -851,15 +677,14 @@ class mstr_layergen:
                                     a = randrange(1, 151)
                                     nc = (mstr_building_detail_colors[0][0]-d, mstr_building_detail_colors[0][1]-d, mstr_building_detail_colors[0][2]-d, a)
                                     roof_det_pix[x,y] = nc
-                    layer_comp.alpha_composite(roof_details)
 
 
                     # Let's see how it works with this method
-                    details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png")
+                    #details.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_details.png")
+                    #layer_comp.alpha_composite(details)
 
                     # Add some random trees
                     div = int(self._imgsize/200)
-                    trees = Image.new("RGBA", (self._imgsize, self._imgsize))
                     for y in range(0, self._imgsize, div):
                         for x in range(0, self._imgsize, div):
                             if x > 0 and x < self._imgsize and y > 0 and y < self._imgsize:
@@ -886,8 +711,8 @@ class mstr_layergen:
                                                 if shf_y > self._imgsize - tree.height: shf_y = self._imgsize - tree.height - 1
                                                 trees.alpha_composite(tree, (shf_x, shf_y))
                     
+                    
                     if mstr_shadow_enabled == True:
-                        tree_shadow = Image.new("RGBA", (self._imgsize, self._imgsize))
                         tree_pix = trees.load()
                         shadow_pix = tree_shadow.load()
                         for y in range(self._imgsize):
@@ -902,18 +727,34 @@ class mstr_layergen:
                         tree_shadow.alpha_composite(trees)
 
                         # Save this separately, so that we can blur buildings, but not the trees
-                        fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png"
+                        #fn = mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_building_trees.png"
 
-                        if os.path.isfile(fn) == True:
+                        #if os.path.isfile(fn) == True:
-                            extrees = Image.open(fn)
+                        #    extrees = Image.open(fn)
-                            extrees.alpha_composite(tree_shadow)
+                        #    extrees.alpha_composite(tree_shadow)
-                            extrees.save(fn)
+                        #    extrees.save(fn)
-                        else:
+                        #else:
-                            tree_shadow.save(fn)
+                        #    tree_shadow.save(fn)
 
-                        #layer_comp.alpha_composite(tree_shadow)
+                # Let's try this one on for size
-                        #tree_shadow.alpha_composite(layer_comp)
+                bld_comp = Image.new("RGBA", (self._imgsize, self._imgsize))
-                        #layer_comp = tree_shadow
+                details = details.filter(ImageFilter.GaussianBlur(radius=1))
+                bld_comp.alpha_composite(details)
+                bld_comp.alpha_composite(tree_shadow)
+                bld_comp.alpha_composite(trees)
+                shd_p  = shadow.load()
+                for y in range(self._imgsize):
+                    for x in range(self._imgsize):
+                        c = shd_p[x,y]
+                        if c[3] > 0:
+                            s = (0,0,0,120-(randrange(0,21)))
+                            shd_p[x,y] = s
+                shadow = shadow.filter(ImageFilter.GaussianBlur(radius=1))
+                bld_comp.alpha_composite(shadow)
+                layer_comp = layer_comp.filter(ImageFilter.GaussianBlur(radius=1.1))
+                bld_comp.alpha_composite(layer_comp)
+                layer_comp = bld_comp
+                layer_comp.alpha_composite(roof_details)
 
             mstr_msg("layergen", "Layer image generated")
 
@@ -922,7 +763,7 @@ class mstr_layergen:
                     
                 # Some funnies with shadows
                 if self._tag == "building" and (self._value == "detached" or self._value == "semidetached_house" or self._value == "apartments" or self._value == "civic" or self._value == "house" or self._value == "terrace"):
-                    mask_pix = osm_mask.load()
+                    mask_pix = mask.load()
                     roofshadow = Image.new("RGBA", (self._imgsize, self._imgsize))
                     roofpix = roofshadow.load()
                     # Generate a pseudo shifted roof shadow
@@ -937,7 +778,7 @@ class mstr_layergen:
 
                     # Now apply the shift where necessary
                     roofpix = roofshadow.load()
-                    mask_pix = osm_mask.load()
+                    mask_pix = mask.load()
                     layer_comp_pix = layer_comp.load()
                     for y in range(self._imgsize):
                         for x in range(self._imgsize):
@@ -1014,7 +855,7 @@ class mstr_layergen:
             # Highways and runways of any kind get some special treatment
             if (self._tag == "highway" and self._value == "motorway") or (self._tag == "highway" and self._value == "primary") or (self._tag == "highway" and self._value == "secondary") or (self._tag == "highway" and self._value == "tertiary") or (self._tag == "aeroway" and self._value == "runway"):
                 # We will now add some white lines for coolness
-                osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
+                osm_edge = mask.filter(ImageFilter.FIND_EDGES)
                 mask_pix = osm_edge.load()
                 layer_comp_pix = layer_comp.load()
                 for y in range(self._imgsize):
@@ -1026,7 +867,7 @@ class mstr_layergen:
                             layer_comp_pix[x, y] = ( w,w,w,a[3] )
 
             if self._tag == "highway" and self._value == "residential":
-                osm_edge = osm_mask.filter(ImageFilter.FIND_EDGES)
+                osm_edge = mask.filter(ImageFilter.FIND_EDGES)
                 mask_pix = osm_edge.load()
                 layer_comp_pix = layer_comp.load()
                 for y in range(self._imgsize):
@@ -1052,7 +893,7 @@ class mstr_layergen:
                         if l[3] > 65:
                             b = 255 - l[3]
                             inl_pix[x,y] = (255,0,255,255)
-                inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
+                #inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
                 mstr_msg("layergen", "Inland water mask generated and saved")
 
             
@@ -1062,7 +903,7 @@ class mstr_layergen:
 
 
             # Store layer
-            layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
+            #layer_comp.save( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
             mstr_msg("layergen", "Layer image finalized and saved.")
 
 
@@ -1077,13 +918,17 @@ class mstr_layergen:
                             break
                     if nm == True:
                         nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
-                        nrm.build_normalmap()
+                        nrm.build_normalmap(layer_comp)
+
+
+            # Return image
+            return layer_comp
 
 
     # Should we find more than one source, the first one found will take precedence.
     # For the others, we will need to generate fading images, so that the final layer 
     # image works with other tiles
-    def generate_adjacent_fades(self):
+    def generate_adjacent_fades(self, mask):
         adj_sources = self.find_all_adjacent_sources()
         precedence = -1
 
@@ -1105,11 +950,11 @@ class mstr_layergen:
 
         # Generate required images
         # Basically a shortened version of the main layergen call
+        adj_image = Image.new("RGBA", (self._imgsize, self._imgsize))
         for s in range(0, 4):
             if adj_sources[s] != precedence and adj_sources[s] != -1:
                 src = adj_sources[s]
 
-                adj_image = Image.new("RGBA", (self._imgsize, self._imgsize))
                 adj_pix = adj_image.load()
 
                 # Root folder
@@ -1122,15 +967,15 @@ class mstr_layergen:
                 for p in ptc:
                     ptc_src.append(Image.open(p))
 
-                osm_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
+                #mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + ".png" )
                 #lyr_mask = Image.open( mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer.png" )
 
                 for i in mstr_mask_blur:
                     if i[0] == self._tag and i[1] == self._value:
                         if self._tag != "place" and (self._value != "sea" or self._value != "ocean"): 
-                            osm_mask = osm_mask.filter(ImageFilter.BoxBlur(radius=i[2]))
+                            mask = mask.filter(ImageFilter.BoxBlur(radius=i[2]))
                             break
-                mask_pix = osm_mask.load()
+                mask_pix = mask.load()
             
                 # Begin producing a largely random image
                 samples = 250   # <- We need this in a moment
@@ -1167,7 +1012,7 @@ class mstr_layergen:
                                 adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
                             else:
                                 adj_pix[x,y] = (0,0,0,0)
-                    adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png")
+                    #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_top.png")
                 
                 if s == 1:
                     for y in range(self._imgsize):
@@ -1178,7 +1023,7 @@ class mstr_layergen:
                                 adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
                             else:
                                 adj_pix[x,y] = (0,0,0,0)
-                    adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png")
+                    #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_right.png")
                 
                 if s == 2:
                     for y in range(self._imgsize):
@@ -1189,7 +1034,7 @@ class mstr_layergen:
                                 adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
                             else:
                                 adj_pix[x,y] = (0,0,0,0)
-                    adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png")
+                    #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_bottom.png")
 
                 if s == 3:
                     for y in range(self._imgsize):
@@ -1200,7 +1045,11 @@ class mstr_layergen:
                                 adj_pix[x,y] = (c[0], c[1], c[2], fade_a[3])
                             else:
                                 adj_pix[x,y] = (0,0,0,0)
-                    adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png")
+                    #adj_image.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_fade_left.png")
+
+        # Return the image
+        return adj_image
+
 
 
     def find_all_adjacent_sources(self):

maskgen.py

@@ -205,12 +205,10 @@ class mstr_maskgen:
                             break
                     imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve")
         
-        # Save image
-        if is_prep == False:
-            mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png")
         if is_prep == True:
             return mask_img
 
+
         # If this is a building, we need to render the shadow here, as we only know the height
         # of the building in this loop.
         if mstr_shadow_enabled == True and is_prep == False:
@@ -265,3 +263,6 @@ class mstr_maskgen:
 
         # Inform
         mstr_msg("maskgen", "Mask built.")
+
+        # Return the image
+        return mask_img

og.py

@@ -49,7 +49,12 @@ if cli == True:
 
     # Create the class and init values
     og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd(), prep)
-    og._buildTile()
+
+    if prep == True:
+        og._prepareTile()
+
+    if prep == False:
+        og._generateOrthos_mt(int(sys.argv[3]))
 
 
 # Only if we find enough arguments, proceed.

orthographic.py

@@ -13,6 +13,8 @@
 import math
 import os
 import glob
+import threading
+from multiprocessing import Process
 from defines import *
 from log import *
 from osmxml import *
@@ -147,8 +149,140 @@ class mstr_orthographic:
                 top_lat = cur_tile_y
 
 
-    # Builds and processes the tile with everything required, in one call.
+    # Start the multi-threaded build of all orthos
-    def _buildTile(self):
+    # amtsmt = AmountSimultaneous - so how many orthos you want to
+    # generate at the same time. You may need to fine tune this value
+    # so that you don't overload your machine.
+    def _generateOrthos_mt(self, amtsmt):
+        # Need to know maximum values first
+        bb_lat = self._lat
+        bb_lng = self._long
+        bb_lat_edge = self._lat+self._vstep
+        bb_lng_edge = self._long+mstr_zl_18
+        mlat = 1
+        mlng = 1
+        while bb_lat < self._lat + 1:
+            bb_lat = bb_lat + self._vstep
+            mlat = mlat+1
+        while bb_lng < self._long + 1:
+            bb_lng = bb_lng + mstr_zl_18
+            mlng = mlng+1
+        mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng))
+        maxlatlng = [ mlat, mlng ]
+
+        procs = []
+        for p in range(1, amtsmt+1):
+            proc = Process(target=self._buildOrtho, args=[1, p, amtsmt])
+            procs.append(proc)
+            proc.start()
+        mstr_msg("orthographic", "Ortho threads started")
+
+
+    # Starts a threading loop to build orthos, with the defined starting point in
+    # the lat-lng grid. You will also need to provide the horizontal stepping so
+    # that the thread keeps running.
+    def _buildOrtho(self, v, h, step):
+
+        # Starting point
+        grid_lat = v
+        grid_lng = h
+
+        # The tile is constructed of many smaller parts. We walk through the
+        # smallest possible, from which the bigger ones are later built.
+        bb_lat = self._lat
+        bb_lng = self._long
+        bb_lat_edge = self._lat+self._vstep
+        bb_lng_edge = self._long+mstr_zl_18
+
+        # We need to know the highest possible latitude and longitude tile numbers,
+        # in case we render at the edge
+        mlat = 1
+        mlng = 1
+        while bb_lat < self._lat + 1:
+            bb_lat = bb_lat + self._vstep
+            mlat = mlat+1
+        while bb_lng < self._long + 1:
+            bb_lng = bb_lng + mstr_zl_18
+            mlng = mlng+1
+        mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng))
+        maxlatlng = [ mlat, mlng ]
+
+        while grid_lat <= maxlatlng[0]:
+            # Reset these two
+            bb_lat = self._lat + ((grid_lat-1)*self._vstep)
+            bb_lng = self._long + ((grid_lng-1)*mstr_zl_18)
+            bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep
+            bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18
+
+            osmxml = mstr_osmxml()
+            osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
+            osmxml.acquire_osm(grid_lat, grid_lng)
+
+            # Let the user know
+            mstr_msg("orthographic", "Generating orthophoto " + str(grid_lat) + "-" + str(grid_lng))
+
+            # Get the data
+            #osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
+            #mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
+
+            # Check for work to be done
+            layers = self.determineLayerWork(osmxml)
+
+            # We need to walk through the array of layers,
+            # in their z-order.
+            # For each layer, we will generate the mask, the layer image
+            # itself, and finally, compose the ortho photo.
+            mstr_msg("orthographic", "Beginning generation of layers")
+
+            # In here we store the layers
+            photolayers = []
+
+            # The masks are handed to layergen in sequence. The layers are then
+            # in turn handed to photogen.
+
+            curlyr = 1
+            for layer in layers:
+                # Let the user know
+                mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
+
+                # Generate the mask
+                mg = mstr_maskgen( [self._lat, grid_lat, self._long, grid_lng], self._vstep, layer[0], layer[1], layer[2])
+                if layer[0] == "building":
+                    mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
+                    mg.set_latlng_numbers(self._lat, grid_lat, self._long, grid_lng)
+                mask = mg._build_mask(osmxml)
+                
+                # Generate the layer
+                lg = mstr_layergen(layer[0], layer[1], self._lat, grid_lat, self._long, grid_lng, layer[2])
+                lg.set_max_latlng_tile(maxlatlng)
+                lg.set_latlng_folder(self._latlngfld)
+                #lg.open_db()
+                lg.open_tile_info()
+                photolayers.append(lg.genlayer(mask, osmxml))
+                curlyr = curlyr+1
+            mstr_msg("orthographic", "All layers created")
+
+            # We should have all layers now.
+            # Snap a photo with our satellite :)
+            mstr_msg("orthographic", "Generating ortho photo")
+            pg = mstr_photogen(self._lat, self._long, grid_lat, grid_lng,  maxlatlng[0], maxlatlng[1])
+            pg.genphoto(photolayers)
+            mstr_msg("orthographic", " -- Ortho photo generated -- ")
+            print("")
+            print("")
+
+            # Perform adjustment of grid position
+            n_lng = grid_lng + step
+            if n_lng > maxlatlng[1]:
+                np = n_lng - maxlatlng[1]
+                grid_lng = np
+                grid_lat = grid_lat+1
+            else:
+                grid_lng = n_lng
+
+
+    # Prepares the entire tile
+    def _prepareTile(self):
         mstr_msg("orthographic", "Beginning construction of tile")
 
         # We need to know which platform we are on
@@ -239,7 +373,6 @@ class mstr_orthographic:
         # Previously, I downloaded all XML files in one go - but to ease the
         # stress on OSM servers and my server, we will do acquire the data
         # only for the current processed part of the tile.
-        if self._prep == True:
         for lat_grid in range(1, maxlatlng[0]+1):
             for lng_grid in range(1, maxlatlng[1]+1):
                 # Adjust bounding box
@@ -287,147 +420,6 @@ class mstr_orthographic:
             if cur_tile_y > top_lat:
                 top_lat = cur_tile_y
 
-        # Need to differentiate
-        if self._prep == False:
-            # The tile is constructed of many smaller parts. We walk through the
-            # smallest possible, from which the bigger ones are later built.
-            bb_lat = self._lat
-            bb_lng = self._long
-            bb_lat_edge = self._lat+self._vstep
-            bb_lng_edge = self._long+mstr_zl_18
-            cur_tile_x = 1
-            cur_tile_y = 1
-            osmxml = mstr_osmxml(0,0)
-
-            # Previously, I downloaded all XML files in one go - but to ease the
-            # stress on OSM servers and my server, we will do acquire the data
-            # only for the current processed part of the tile.
-            for lat_grid in range(1, maxlatlng[0]+1):
-                for lng_grid in range(1, maxlatlng[1]+1):
-                    # Adjust bounding box
-                    osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
-                    mstr_msg("orthographic", "Adjusted bounding box for XML object")
-
-                    # Determine what to do... maybe work was interrupted
-                    if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False:
-
-                        # Let the user know
-                        mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
-
-                        # Get the data
-                        #osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
-                        #mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
-
-                        # Check for work to be done
-                        layers = self.determineLayerWork()
-
-                        # We need to walk through the array of layers,
-                        # in their z-order.
-                        # For each layer, we will generate the mask, the layer image
-                        # itself, and finally, compose the ortho photo.
-                        mstr_msg("orthographic", "Beginning generation of layers")
-
-                        curlyr = 1
-                        for layer in layers:
-                            # Let the user know
-                            mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
-
-                            # Generate the mask
-                            mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
-                            if layer[0] == "building":
-                                mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
-                                mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
-                            mg._build_mask()
-                            
-                            # Generate the layer
-                            lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
-                            lg.set_max_latlng_tile(maxlatlng)
-                            lg.set_latlng_folder(self._latlngfld)
-                            #lg.open_db()
-                            lg.open_tile_info()
-                            lg.genlayer()
-                            curlyr = curlyr+1
-                        mstr_msg("orthographic", "All layers created")
-
-                        # We should have all layers now.
-                        # Snap a photo with our satellite :)
-                        mstr_msg("orthographic", "Generating ortho photo")
-                        pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x,  maxlatlng[0], maxlatlng[1])
-                        pg.genphoto()
-                        mstr_msg("orthographic", " -- Ortho photo generated -- ")
-                        print("")
-                        print("")
-
-                    # Adjust longitude coordinates
-                    cur_tile_x = cur_tile_x+1
-                    bb_lng = bb_lng + mstr_zl_18
-                    bb_lng_edge = bb_lng_edge + mstr_zl_18
-                    mstr_msg("orthographic", "Adjustment of longitude performed")
-                    # Adjust peak longitude tile number
-                    if cur_tile_x > top_lng:
-                        top_lng = cur_tile_x
-
-                    # Clear out cache
-                    """
-                    if mstr_clear_cache == True:
-                        ch = glob.glob(mstr_datafolder + "_cache/*")
-                        for f in ch:
-                            if os_platform == "nt":
-                                if self._isFileAccessibleWin(f) == True:
-                                    os.remove(f)
-                            if os_platform == "posix":
-                                if self._isFileAccessiblePosix(f) == True:
-                                    os.remove(f)
-                        mstr_msg("orthographic", "Cleared cache")
-                    """
-                
-
-                # Adjust latitude and all other values when we get here
-                cur_tile_y = cur_tile_y+1
-                cur_tile_x = 1
-                bb_lng = self._long
-                bb_lng_edge = self._long + mstr_zl_18
-                bb_lat = bb_lat + self._vstep
-                bb_lat_edge = bb_lat_edge + self._vstep
-                mstr_msg("orthographic", "Adjustment of latitude performed")
-                # Adjust peak latitude number
-                if cur_tile_y > top_lat:
-                    top_lat = cur_tile_y
-            
-            mstr_msg("orthographic", "Generation of all tiles completed!")
-
-
-            # Complete scenery
-            if mstr_xp_genscenery == True:
-                scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld)
-                scn.acquire_elevation_data()
-                scn.acquire_xes_data()
-                scn.build_mesh_script()
-                scn.build_mesh()
-                scn.build_ter_files()
-                mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed")
-            
-            mstr_msg("orthographic", "Final step completed.")
-            mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld)
-            print("")
-            mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
-            print("")
-
-            # Let's leave this out for the moment
-            """
-            mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles")
-            tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng)
-            tg.genTiles()
-            mstr_msg("orthographic", "Final step completed.")
-            print("")
-            mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng)
-            print("")
-            print("")
-            mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
-            print("")
-            """
-
-
 
 
 

photogen.py

@@ -4,6 +4,7 @@ from PIL import Image, ImageFilter, ImageEnhance
 from defines import *
 from layergen import *
 from log import *
+from functions import *
 
 # -------------------------------------------------------------------
 # ORTHOGRAPHIC
@@ -38,13 +39,14 @@ class mstr_photogen:
 
     
     # This puts it all together. Bonus: AND saves it.
-    def genphoto(self):
+    def genphoto(self, layers):
         # Template for the file name which is always the same
-        root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_"
+        #root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_"
 
         # First, we walk through all layers and blend them on top of each other, in order
         mstr_msg("photogen", "Merging layers")
 
+        """
         # Note if we added building shadows
         bldg_shadow_added = False
 
@@ -86,8 +88,11 @@ class mstr_photogen:
         #bldg_final.alpha_composite(bldg_shadow)
         bldg_final.alpha_composite(tree_main)
         bldg_final.alpha_composite(bldg_main)
+        """
 
-        for l in mstr_ortho_layers:
+        for l in layers:
+            self._tile.alpha_composite(l)
+            """
             if os.path.isfile(root_filename + l[0] + "-" + l[1] + "_layer.png"):
                 # Need to divert in case we have shadows
                 if mstr_shadow_enabled == True:
@@ -117,9 +122,10 @@ class mstr_photogen:
                         layer = layer.filter(ImageFilter.GaussianBlur(radius=0.2))
                     # Converge the layer with this image
                     self._tile.alpha_composite(layer)
+            """
 
         # Drop the buildings on top
-        self._tile.alpha_composite(bldg_final)
+        #self._tile.alpha_composite(bldg_final)
 
 
         # When we have run through this loop, we will end up with a sandwiched
@@ -136,29 +142,68 @@ class mstr_photogen:
         # If this check comes back as true, we need to perform
         # aforementioned fix:
         if emptyspace == True:
-            # Choose a suitable layer type
-            tag   = "landuse"
-            value = "meadow"
 
             mstr_msg("photogen", "Patching empty space")
-            self.buildCompletionMask()
+            mask = self.buildCompletionMask()
 
-            # Generate the layer as if it were part of the OSM data
+            # Load the mask
-            lg = mstr_layergen(tag, value, self._lat, self._ty, self._lng, self._tx, False, is_completion=True)
+            mask_px = mask.load()
-            lg.set_max_latlng_tile(self._maxlatlng)
-            lg.set_latlng_folder(self._latlngfld)
-            #lg.open_db()
-            lg.open_tile_info()
-            lg.genlayer()
 
-            # Load the image
+            cmpl = Image.new("RGBA", (self._imgsize, self._imgsize))
-            completion = Image.open(root_filename + "tile-completion_layer.png")
+            cmp_px = cmpl.load()
+
+            edn = self.find_earthnavdata_number()
+            edns = self.latlng_folder(edn)
+            idx = 0
+            for r in mstr_completion_colors:
+                if r[0] == edns:
+                    break
+                else:
+                    idx = idx+1
+            
+            for y in range(self._imgsize):
+                for x in range(self._imgsize):
+                    p = mask_px[x,y]
+                    if p[3] > 0:
+                        cidx = randrange(0, len(mstr_completion_colors[idx][1])-1)
+                        clr = mstr_completion_colors[idx][1][cidx]
+                        cmp_px[x,y] = (clr[0], clr[1], clr[2], 255)
+
+            # Some features
+            patches = glob.glob(mstr_datafolder + "textures/tile/completion/*.png")
+
+            # Pick an amount of features to add
+            patch_amt = randrange(1, 7)
+
+            # Add those somewhere
+            for p in range(1, patch_amt+1):
+                # Load some patch
+                ptc = Image.open(mstr_datafolder + "textures/tile/completion/p" + str(randrange(1, len(patches)+1)) + ".png")
+                # Rotate it
+                ptc = ptc.rotate(randrange(0, 360), expand=True)
+                # Adjust alpha on this image
+                ptc_p = ptc.load()
+                for y in range(ptc.height):
+                    for x in range(ptc.width):
+                        c = ptc_p[x,y]
+                        if c[3] > 0:
+                            na = c[3] - 160
+                            if na < 0: na = 0
+                            nc = (c[0], c[1], c[2], na)
+                            ptc_p[x,y] = nc
+
+                # Find a location INSIDE the image!
+                px = randrange(1, randrange(self._imgsize - ptc.width - 1))
+                py = randrange(1, randrange(self._imgsize - ptc.height - 1))
+
+                # Add it to the completion image
+                cmpl.alpha_composite(ptc)
 
             # Merge the images
-            completion.alpha_composite(self._tile)
+            cmpl.alpha_composite(self._tile)
 
             # Make this the real one
-            self._tile = completion
+            self._tile = cmpl
 
 
         # There may be some tiles that have a larger sea or even an ocean in them - these need to be
@@ -262,8 +307,9 @@ class mstr_photogen:
         # We do not apply any blur or other effects here - we only want the
         # exact pixel positions.
 
-        mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" )
+        #mask.save( mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_tile-completion.png" )
         mstr_msg("photogen", "Generated and saved empty space mask")
+        return mask
 
 
     # Construct a folder name for latitude and longitude
@@ -281,3 +327,13 @@ class mstr_photogen:
         if abs(numbers[1]) >= 100 : fstr = fstr + str(numbers[1])
 
         return fstr
+
+
+    # Find the next "by-ten" numbers for the current latitude and longitude
+    def find_earthnavdata_number(self):
+        earthnavdata = []
+        lat = abs(int(self._lat / 10) * 10)
+        lng = abs(int(self._lng / 10) * 10)
+        earthnavdata.append(lat)
+        earthnavdata.append(lng)
+        return earthnavdata

repoinfo

@@ -6,6 +6,16 @@ While tools like Ortho4XP or AutoOrtho can provide next-level realism into X-Pla
 
 Currently, zoom level 18 is implemented.
 
+
+Orthographic | Real World
+
+[img]comp_1_1.jpg[/img]
+
+[img]comp_2_11.jpg[/img]
+
+[img]comp_2_23.jpg[/img]
+
+
 [section]Wait, just what are ortho photos?[/section]
 
 Aerial photos taken with a satellite, of varying resolution, altitude, and detail. When you switch to satellite in Google or Bing Maps - what you see is a collage of ortho photos, depending on how close or far away you are zoomed in.
@@ -70,8 +80,7 @@ Apart from that I am aware that the code is most likely not the best and can be
 [section]Requirements[/section]
 
 - Current Python version (3.10 and up)
-- Python modules: Pillow (formerly PIL), wand, numpy
+- Python modules: Pillow (formerly PIL), requests, numpy
-- SQLite must be available to Python. On Windows it is by default, you may need to install the sqlite3 developer libraries on your distribution, then compile Python to automatically avail of this built-in module. On my Endeavour OS install, SQLite was already built-in to the Python package.
 
 
 [section]Configuration[/section]
@@ -82,11 +91,7 @@ You will find various configuration options, each of which is documented what it
 
 In this file you can also define how large you want the final products to be - you can choose between 4k resolution (2048px) or 16k resolution (4096). The choice you make here should be based on 1) how much detail you want, and 2) how much VRAM your GPU has. The larger a texture, the more VRAM is needed, and the more processing is required by the GPU. I personally go with 2048, as I have a RTX2060, so my VRAM is limited. When at altitudes of 10,000 feet and above, you will most definitely not notice the difference between 4096 and 2048.
 
-NOTE! 4096 not yet implemented, but planned.
+NOTE! 4096 not yet implemented.
-
-Change the mstr_photores variable to 4096 if you want the maximum possible.
-
-Just a note: 4096 also uses 4x more hard drive space. 4k uses about 2MB per image, where as 16k uses about 8-10MB per image. This may not seem much for one image, but keep in mind we are talking about quite a considerable number of images. To get an idea - if you have it, look into any folder of an Ortho4XP tile in your X-Plane folder, and check the size of the "textures" folder.
 
 Also in defines.py, you will find the single layers, along with their corresponding mask blurring values. It is my strong recommendation NOT to change these values, as I have taken a lot of time to fine-tune these values.
 
@@ -95,35 +100,38 @@ Also in defines.py, you will find the single layers, along with their correspond
 
 Very simple.
 
-[codebox]python og.py LATITUDE LONGITUDE[/codebox]
+First step: 
+[codebox]python og.py LATITUDE LONGITUDE true[/codebox]
 
 So for example
 
-[codebox]python main.py 51 7[codebox]
+[codebox]python main.py 51 7 true[codebox]
+
+This will generate a complete dataset containing which sources to use in which quadrant of the image grid.
+
+Then:
+
+[codebox]python og.py LATITUDE LONGITUDE NUMBER_OF_SIMULTANEOUS_IMAGES_TO_PRODUCE[/codebox]
+
+So for example
+
+[codebox]python main.py 51 7 32[codebox]
+
+This will generate 32 "orthos" at once. Which number you can use highly depends on the performance of your hardware.
 
 This does everything for you.
 
 ATTENTION! This process will take a considerable amount of time due to the work being involved. Please keep this in mind, especially if you seek to do this for an entire country.
 
+For X-Plane scenery, you will need to do this once step 2 is done:
+
+[codebox]python main.py 51 7 xpscenery[codebox]
+
+This will generate the terrain mesh and associate the textures with the mesh.
+
 I am considering hosting the finished tiles on my server, so that you only need to download the finished photos / tile packs.
 
 
-[section]Examples[/section]
-
-Dortmund, Germany
-
-[REAL WORLD]
-
-[RESULT]
-
-
-Leverkusen, Germany
-
-[REAL WORLD]
-
-[RESULT]
-
-
 [section]Improving the code[/section]
 
 As I have hosted this on my private but publicly accessible git repository, it should be clear that I am making the code available to everyone. Of course, you are free to improve (I am sure the code needs some optimisation). If you changed or improved something, and you publish it (no matter where), you MUST adhere to the license this software ships with, which is OSL 3.0.

xp_normalmap.py

@@ -86,6 +86,10 @@ class mstr_xp_normalmap:
         mstr_msg("xp_normalmap", "[X-Plane] Beginning normal map generation")
         # No specularity, no reflectivity - but standard color
         # Blue (reflectivity) and alpha (specularity) need to be 1 - but can be adjusted as needed
+
+        # Resize original
+        image = image.resize((int(mstr_photores/4), int(mstr_photores/4)), Image.Resampling.BILINEAR)
+
         nmp = Image.new("RGBA", (image.width, image.height), (128,128,1,1))
         org = image.load()
         nmp_pix = nmp.load()
@@ -137,13 +141,13 @@ class mstr_xp_normalmap:
     
 
     # The funnction to call. Blends with the existing map, or creates a new one
-    def build_normalmap(self):
+    def build_normalmap(self, layer):
         mstr_msg("xp_normalmap", "[X-Plane] Building normal map")
         # The layer image
-        lyr = self.load_layer()
+        #lyr = self.load_layer()
 
         # Make the normal map for the layer
-        nrm = self.generate_normal_map_for_layer(lyr)
+        nrm = self.generate_normal_map_for_layer(layer)
 
         # Normal map final file name
         nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._tv) + "_" + str(self._th) + ".png"