From: Marcus Str. <marcus@marstr.online>
Date: Tue, 12 Nov 2024 19:25:46 +0000 (+0100)
Subject: Initial step toward all-in-memory management, osmxml is now completely in-RAM driven... 
X-Git-Url: https://marstr.online/code/gitweb.cgi?a=commitdiff_plain;h=fc889477681684c123ede09f0d1a27fce58f0bad;p=orthographic

Initial step toward all-in-memory management, osmxml is now completely in-RAM driven. Remaining parts to follow. Initial changes to all other elements toward preparation-then-render approach.
---

diff --git a/functions.py b/functions.py
index b424fe2..c1d89b4 100644
--- a/functions.py
+++ b/functions.py
@@ -91,3 +91,20 @@ def in_circle(center_x, center_y, radius, x, y):
 def meters_per_pixel(lngwidth):
     mpx = lngwidth / mstr_photores
     return mpx
+
+
+# Construct an X-Plane compatible folder name for latitude and longitude
+def xplane_latlng_folder(numbers):
+    fstr = ""
+    if numbers[0] >= 0: fstr = "+"
+    if numbers[0] <  0: fstr = "-"
+    if abs(numbers[0]) < 10: fstr = fstr + "0" + str(numbers[0])
+    if abs(numbers[0]) >= 10 and numbers[0] <= 90: fstr = fstr + str(numbers[0])
+
+    if numbers[1] >= 0: fstr = fstr + "+"
+    if numbers[1] <  0: fstr = fstr + "-"
+    if abs(numbers[1]) < 10: fstr = fstr + "00" + str(numbers[1])
+    if abs(numbers[1]) >= 10 and numbers[0] <= 99: fstr = fstr + "0" + str(numbers[1])
+    if abs(numbers[1]) >= 100 : fstr = fstr + str(numbers[1])
+
+    return fstr
\ No newline at end of file
diff --git a/maskgen.py b/maskgen.py
index 9515e33..d60f866 100644
--- a/maskgen.py
+++ b/maskgen.py
@@ -75,17 +75,18 @@ class mstr_maskgen:
 
 
     # Builds the required mask
-    def _build_mask(self):
+    def _build_mask(self, xml, is_prep=False):
         # Generate empty image
         imgsize = 2048
         mask_img = Image.new("RGBA", (imgsize, imgsize))
 
-        tilexml = mstr_datafolder + "_cache/tile.xml"
-        xml = mstr_osmxml(0,0)
+        #tilexml = mstr_datafolder + "_cache/tile_" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + ".xml"
+        #tilexml = mstr_datafolder + "_cache/tile.xml"
+        #xml = mstr_osmxml(0,0)
         fstr = str(self._box[0]) + "-" + str(self._box[1]) + "_" + str(self._box[2]) + "-" + str(self._box[3])
-        nds = xml.acquire_nodes(tilexml)
-        way = xml.acquire_waypoint_data(tilexml)
-        rls = xml.acquire_relations(tilexml)
+        nds = xml.acquire_nodes()
+        way = xml.acquire_waypoint_data()
+        rls = xml.acquire_relations()
 
         mstr_msg("maskgen", "Building mask for " + str(self._box[0]) + "-" + str(self._box[1]) + ", " + str(self._box[2]) + "-" + str(self._box[3]) + ", for " + self._tag + ": " + self._value )
 
@@ -205,11 +206,14 @@ class mstr_maskgen:
                     imgd.line(pts, fill="#000000", width=mstr_ortho_layers[idx][2], joint="curve")
         
         # Save image
-        mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png")
+        if is_prep == False:
+            mask_img.save(mstr_datafolder + "_cache/" + fstr + "_" + self._tag + "-" + self._value + ".png")
+        if is_prep == True:
+            return mask_img
 
         # If this is a building, we need to render the shadow here, as we only know the height
         # of the building in this loop.
-        if mstr_shadow_enabled == True:
+        if mstr_shadow_enabled == True and is_prep == False:
             if self._tag == "building":
                 mpp = meters_per_pixel(self._tile_width) * mstr_zl_18
                 pix_per_floor = mstr_shadow_floor_h / mpp
diff --git a/og.py b/og.py
index f8f128e..57a95f4 100644
--- a/og.py
+++ b/og.py
@@ -31,12 +31,14 @@ print(" ")
 
 cli = False
 pbf = False
+prep = False
 
-if len(sys.argv) == 3:
+if len(sys.argv) == 4:
     cli = True
+    if sys.argv[3] == "true": prep = True
 
-if len(sys.argv) == 4:
-    pbf = True
+#if len(sys.argv) == 4:
+#    pbf = True
 
 # Only if we find enough arguments, proceed.
 if cli == True:
@@ -46,7 +48,7 @@ if cli == True:
     mstr_msg("_main", "Beginning tile generation process.")
 
     # Create the class and init values
-    og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd())
+    og = mstr_orthographic(lat, lng, mstr_datafolder, os.getcwd(), prep)
     og._buildTile()
 
 
diff --git a/orthographic.py b/orthographic.py
index 16eb887..9cccd68 100644
--- a/orthographic.py
+++ b/orthographic.py
@@ -19,6 +19,7 @@ from osmxml import *
 from maskgen import *
 from layergen import *
 from photogen import *
+from tileprep import *
 from xp_scenery import *
 
 
@@ -26,13 +27,14 @@ from xp_scenery import *
 class mstr_orthographic:
 
     # Constructor of class. Takes longitude and latitude.
-    def __init__(self, lat, lng, outfolder, pwd):
+    def __init__(self, lat, lng, outfolder, pwd, prep=False):
         self._lat  = lat
         self._long = lng
         self._output = outfolder
         self._pwd = pwd
         self._vstep = self._findVerticalStepping()
         self._latlngfld = self.latlng_folder([lat,lng])
+        self._prep = prep
         mstr_msg("orthographic", "Initiated with LAT: " + str(lat) + ", LNG: " + str(lng))
 
 
@@ -197,7 +199,7 @@ class mstr_orthographic:
                     mstr_msg("orthographic", "[X-Plane] created tile normal maps folder")
                 if not os.path.exists(self._output + "/z_orthographic/normals/" + self._latlngfld):
                     os.makedirs(self._output + "/z_orthographic/normals/" + self._latlngfld)
-        
+
         # The tile is constructed of many smaller parts. We walk through the
         # smallest possible, from which the bigger ones are later built.
         bb_lat = self._lat
@@ -206,7 +208,7 @@ class mstr_orthographic:
         bb_lng_edge = self._long+mstr_zl_18
         cur_tile_x = 1
         cur_tile_y = 1
-        osmxml = mstr_osmxml(0,0)
+        #osmxml = mstr_osmxml(0,0)
         mstr_msg("orthographic", "Set initial coordinates and bounding box for OSM acquisition")
 
         # The highest encountered tile numbers
@@ -231,147 +233,215 @@ class mstr_orthographic:
         bb_lat = self._lat
         bb_lng = self._long
 
+
+        # We will now prepare the graphic tile generation. We do this by only generating
+        # the masks and determine which sources to use in the actual images.
         # Previously, I downloaded all XML files in one go - but to ease the
         # stress on OSM servers and my server, we will do acquire the data
         # only for the current processed part of the tile.
-        for lat_grid in range(1, maxlatlng[0]+1):
-            for lng_grid in range(1, maxlatlng[1]+1):
-                # Adjust bounding box
-                osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
-                mstr_msg("orthographic", "Adjusted bounding box for XML object")
-
-                # Determine what to do... maybe work was interrupted
-                if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False:
-
-                    # Let the user know
-                    mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
-
-                    # Get the data
-                    osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
-                    mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
+        if self._prep == True:
+            for lat_grid in range(1, maxlatlng[0]+1):
+                for lng_grid in range(1, maxlatlng[1]+1):
+                    # Adjust bounding box
+                    osmxml = mstr_osmxml()
+                    osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
+                    osmxml.acquire_osm(lat_grid, lng_grid)
+                    mstr_msg("orthographic", "Adjusted bounding box for XML object")
 
                     # Check for work to be done
-                    layers = self.determineLayerWork()
-
-                    # We need to walk through the array of layers,
-                    # in their z-order.
-                    # For each layer, we will generate the mask, the layer image
-                    # itself, and finally, compose the ortho photo.
-                    mstr_msg("orthographic", "Beginning generation of layers")
+                    layers = self.determineLayerWork(osmxml)
 
                     curlyr = 1
                     for layer in layers:
-                        # Let the user know
-                        mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
-
-                        # Generate the mask
-                        mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
-                        if layer[0] == "building":
-                            mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
-                            mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
-                        mg._build_mask()
-                        
-                        # Generate the layer
-                        lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
-                        lg.set_max_latlng_tile(maxlatlng)
-                        lg.set_latlng_folder(self._latlngfld)
-                        #lg.open_db()
-                        lg.open_tile_info()
-                        lg.genlayer()
-                        curlyr = curlyr+1
-                    mstr_msg("orthographic", "All layers created")
+                        if layer[2] == False and layer[0] != "building":
+                            # Let the user know
+                            mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
 
-                    # We should have all layers now.
-                    # Snap a photo with our satellite :)
-                    mstr_msg("orthographic", "Generating ortho photo")
-                    pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x,  maxlatlng[0], maxlatlng[1])
-                    pg.genphoto()
-                    mstr_msg("orthographic", " -- Ortho photo generated -- ")
-                    print("")
-                    print("")
+                            # Generate the mask
+                            mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
+                            mask = mg._build_mask(osmxml, is_prep=True) # We need an object here
 
-                # Adjust longitude coordinates
-                cur_tile_x = cur_tile_x+1
-                bb_lng = bb_lng + mstr_zl_18
-                bb_lng_edge = bb_lng_edge + mstr_zl_18
-                mstr_msg("orthographic", "Adjustment of longitude performed")
-                # Adjust peak longitude tile number
-                if cur_tile_x > top_lng:
-                    top_lng = cur_tile_x
+                            tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False)
+                            tp._prepareTile()
 
-                # Clear out cache
-                if mstr_clear_cache == True:
-                    ch = glob.glob(mstr_datafolder + "_cache/*")
-                    for f in ch:
-                        if os_platform == "nt":
-                            if self._isFileAccessibleWin(f) == True:
-                                os.remove(f)
-                        if os_platform == "posix":
-                            if self._isFileAccessiblePosix(f) == True:
-                                os.remove(f)
-                    mstr_msg("orthographic", "Cleared cache")
-            
+                        curlyr = curlyr+1
 
-            # Adjust latitude and all other values when we get here
-            cur_tile_y = cur_tile_y+1
-            cur_tile_x = 1
+                    # Adjust longitude coordinates
+                    cur_tile_x = cur_tile_x+1
+                    bb_lng = bb_lng + mstr_zl_18
+                    bb_lng_edge = bb_lng_edge + mstr_zl_18
+                    mstr_msg("orthographic", "Adjustment of longitude performed")
+                    # Adjust peak longitude tile number
+                    if cur_tile_x > top_lng:
+                        top_lng = cur_tile_x
+
+                # Adjust latitude and all other values when we get here
+                cur_tile_y = cur_tile_y+1
+                cur_tile_x = 1
+                bb_lng = self._long
+                bb_lng_edge = self._long + mstr_zl_18
+                bb_lat = bb_lat + self._vstep
+                bb_lat_edge = bb_lat_edge + self._vstep
+                mstr_msg("orthographic", "Adjustment of latitude performed")
+                # Adjust peak latitude number
+                if cur_tile_y > top_lat:
+                    top_lat = cur_tile_y
+
+        # Need to differentiate
+        if self._prep == False:
+            # The tile is constructed of many smaller parts. We walk through the
+            # smallest possible, from which the bigger ones are later built.
+            bb_lat = self._lat
             bb_lng = self._long
-            bb_lng_edge = self._long + mstr_zl_18
-            bb_lat = bb_lat + self._vstep
-            bb_lat_edge = bb_lat_edge + self._vstep
-            mstr_msg("orthographic", "Adjustment of latitude performed")
-            # Adjust peak latitude number
-            if cur_tile_y > top_lat:
-                top_lat = cur_tile_y
-        
-        mstr_msg("orthographic", "Generation of all tiles completed!")
+            bb_lat_edge = self._lat+self._vstep
+            bb_lng_edge = self._long+mstr_zl_18
+            cur_tile_x = 1
+            cur_tile_y = 1
+            osmxml = mstr_osmxml(0,0)
 
+            # Previously, I downloaded all XML files in one go - but to ease the
+            # stress on OSM servers and my server, we will do acquire the data
+            # only for the current processed part of the tile.
+            for lat_grid in range(1, maxlatlng[0]+1):
+                for lng_grid in range(1, maxlatlng[1]+1):
+                    # Adjust bounding box
+                    osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
+                    mstr_msg("orthographic", "Adjusted bounding box for XML object")
 
-        # Complete scenery
-        if mstr_xp_genscenery == True:
-            scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld)
-            scn.acquire_elevation_data()
-            scn.acquire_xes_data()
-            scn.build_mesh_script()
-            scn.build_mesh()
-            scn.build_ter_files()
-            mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed")
-        
-        mstr_msg("orthographic", "Final step completed.")
-        mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld)
-        print("")
-        mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
-        print("")
+                    # Determine what to do... maybe work was interrupted
+                    if os.path.isfile(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(cur_tile_y) + "_" + str(cur_tile_x) + ".dds") == False:
+
+                        # Let the user know
+                        mstr_msg("orthographic", "Generating missing orthophoto " + str(cur_tile_y) + "-" + str(cur_tile_x))
+
+                        # Get the data
+                        #osmxml.acquire_osm(cur_tile_y, cur_tile_x) # <- This acquires current OSM info
+                        #mstr_msg("orthographic", "Acquired current OSM info from marstr.online repository")
+
+                        # Check for work to be done
+                        layers = self.determineLayerWork()
+
+                        # We need to walk through the array of layers,
+                        # in their z-order.
+                        # For each layer, we will generate the mask, the layer image
+                        # itself, and finally, compose the ortho photo.
+                        mstr_msg("orthographic", "Beginning generation of layers")
+
+                        curlyr = 1
+                        for layer in layers:
+                            # Let the user know
+                            mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
+
+                            # Generate the mask
+                            mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2])
+                            if layer[0] == "building":
+                                mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
+                                mg.set_latlng_numbers(self._lat, lat_grid, self._long, lng_grid)
+                            mg._build_mask()
+                            
+                            # Generate the layer
+                            lg = mstr_layergen(layer[0], layer[1], self._lat, cur_tile_y, self._long, cur_tile_x, layer[2])
+                            lg.set_max_latlng_tile(maxlatlng)
+                            lg.set_latlng_folder(self._latlngfld)
+                            #lg.open_db()
+                            lg.open_tile_info()
+                            lg.genlayer()
+                            curlyr = curlyr+1
+                        mstr_msg("orthographic", "All layers created")
+
+                        # We should have all layers now.
+                        # Snap a photo with our satellite :)
+                        mstr_msg("orthographic", "Generating ortho photo")
+                        pg = mstr_photogen(self._lat, self._long, cur_tile_y, cur_tile_x,  maxlatlng[0], maxlatlng[1])
+                        pg.genphoto()
+                        mstr_msg("orthographic", " -- Ortho photo generated -- ")
+                        print("")
+                        print("")
+
+                    # Adjust longitude coordinates
+                    cur_tile_x = cur_tile_x+1
+                    bb_lng = bb_lng + mstr_zl_18
+                    bb_lng_edge = bb_lng_edge + mstr_zl_18
+                    mstr_msg("orthographic", "Adjustment of longitude performed")
+                    # Adjust peak longitude tile number
+                    if cur_tile_x > top_lng:
+                        top_lng = cur_tile_x
+
+                    # Clear out cache
+                    """
+                    if mstr_clear_cache == True:
+                        ch = glob.glob(mstr_datafolder + "_cache/*")
+                        for f in ch:
+                            if os_platform == "nt":
+                                if self._isFileAccessibleWin(f) == True:
+                                    os.remove(f)
+                            if os_platform == "posix":
+                                if self._isFileAccessiblePosix(f) == True:
+                                    os.remove(f)
+                        mstr_msg("orthographic", "Cleared cache")
+                    """
+                
+
+                # Adjust latitude and all other values when we get here
+                cur_tile_y = cur_tile_y+1
+                cur_tile_x = 1
+                bb_lng = self._long
+                bb_lng_edge = self._long + mstr_zl_18
+                bb_lat = bb_lat + self._vstep
+                bb_lat_edge = bb_lat_edge + self._vstep
+                mstr_msg("orthographic", "Adjustment of latitude performed")
+                # Adjust peak latitude number
+                if cur_tile_y > top_lat:
+                    top_lat = cur_tile_y
+            
+            mstr_msg("orthographic", "Generation of all tiles completed!")
+
+
+            # Complete scenery
+            if mstr_xp_genscenery == True:
+                scn = mstr_xp_scenery(self._lat, self._long, mlat, mlng, self._vstep, self._latlngfld)
+                scn.acquire_elevation_data()
+                scn.acquire_xes_data()
+                scn.build_mesh_script()
+                scn.build_mesh()
+                scn.build_ter_files()
+                mstr_msg("orthographic", "[X-Plane] Mesh built, and scenery completed")
+            
+            mstr_msg("orthographic", "Final step completed.")
+            mstr_msg("orthographic", "Tile data in: " + self._output + "z_orthographic/" + self._latlngfld)
+            print("")
+            mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
+            print("")
 
-        # Let's leave this out for the moment
-        """
-        mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles")
-        tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng)
-        tg.genTiles()
-        mstr_msg("orthographic", "Final step completed.")
-        print("")
-        mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng)
-        print("")
-        print("")
-        mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
-        print("")
-        """
+            # Let's leave this out for the moment
+            """
+            mstr_msg("orthographic", "Generating ZL16 tiles and keeping airport tiles")
+            tg = mstr_tilegen(self._lat, self._lng, self._vstep, top_lat, top_lng)
+            tg.genTiles()
+            mstr_msg("orthographic", "Final step completed.")
+            print("")
+            mstr_msg("orthographic", "Tile data in: " + mstr_datafolder + "/Tiles/" + str(self._lat) + "_" + self._lng)
+            print("")
+            print("")
+            mstr_msg("orthographic", "Thanks for using Orthographic! -- Best, Marcus")
+            print("")
+            """
 
 
 
 
 
     # Checks which layers need to be generated, and what kind of layer it is
-    def determineLayerWork(self):
+    def determineLayerWork(self, xmlobj):
 
         mstr_msg("orthographic", "Checking for work to be performed")
 
         layers = []
 
-        tilexml = mstr_datafolder + "_cache/tile.xml"
-        xml = mstr_osmxml(0,0)
-        way = xml.acquire_waypoint_data(tilexml)
-        rls = xml.acquire_relations(tilexml)
+        #tilexml = mstr_datafolder + "_cache/tile.xml"
+        #xml = mstr_osmxml(0,0)
+        way = xmlobj.acquire_waypoint_data()
+        rls = xmlobj.acquire_relations()
 
         for l in mstr_ortho_layers:
             # Check if there is anything to render
diff --git a/osmxml.py b/osmxml.py
index 7349ea3..05e376e 100644
--- a/osmxml.py
+++ b/osmxml.py
@@ -54,11 +54,11 @@ class mstr_osmxml:
 
 
     # Acquire XMLs in chunks, then store them
-    def acquire_osm(self, v, h, asobject=False):
+    def acquire_osm(self, v, h):
         mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng))
         
         # We will use our self-hosted API for this.
-        while os.path.isfile(self._xmlfn) == False:
+        while self._xmlcontent == "":
             data = { 
                 "bbox": { 
                         "lat": str(self._lat),
@@ -86,7 +86,7 @@ class mstr_osmxml:
                 sleep(1)
         
         # Store the content in memory
-        self._xmldata = xml.dom.minidom.parse(self._xmlcontent)
+        self._xmldata = xml.dom.minidom.parseString(self._xmlcontent)
         self._xmlcontent = "" # Clear
 
 
diff --git a/tileprep.py b/tileprep.py
index 8de6c39..f5197a2 100644
--- a/tileprep.py
+++ b/tileprep.py
@@ -43,6 +43,16 @@ class mstr_tileprep:
         self._is_completion = is_completion
 
 
+    def _findCorrectTextureFolder(self):
+        srcfld = []
+        for lyr in mstr_ortho_layers:
+            if lyr[0] == self._tag and lyr[1] == self._value:
+                srcfld.append(lyr[2])
+                srcfld.append(lyr[3])
+                break
+        return srcfld
+
+
     # Prepare the tile accordingly
     def _prepareTile(self):
         # Load the mask pixels
@@ -117,7 +127,8 @@ class mstr_tileprep:
 
         # If there was nothing in the info still, we need to select some source
         if self._source == -1:
-            tx = mstr_datafolder + "textures/" + self._tag + "/" + self._value + "/brd/b*.png"
+            srcfld = self._findCorrectTextureFolder()
+            tx = mstr_datafolder + "textures/" + srcfld[0] + "/" + srcfld[1] + "/brd/b*.png"
             lst = glob.glob(tx)
             if len(lst) == 1: self._source = 1
             if len(lst) >= 2: self._source = randrange(1, len(lst)+1)