XML handling moved completely into memory for performance. Initial commit of tileprep class in preparation for ortho generation redesign

osmxml.py

@@ -19,13 +19,10 @@ from defines import *
 from log import *
 
 class mstr_osmxml:
-    def __init__(self, lat, lng):
+    def __init__(self):
-        self._latv = lat
+        #self._xmlfn = mstr_datafolder + "_cache/tile_" + str(lat) + "-" + str(v) + "_" + str(lng) + "-" + str(h) + ".xml"
-        self._lngv = lng
+        self._xmldata = None
-        self._lat = lat
+        self._xmlcontent = ""
-        self._lng = lng
-        self._curB_lat = lat + mstr_zl_18
-        self._curB_lng = lng + mstr_zl_18
 
     
     # Adjust bbox for when this class should persost, but acquire data for a different bbox
@@ -61,7 +58,7 @@ class mstr_osmxml:
         mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng))
         
         # We will use our self-hosted API for this.
-        while os.path.isfile(mstr_datafolder + "_cache/tile.xml") == False:
+        while os.path.isfile(self._xmlfn) == False:
             data = { 
                 "bbox": { 
                         "lat": str(self._lat),
@@ -76,26 +73,27 @@ class mstr_osmxml:
                     }
             r = requests.post(mstr_osm_endpoint, json=data)
 
-            xmlf = mstr_datafolder + "_cache/tile.xml"
+            self._xmlcontent = r.content
-            if os.path.isfile(xmlf):
+
-                os.remove(xmlf)
+            #if os.path.isfile(self._xmlfn):
-            with open(xmlf, 'wb') as textfile:
+            #    os.remove(self._xmlfn)
-                textfile.write(r.content)
+            #with open(self._xmlfn, 'wb') as textfile:
+            #    textfile.write(r.content)
 
             # 1 second delay in case the request fails
-            if os.path.isfile(mstr_datafolder + "_cache/tile.xml") == False:
+            if self._xmlcontent == "":
+            #if os.path.isfile(self._xmlfn) == False:
                 sleep(1)
-            
+        
-        # Provide the object directly
+        # Store the content in memory
-        if asobject == True:
+        self._xmldata = xml.dom.minidom.parse(self._xmlcontent)
-            xml_doc = xml.dom.minidom.parse("_cache/tile.xml")
+        self._xmlcontent = "" # Clear
-            return xml_doc
 
 
     # Get all nodes from the specified OSM file
-    def acquire_nodes(self, xmlfile):
+    def acquire_nodes(self):
-        xml_doc = xml.dom.minidom.parse(xmlfile)
+        #xml_doc = xml.dom.minidom.parse(xmlfile)
-        nodedata = xml_doc.getElementsByTagName("node")
+        nodedata = self._xmldata.getElementsByTagName("node")
         nodes = []
         for node in nodedata:
             p = (node.getAttribute("id"), node.getAttribute("lat"), node.getAttribute("lon"))
@@ -104,9 +102,9 @@ class mstr_osmxml:
     
 
     # Get all waypoint data
-    def acquire_waypoint_data(self, xmlfile):
+    def acquire_waypoint_data(self):
-        xml_doc = xml.dom.minidom.parse(xmlfile)
+        #xml_doc = xml.dom.minidom.parse(xmlfile)
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         wps = []
         for wp in wpdata:
             nddata = wp.getElementsByTagName("nd")
@@ -125,10 +123,10 @@ class mstr_osmxml:
 
     # Checks if there is an airport or many airports with an ICAO code in the
     # supplied XML data chunk
-    def find_icao_codes(self, xmlfile):
+    def find_icao_codes(self):
         icao = []
-        xml_doc = xml.dom.minidom.parse(xmlfile)
+        #xml_doc = xml.dom.minidom.parse(xmlfile)
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             tags = wp.getElementsByTagName("tag")
             for tag in tags:
@@ -144,11 +142,11 @@ class mstr_osmxml:
     # If no surface type is specified, the runway will be rendered similar
     # to how motorways are rendered.
     # This gets called only if some ICAO code was found
-    def find_runway_surface(self, xmlfile):
+    def find_runway_surface(self):
         surface = ""
         wpid = ""
-        xml_doc = xml.dom.minidom.parse(xmlfile)
+        #xml_doc = xml.dom.minidom.parse(xmlfile)
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             tags = wp.getElementsByTagName("tag")
             for tag in tags:
@@ -174,8 +172,8 @@ class mstr_osmxml:
     # Find the levels of a building (for shadow rendering)
     def find_building_levels(self, way_id):
         lvl = 2 # Standard if we don't find anything else
-        xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
+        #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             if wp.getAttribute("id") == way_id:
                 tags = wp.getElementsByTagName("tag")
@@ -190,8 +188,8 @@ class mstr_osmxml:
     # Find minimum level of a building
     def find_building_minlevel(self, way_id):
         lvl = 0 # Standard if we don't find anything else
-        xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
+        #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             if wp.getAttribute("id") == way_id:
                 tags = wp.getElementsByTagName("tag")
@@ -206,8 +204,8 @@ class mstr_osmxml:
     # Find the roof shape
     def find_building_roof_shape(self, way_id):
         rs = "flat" # Standard if we don't find anything else
-        xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
+        #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             if wp.getAttribute("id") == way_id:
                 tags = wp.getElementsByTagName("tag")
@@ -222,8 +220,8 @@ class mstr_osmxml:
     # Find the roof levels
     def find_building_roof_levels(self, way_id):
         lvl = 0 # Standard if we don't find anything else
-        xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
+        #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml")
-        wpdata = xml_doc.getElementsByTagName("way")
+        wpdata = self._xmldata.getElementsByTagName("way")
         for wp in wpdata:
             if wp.getAttribute("id") == way_id:
                 tags = wp.getElementsByTagName("tag")
@@ -243,9 +241,9 @@ class mstr_osmxml:
     # the relations and then scan through the outer and inner way IDs.
     #
     # Get all relation entries of importance.
-    def acquire_relations(self, xmlfile):
+    def acquire_relations(self):
-        xml_doc = xml.dom.minidom.parse(xmlfile)
+        #xml_doc = xml.dom.minidom.parse(xmlfile)
-        rlxml = xml_doc.getElementsByTagName("relation")
+        rlxml = self._xmldata.getElementsByTagName("relation")
         rls = []
         for rl in rlxml:
             rldata  = rl.getElementsByTagName("member")

tileprep.py

@@ -0,0 +1,139 @@
+
+# -------------------------------------------------------------------
+# ORTHOGRAPHIC
+# Your personal aerial satellite. Always on. At any altitude.*
+# Developed by MarStrMind
+# License: Open Software License 3.0
+# Up to date version always on marstr.online
+# -------------------------------------------------------------------
+# tileprep.py
+# This prepares a zoom level 18 tile for ortho generation. This class
+# performs the edge detection, selects the required sources for each
+# layer, and stores this information into files which are then used
+# by layergen.
+# -------------------------------------------------------------------
+
+import glob
+from random import randrange
+from PIL import Image
+from osmxml import *
+from maskgen import *
+from defines import *
+from functions import *
+from log import *
+from tileinfo import *
+
+class mstr_tileprep:
+
+    # For this to work we need some fundamentals.
+    def __init__(self, lat, lng, v, h, tag, value, mask, is_completion):
+        self._lat       = lat
+        self._lng       = lng
+        self._tile_h    = h
+        self._tile_v    = v
+        self._tag       = tag
+        self._value     = value
+        self._edges     = ""   # To be filled by _edgeDetect call
+        self._source    = -1
+        self._mask      = mask
+        latlngfld = xplane_latlng_folder([lat, lng])
+        self._tileinfo  = mstr_tileinfo(lat, lng, v, h, latlngfld)
+        
+        # Special case for the final step of an ortho
+        self._is_completion = is_completion
+
+
+    # Prepare the tile accordingly
+    def _prepareTile(self):
+        # Load the mask pixels
+        mp = self._mask.load()
+        imgsize = self._mask.width
+
+        # Run scan
+
+        # Check if pixels touch the borders of the image, and if so -
+        # make a not of that in the database.
+        at=False
+        ar=False
+        ab=False
+        al=False
+
+        # Top scan
+        for i in range(0, imgsize-1):
+            p = mp[i,0]
+            if p[3] > 0:
+                at=True
+                break
+       
+        # Right scan
+        for i in range(0, imgsize-1):
+            p = mp[imgsize-1,i]
+            if p[3] > 0:
+                ar=True
+                break
+        
+        # Bottom scan
+        for i in range(0, imgsize-1):
+            p = mp[i,imgsize-1]
+            if p[3] > 0:
+                ab=True
+                break
+        
+        # Left scan
+        for i in range(0, imgsize-1):
+            p = mp[1,i]
+            if p[3] > 0:
+                al=True
+                break
+        
+        # Construct adjacency string
+        adjstr = ""
+        if at==True: adjstr = adjstr + "t"
+        if ar==True: adjstr = adjstr + "r"
+        if ab==True: adjstr = adjstr + "b"
+        if al==True: adjstr = adjstr + "l"
+
+        # Now find out of there is a source from any adjacent tile
+        adjtiles = findAdjacentTilesTo(self._tile_v, self._tile_h)
+        sat = []
+        sar = []
+        sab = []
+        sal = []
+        if self._is_completion == False:
+            if at == True: sat = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top
+            if ar == True: sar = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right
+            if ab == True: sab = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom
+            if al == True: sal = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left
+        if self._is_completion == True:
+            if at == True: sat = self._tileinfo.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top
+            if ar == True: sar = self._tileinfo.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right
+            if ab == True: sab = self._tileinfo.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom
+            if al == True: sal = self._tileinfo.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left
+
+        if self._source == -1 and len(sat) == 2: self._source = sat[0]
+        if self._source == -1 and len(sar) == 2: self._source = sar[0]
+        if self._source == -1 and len(sab) == 2: self._source = sab[0]
+        if self._source == -1 and len(sal) == 2: self._source = sal[0]
+
+        # If there was nothing in the info still, we need to select some source
+        if self._source == -1:
+            tx = mstr_datafolder + "textures/" + self._tag + "/" + self._value + "/brd/b*.png"
+            lst = glob.glob(tx)
+            if len(lst) == 1: self._source = 1
+            if len(lst) >= 2: self._source = randrange(1, len(lst)+1)
+
+
+        # Store into DB - but only if there is something to store
+        if adjstr != "":
+            if self._is_completion == False:
+                r = self._tileinfo.get_adjacency_for_tag_and_value(self._tile_v, self._tile_h, self._tag, self._value)
+                if len(r) == 0:
+                    self._tileinfo.add_adjacency_data(self._tile_v, self._tile_h, self._tag, self._value, self._source, adjstr)
+                    mstr_msg("tileprep", "Adjacency info stored in database")
+
+            if self._is_completion == True:
+                r = self._tileinfo.get_adjacency_for_completion(self._tile_v, self._tile_h)
+                if len(r) == 0:
+                    self._tileinfo.add_completion_data(self._tile_v, self._tile_h, self._tag, self._value, self._source, adjstr)
+                    mstr_msg("tileprep", "Adjacency info for completion stored in database")
+