From: Marcus Str. Date: Tue, 12 Nov 2024 11:44:22 +0000 (+0100) Subject: XML handling moved completely into memory for performance. Initial commit of tileprep... X-Git-Url: https://marstr.online/code/gitweb.cgi?a=commitdiff_plain;h=925f7bee27f90219c92efec702f48735f7139223;p=orthographic XML handling moved completely into memory for performance. Initial commit of tileprep class in preparation for ortho generation redesign --- diff --git a/osmxml.py b/osmxml.py index c80dc4b..7349ea3 100644 --- a/osmxml.py +++ b/osmxml.py @@ -19,13 +19,10 @@ from defines import * from log import * class mstr_osmxml: - def __init__(self, lat, lng): - self._latv = lat - self._lngv = lng - self._lat = lat - self._lng = lng - self._curB_lat = lat + mstr_zl_18 - self._curB_lng = lng + mstr_zl_18 + def __init__(self): + #self._xmlfn = mstr_datafolder + "_cache/tile_" + str(lat) + "-" + str(v) + "_" + str(lng) + "-" + str(h) + ".xml" + self._xmldata = None + self._xmlcontent = "" # Adjust bbox for when this class should persost, but acquire data for a different bbox @@ -61,7 +58,7 @@ class mstr_osmxml: mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng)) # We will use our self-hosted API for this. - while os.path.isfile(mstr_datafolder + "_cache/tile.xml") == False: + while os.path.isfile(self._xmlfn) == False: data = { "bbox": { "lat": str(self._lat), @@ -76,26 +73,27 @@ class mstr_osmxml: } r = requests.post(mstr_osm_endpoint, json=data) - xmlf = mstr_datafolder + "_cache/tile.xml" - if os.path.isfile(xmlf): - os.remove(xmlf) - with open(xmlf, 'wb') as textfile: - textfile.write(r.content) + self._xmlcontent = r.content + + #if os.path.isfile(self._xmlfn): + # os.remove(self._xmlfn) + #with open(self._xmlfn, 'wb') as textfile: + # textfile.write(r.content) # 1 second delay in case the request fails - if os.path.isfile(mstr_datafolder + "_cache/tile.xml") == False: + if self._xmlcontent == "": + #if os.path.isfile(self._xmlfn) == False: sleep(1) - - # Provide the object directly - if asobject == True: - xml_doc = xml.dom.minidom.parse("_cache/tile.xml") - return xml_doc + + # Store the content in memory + self._xmldata = xml.dom.minidom.parse(self._xmlcontent) + self._xmlcontent = "" # Clear # Get all nodes from the specified OSM file - def acquire_nodes(self, xmlfile): - xml_doc = xml.dom.minidom.parse(xmlfile) - nodedata = xml_doc.getElementsByTagName("node") + def acquire_nodes(self): + #xml_doc = xml.dom.minidom.parse(xmlfile) + nodedata = self._xmldata.getElementsByTagName("node") nodes = [] for node in nodedata: p = (node.getAttribute("id"), node.getAttribute("lat"), node.getAttribute("lon")) @@ -104,9 +102,9 @@ class mstr_osmxml: # Get all waypoint data - def acquire_waypoint_data(self, xmlfile): - xml_doc = xml.dom.minidom.parse(xmlfile) - wpdata = xml_doc.getElementsByTagName("way") + def acquire_waypoint_data(self): + #xml_doc = xml.dom.minidom.parse(xmlfile) + wpdata = self._xmldata.getElementsByTagName("way") wps = [] for wp in wpdata: nddata = wp.getElementsByTagName("nd") @@ -125,10 +123,10 @@ class mstr_osmxml: # Checks if there is an airport or many airports with an ICAO code in the # supplied XML data chunk - def find_icao_codes(self, xmlfile): + def find_icao_codes(self): icao = [] - xml_doc = xml.dom.minidom.parse(xmlfile) - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(xmlfile) + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: tags = wp.getElementsByTagName("tag") for tag in tags: @@ -144,11 +142,11 @@ class mstr_osmxml: # If no surface type is specified, the runway will be rendered similar # to how motorways are rendered. # This gets called only if some ICAO code was found - def find_runway_surface(self, xmlfile): + def find_runway_surface(self): surface = "" wpid = "" - xml_doc = xml.dom.minidom.parse(xmlfile) - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(xmlfile) + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: tags = wp.getElementsByTagName("tag") for tag in tags: @@ -174,8 +172,8 @@ class mstr_osmxml: # Find the levels of a building (for shadow rendering) def find_building_levels(self, way_id): lvl = 2 # Standard if we don't find anything else - xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: if wp.getAttribute("id") == way_id: tags = wp.getElementsByTagName("tag") @@ -190,8 +188,8 @@ class mstr_osmxml: # Find minimum level of a building def find_building_minlevel(self, way_id): lvl = 0 # Standard if we don't find anything else - xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: if wp.getAttribute("id") == way_id: tags = wp.getElementsByTagName("tag") @@ -206,8 +204,8 @@ class mstr_osmxml: # Find the roof shape def find_building_roof_shape(self, way_id): rs = "flat" # Standard if we don't find anything else - xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: if wp.getAttribute("id") == way_id: tags = wp.getElementsByTagName("tag") @@ -222,8 +220,8 @@ class mstr_osmxml: # Find the roof levels def find_building_roof_levels(self, way_id): lvl = 0 # Standard if we don't find anything else - xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") - wpdata = xml_doc.getElementsByTagName("way") + #xml_doc = xml.dom.minidom.parse(mstr_datafolder + "_cache/tile.xml") + wpdata = self._xmldata.getElementsByTagName("way") for wp in wpdata: if wp.getAttribute("id") == way_id: tags = wp.getElementsByTagName("tag") @@ -243,9 +241,9 @@ class mstr_osmxml: # the relations and then scan through the outer and inner way IDs. # # Get all relation entries of importance. - def acquire_relations(self, xmlfile): - xml_doc = xml.dom.minidom.parse(xmlfile) - rlxml = xml_doc.getElementsByTagName("relation") + def acquire_relations(self): + #xml_doc = xml.dom.minidom.parse(xmlfile) + rlxml = self._xmldata.getElementsByTagName("relation") rls = [] for rl in rlxml: rldata = rl.getElementsByTagName("member") diff --git a/tileprep.py b/tileprep.py new file mode 100644 index 0000000..8de6c39 --- /dev/null +++ b/tileprep.py @@ -0,0 +1,139 @@ + +# ------------------------------------------------------------------- +# ORTHOGRAPHIC +# Your personal aerial satellite. Always on. At any altitude.* +# Developed by MarStrMind +# License: Open Software License 3.0 +# Up to date version always on marstr.online +# ------------------------------------------------------------------- +# tileprep.py +# This prepares a zoom level 18 tile for ortho generation. This class +# performs the edge detection, selects the required sources for each +# layer, and stores this information into files which are then used +# by layergen. +# ------------------------------------------------------------------- + +import glob +from random import randrange +from PIL import Image +from osmxml import * +from maskgen import * +from defines import * +from functions import * +from log import * +from tileinfo import * + +class mstr_tileprep: + + # For this to work we need some fundamentals. + def __init__(self, lat, lng, v, h, tag, value, mask, is_completion): + self._lat = lat + self._lng = lng + self._tile_h = h + self._tile_v = v + self._tag = tag + self._value = value + self._edges = "" # To be filled by _edgeDetect call + self._source = -1 + self._mask = mask + latlngfld = xplane_latlng_folder([lat, lng]) + self._tileinfo = mstr_tileinfo(lat, lng, v, h, latlngfld) + + # Special case for the final step of an ortho + self._is_completion = is_completion + + + # Prepare the tile accordingly + def _prepareTile(self): + # Load the mask pixels + mp = self._mask.load() + imgsize = self._mask.width + + # Run scan + + # Check if pixels touch the borders of the image, and if so - + # make a not of that in the database. + at=False + ar=False + ab=False + al=False + + # Top scan + for i in range(0, imgsize-1): + p = mp[i,0] + if p[3] > 0: + at=True + break + + # Right scan + for i in range(0, imgsize-1): + p = mp[imgsize-1,i] + if p[3] > 0: + ar=True + break + + # Bottom scan + for i in range(0, imgsize-1): + p = mp[i,imgsize-1] + if p[3] > 0: + ab=True + break + + # Left scan + for i in range(0, imgsize-1): + p = mp[1,i] + if p[3] > 0: + al=True + break + + # Construct adjacency string + adjstr = "" + if at==True: adjstr = adjstr + "t" + if ar==True: adjstr = adjstr + "r" + if ab==True: adjstr = adjstr + "b" + if al==True: adjstr = adjstr + "l" + + # Now find out of there is a source from any adjacent tile + adjtiles = findAdjacentTilesTo(self._tile_v, self._tile_h) + sat = [] + sar = [] + sab = [] + sal = [] + if self._is_completion == False: + if at == True: sat = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[0][0], adjtiles[0][1], self._tag, self._value) # Top + if ar == True: sar = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[1][0], adjtiles[1][1], self._tag, self._value) # Right + if ab == True: sab = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[2][0], adjtiles[2][1], self._tag, self._value) # Bottom + if al == True: sal = self._tileinfo.get_adjacency_for_tag_and_value(adjtiles[3][0], adjtiles[3][1], self._tag, self._value) # Left + if self._is_completion == True: + if at == True: sat = self._tileinfo.get_adjacency_for_completion(adjtiles[0][0], adjtiles[0][1]) # Top + if ar == True: sar = self._tileinfo.get_adjacency_for_completion(adjtiles[1][0], adjtiles[1][1]) # Right + if ab == True: sab = self._tileinfo.get_adjacency_for_completion(adjtiles[2][0], adjtiles[2][1]) # Bottom + if al == True: sal = self._tileinfo.get_adjacency_for_completion(adjtiles[3][0], adjtiles[3][1]) # Left + + if self._source == -1 and len(sat) == 2: self._source = sat[0] + if self._source == -1 and len(sar) == 2: self._source = sar[0] + if self._source == -1 and len(sab) == 2: self._source = sab[0] + if self._source == -1 and len(sal) == 2: self._source = sal[0] + + # If there was nothing in the info still, we need to select some source + if self._source == -1: + tx = mstr_datafolder + "textures/" + self._tag + "/" + self._value + "/brd/b*.png" + lst = glob.glob(tx) + if len(lst) == 1: self._source = 1 + if len(lst) >= 2: self._source = randrange(1, len(lst)+1) + + + # Store into DB - but only if there is something to store + if adjstr != "": + if self._is_completion == False: + r = self._tileinfo.get_adjacency_for_tag_and_value(self._tile_v, self._tile_h, self._tag, self._value) + if len(r) == 0: + self._tileinfo.add_adjacency_data(self._tile_v, self._tile_h, self._tag, self._value, self._source, adjstr) + mstr_msg("tileprep", "Adjacency info stored in database") + + if self._is_completion == True: + r = self._tileinfo.get_adjacency_for_completion(self._tile_v, self._tile_h) + if len(r) == 0: + self._tileinfo.add_completion_data(self._tile_v, self._tile_h, self._tag, self._value, self._source, adjstr) + mstr_msg("tileprep", "Adjacency info for completion stored in database") +