Re-enabled download of elevation, activated proper code in orthographic.py

defines.py

@@ -74,8 +74,10 @@ mstr_xp_scn_normalmaps = True
 # Paths to required X-Plane scenery tools
 mstr_xp_meshtool = "/home/marcus/Developer/Projects/orthographic/bin/MeshTool"
 mstr_xp_ddstool = "/home/marcus/Developer/Projects/orthographic/bin/DDSTool"
+mstr_xp_dsftool = "/home/marcus/Developer/Projects/orthographic/bin/DSFTool"
 mstr_xp_xessrc = "https://dev.x-plane.com/update/misc/MeshTool/"
 mstr_xp_floor_height = 2.8 # 2.5m ceiling height + 30cm concrete per floor
+mstr_xp_ortho_location = "/home/marcus/Data/Sim/Simulator/orthographic/"
 
 # If you set the above to true, you can define for which features you
 # want to generate normal maps for. The below is my recommendation for

layergen.py

@@ -22,7 +22,6 @@ from log import *
 from tileinfo import *
 from osmxml import *
 from functions import *
-from xp_normalmap import *
 
 class mstr_layergen:
 
@@ -409,6 +408,7 @@ class mstr_layergen:
 
             # Depending on if scenery for XP should be made, AND if normal maps should be made, we would
             # need to make them at this exact point
+            """
             if mstr_xp_genscenery == True:
                 if mstr_xp_scn_normalmaps == True and self._is_completion == False:
                     nm = False
@@ -419,6 +419,7 @@ class mstr_layergen:
                     if nm == True:
                         nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
                         nrm.build_normalmap(layer_comp)
+            """
 
 
             # Let's try our hand at pseudo shadows
@@ -457,20 +458,32 @@ class mstr_layergen:
 
 
             # Create a water mask we need to remove from the DDS later
+            """
             if (self._tag == "natural" and self._value == "water") or (self._tag == "water" and self._value == "lake") or (self._tag == "water" and self._value == "pond") or (self._tag == "water" and self._value == "river") or (self._tag == "leisure" and self._value == "swimming_pool"):
                 mstr_msg("layergen", "Generating inland water mask")
-                inl_mask = Image.new("RGBA", (self._imgsize, self._imgsize), (0,0,0,0))
+                water_file = mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(self._lat_number) + "_" + str(self._lng_number) + "_water.png"
+                inl_mask = None
+                if os.path.isfile(water_file):
+                    inl_mask = Image.open(water_file)
+                else:
+                    inl_mask = Image.new("L", (self._imgsize, self._imgsize), (255))
                 lyr_pix = layer_comp.load()
                 inl_pix = inl_mask.load()
                 for y in range(self._imgsize):
                     for x in range(self._imgsize):
                         l = lyr_pix[x,y]
-                        if l[3] > 65:
-                            b = 255 - l[3]
-                            inl_pix[x,y] = (255,0,255,255)
+                        if l[3] > 50:
+                            clr = 255-l[3]
+                            c = (clr)
+                            inl_pix[x,y] = c
+                inl_mask.save(water_file)
+                        #if l[3] > 65:
+                        #    b = 255 - l[3]
+                        #    inl_pix[x,y] = (255,0,255,255)
                 #inl_mask.save(mstr_datafolder + "_cache/" + str(self._latitude) + "-" + str(self._lat_number) + "_" + str(self._longitude) + "-" + str(self._lng_number) + "_" + self._tag + "-" + self._value + "_layer_mask.png")
-                layer_comp = inl_mask
+                #layer_comp = inl_mask
                 mstr_msg("layergen", "Inland water mask generated and saved")
+            """
 
             # Return the completed image
             return layer_comp
@@ -525,6 +538,10 @@ class mstr_layergen:
                             if rw_surface == "" or rw_surface == "asphalt":
                                 d = randrange(81, 101)
                                 layer_comp_pix[x, y] = ( d,d,d,a[3] )
+                        if self._tag == "aeroway" and self._value == "taxiway":
+                            # Almost the same as above
+                            d = randrange(81, 101)
+                            layer_comp_pix[x, y] = ( d,d,d,a[3] )
                         if self._tag == "railway":
                             d = randrange(41, 61)
                             layer_comp_pix[x, y] = ( d,d,d,a[3] )
@@ -874,6 +891,7 @@ class mstr_layergen:
 
             # Depending on if scenery for XP should be made, AND if normal maps should be made, we would
             # need to make them at this exact point
+            """
             if mstr_xp_genscenery == True:
                 if mstr_xp_scn_normalmaps == True and self._is_completion == False:
                     nm = False
@@ -884,6 +902,7 @@ class mstr_layergen:
                     if nm == True:
                         nrm = mstr_xp_normalmap(self._latitude, self._longitude, self._tag, self._value, self._lat_number, self._lng_number, self._latlngfld)
                         nrm.build_normalmap(layer_comp)
+            """
 
 
             # Return image

og.py

@@ -54,7 +54,12 @@ if cli == True:
         og._prepareTile()
 
     if prep == False:
-        og._generateOrthos_mt(int(sys.argv[3]))
+        if sys.argv[3] != "xpscenery":
+            og._generateOrthos_mt(int(sys.argv[3]))
+        
+        # Build the terrain mesh and assign ground textures
+        if sys.argv[3] == "xpscenery":
+            og.generate_xp_scenery()
 
 
 # Only if we find enough arguments, proceed.

orthographic.py

@@ -208,64 +208,76 @@ class mstr_orthographic:
         maxlatlng = [ mlat, mlng ]
 
         while grid_lat <= maxlatlng[0]:
-            # Reset these two
-            bb_lat = self._lat + ((grid_lat-1)*self._vstep)
-            bb_lng = self._long + ((grid_lng-1)*mstr_zl_18)
-            bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep
-            bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18
+            ddsf = mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(grid_lat) + "_" + str(grid_lng) + ".dds"
+            if os.path.isfile(ddsf) == False:
+                # Reset these two
+                bb_lat = self._lat + ((grid_lat-1)*self._vstep)
+                bb_lng = self._long + ((grid_lng-1)*mstr_zl_18)
+                bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep
+                bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18
 
-            osmxml = mstr_osmxml()
-            osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
-            osmxml.acquire_osm(grid_lat, grid_lng)
+                osmxml = mstr_osmxml()
+                osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge)
+                osmxml.acquire_osm(grid_lat, grid_lng)
 
-            # Let the user know
-            mstr_msg("orthographic", "Generating orthophoto " + str(grid_lat) + "-" + str(grid_lng))
-
-            # Check for work to be done
-            layers = self.determineLayerWork(osmxml)
-
-            # We need to walk through the array of layers,
-            # in their z-order.
-            # For each layer, we will generate the mask, the layer image
-            # itself, and finally, compose the ortho photo.
-            mstr_msg("orthographic", "Beginning generation of layers")
-
-            # In here we store the layers
-            photolayers = []
-
-            # The masks are handed to layergen in sequence. The layers are then
-            # in turn handed to photogen.
-
-            curlyr = 1
-            for layer in layers:
                 # Let the user know
-                mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
+                mstr_msg("orthographic", "Generating missing orthophoto " + str(grid_lat) + "-" + str(grid_lng))
 
-                # Generate the mask
-                mg = mstr_maskgen( [self._lat, grid_lat, self._long, grid_lng], self._vstep, layer[0], layer[1], layer[2])
-                if layer[0] == "building":
-                    mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
-                    mg.set_latlng_numbers(self._lat, grid_lat, self._long, grid_lng)
-                mask = mg._build_mask(osmxml)
-                
-                # Generate the layer
-                lg = mstr_layergen(layer[0], layer[1], self._lat, grid_lat, self._long, grid_lng, layer[2])
-                lg.set_max_latlng_tile(maxlatlng)
-                lg.set_latlng_folder(self._latlngfld)
-                #lg.open_db()
-                lg.open_tile_info()
-                photolayers.append(lg.genlayer(mask, osmxml))
-                curlyr = curlyr+1
-            mstr_msg("orthographic", "All layers created")
+                # Check for work to be done
+                layers = self.determineLayerWork(osmxml)
 
-            # We should have all layers now.
-            # Snap a photo with our satellite :)
-            mstr_msg("orthographic", "Generating ortho photo")
-            pg = mstr_photogen(self._lat, self._long, grid_lat, grid_lng,  maxlatlng[0], maxlatlng[1])
-            pg.genphoto(photolayers)
-            mstr_msg("orthographic", " -- Ortho photo generated -- ")
-            print("")
-            print("")
+                # We need to walk through the array of layers,
+                # in their z-order.
+                # For each layer, we will generate the mask, the layer image
+                # itself, and finally, compose the ortho photo.
+                mstr_msg("orthographic", "Beginning generation of layers")
+
+                # In here we store the layers
+                photolayers = []
+                waterlayers = []
+
+                # The masks are handed to layergen in sequence. The layers are then
+                # in turn handed to photogen.
+
+                curlyr = 1
+                wtr_info = False
+                for layer in layers:
+                    # Let the user know
+                    mstr_msg("orthographic", "Processing layer " + str(curlyr) + " of " + str(len(layers)))
+
+                    # Generate the mask
+                    mg = mstr_maskgen( [self._lat, grid_lat, self._long, grid_lng], self._vstep, layer[0], layer[1], layer[2])
+                    if layer[0] == "building":
+                        mg.set_tile_width(self._findWidthOfLongitude(bb_lat))
+                        mg.set_latlng_numbers(self._lat, grid_lat, self._long, grid_lng)
+                    mask = mg._build_mask(osmxml)
+                    
+                    # Generate the layer
+                    lg = mstr_layergen(layer[0], layer[1], self._lat, grid_lat, self._long, grid_lng, layer[2])
+                    lg.set_max_latlng_tile(maxlatlng)
+                    lg.set_latlng_folder(self._latlngfld)
+                    #lg.open_db()
+                    lg.open_tile_info()
+                    lyr = lg.genlayer(mask, osmxml)
+                    photolayers.append(lyr)
+                    if (layer[0] == "natural" and layer[1] == "water") or (layer[0] == "water" and layer[1] == "lake") or (layer[0] == "water" and layer[1] == "pond") or (layer[0] == "water" and layer[1] == "river") or (layer[0] == "waterway" and layer[1] == "river"):
+                        waterlayers.append(lyr)
+                        if wtr_info == False:
+                            wtr_info = True
+                            wtrfile = mstr_datafolder + "z_orthographic/data/" + self._latlngfld + "/wtrfile"
+                            with open(wtrfile, 'a') as textfile:
+                                textfile.write(str(grid_lat) + " " + str(grid_lng) + "\r\n")
+                    curlyr = curlyr+1
+                mstr_msg("orthographic", "All layers created")
+
+                # We should have all layers now.
+                # Snap a photo with our satellite :)
+                mstr_msg("orthographic", "Generating ortho photo")
+                pg = mstr_photogen(self._lat, self._long, grid_lat, grid_lng,  maxlatlng[0], maxlatlng[1])
+                pg.genphoto(photolayers, waterlayers)
+                mstr_msg("orthographic", " -- Ortho photo generated -- ")
+                print("")
+                print("")
 
             # Perform adjustment of grid position
             n_lng = grid_lng + step
@@ -418,7 +430,49 @@ class mstr_orthographic:
 
 
 
+    # Generates X-Plane 11/12 scenery with
+    # - the finished orthos
+    # - a current LIDAR scan of the terrain
+    def generate_xp_scenery(self):
+        mstr_msg("orthographic", "[X-Plane] Generation of scenery started")
 
+        # This call appears quite often... surely this can be done better
+        mlat = 1
+        mlng = 1
+        bb_lat = self._lat
+        bb_lng = self._long
+        bb_lat_edge = self._lat+self._vstep
+        bb_lng_edge = self._long+mstr_zl_18
+        while bb_lat < self._lat + 1:
+            bb_lat = bb_lat + self._vstep
+            mlat = mlat+1
+        while bb_lng < self._long + 1:
+            bb_lng = bb_lng + mstr_zl_18
+            mlng = mlng+1
+        mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng))
+        maxlatlng = [ mlat, mlng ]
+        
+        # The object that handles it all
+        xpscn = mstr_xp_scenery(self._lat, self._long, maxlatlng[0], maxlatlng[1], self._vstep, self._latlngfld)
+        mstr_msg("orthographic", "[X-Plane] Scenery object instantiated")
+
+        # Download LIDAR scan from our endpoint
+        xpscn.acquire_elevation_data()
+        mstr_msg("orthographic", "[X-Plane] Elevation data acquired")
+
+        # Generate the .ter files
+        xpscn.build_ter_files()
+        mstr_msg("orthographic", "[X-Plane] Terrain files (.ter) generated and written")
+
+        # And lastly, generate the mesh
+        xpscn.generate_terrain_mesh()
+        mstr_msg("orthographic", "[X-Plane] Scenery mesh constructed")
+
+        # Convert the DSF
+        xpscn.build_and_convert_dsf()
+        mstr_msg("orthographic", "[X-Plane] DSF generated")
+        
+    
     # Checks which layers need to be generated, and what kind of layer it is
     def determineLayerWork(self, xmlobj):
 

osmxml.py

@@ -13,16 +13,23 @@
 
 
 import xml.dom.minidom
+from pyexpat import ExpatError
+
 import requests
 import os
 from defines import *
 from log import *
+import time
 
 class mstr_osmxml:
     def __init__(self):
         #self._xmlfn = mstr_datafolder + "_cache/tile_" + str(lat) + "-" + str(v) + "_" + str(lng) + "-" + str(h) + ".xml"
         self._xmldata = None
         self._xmlcontent = ""
+        self._lat = 0
+        self._lng = 0
+        self._curB_lat = 0
+        self._curB_lng = 0
 
     
     # Adjust bbox for when this class should persost, but acquire data for a different bbox
@@ -58,7 +65,8 @@ class mstr_osmxml:
         mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng))
         
         # We will use our self-hosted API for this.
-        while self._xmlcontent == "":
+        parse = False
+        while parse == False:
             data = { 
                 "bbox": { 
                         "lat": str(self._lat),
@@ -73,21 +81,24 @@ class mstr_osmxml:
                     }
             r = requests.post(mstr_osm_endpoint, json=data)
 
-            self._xmlcontent = r.content
+            try:
+                # Attempt to parse the XML string
+                dom = xml.dom.minidom.parseString(r.content)
 
-            #if os.path.isfile(self._xmlfn):
-            #    os.remove(self._xmlfn)
-            #with open(self._xmlfn, 'wb') as textfile:
-            #    textfile.write(r.content)
+                # Check if the DOM object has a document element
+                if dom.documentElement:
+                    # Store the content in memory
+                    self._xmlcontent = r.content
+                    self._xmldata = xml.dom.minidom.parseString(self._xmlcontent)
+                    self._xmlcontent = ""  # Clear
+                    parse = True
 
-            # 1 second delay in case the request fails
-            if self._xmlcontent == "":
-            #if os.path.isfile(self._xmlfn) == False:
-                sleep(1)
-        
-        # Store the content in memory
-        self._xmldata = xml.dom.minidom.parseString(self._xmlcontent)
-        self._xmlcontent = "" # Clear
+            except ExpatError as e:
+                parse = False
+                time.sleep(1)
+            except Exception as e:
+                parse = False
+                time.sleep(1)
 
 
     # Get all nodes from the specified OSM file
@@ -181,7 +192,7 @@ class mstr_osmxml:
                     a = tag.getAttribute("k")
                     v = tag.getAttribute("v")
                     if a == "building:levels":
-                        lvl = int(v)
+                        lvl = int(float(v)) # <- This blew layergen and maskgen at some buildings with 1.5 floors
                         break
         return lvl
     
@@ -197,7 +208,7 @@ class mstr_osmxml:
                     a = tag.getAttribute("k")
                     v = tag.getAttribute("v")
                     if a == "building:min_level":
-                        lvl = int(v)
+                        lvl = int(float(v))
                         break
         return lvl
     

photogen.py

@@ -5,6 +5,7 @@ from defines import *
 from layergen import *
 from log import *
 from functions import *
+from xp_normalmap import *
 
 # -------------------------------------------------------------------
 # ORTHOGRAPHIC
@@ -29,9 +30,7 @@ class mstr_photogen:
         self._tx = tx
         self._maxlatlng = [ maxlat, maxlng ]
         # Define layer size depending on what is wanted
-        self._imgsize = 0
-        if mstr_photores == 2048: self._imgsize = 2048
-        if mstr_photores == 4096: self._imgsize = 6000
+        self._imgsize = mstr_photores
         # Empty image where everything goes into
         self._tile = Image.new("RGBA", (self._imgsize, self._imgsize))
         self._latlngfld = self.latlng_folder([lat,lng])
@@ -39,7 +38,7 @@ class mstr_photogen:
 
     
     # This puts it all together. Bonus: AND saves it.
-    def genphoto(self, layers):
+    def genphoto(self, layers, waterlayers):
         # Template for the file name which is always the same
         #root_filename = mstr_datafolder + "/_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_"
 
@@ -103,6 +102,11 @@ class mstr_photogen:
                 ptc = Image.open(mstr_datafolder + "textures/tile/completion/p" + str(randrange(1, len(patches)+1)) + ".png")
                 # Rotate it
                 ptc = ptc.rotate(randrange(0, 360), expand=True)
+
+                # Make sure ortho generation does not crash
+                if ptc.width >= mstr_photores:
+                    ptc = ptc.resize((1536, 1536), Image.Resampling.BILINEAR)
+
                 # Adjust alpha on this image
                 ptc_p = ptc.load()
                 for y in range(ptc.height):
@@ -119,7 +123,7 @@ class mstr_photogen:
                 py = randrange(1, randrange(self._imgsize - ptc.height - 1))
 
                 # Add it to the completion image
-                cmpl.alpha_composite(ptc)
+                cmpl.alpha_composite(ptc, dest=(px,py))
 
             # Merge the images
             cmpl.alpha_composite(self._tile)
@@ -138,26 +142,6 @@ class mstr_photogen:
                     t = (0,0,0,0)
                     ocean_pix[x,y] = t
 
-        # Now cut out inland water
-        water_layers = (
-            ["natural", "water"],
-            ["water", "lake"],
-            ["water", "pond"],
-            ["water", "river"],
-            ["leisure", "swimming_pool"]
-        )
-        for l in water_layers:
-            fn = mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._ty) + "_" + str(self._lng) + "-" + str(self._tx) + "_" + l[0] + "-" + l[1] + "_layer_mask.png"
-            if os.path.isfile(fn) == True:
-                wtr = Image.open(fn)
-                wtr_pix = wtr.load()
-                tilepix = self._tile.load()
-                for y in range(wtr.height):
-                    for x in range(wtr.width):
-                        wp = wtr_pix[x,y]
-                        if wp[0] == 255 and wp[1] == 0 and wp[2] == 255 and wp[3] == 255:
-                            tilepix[x,y] = (0,0,0,0)
-
         # Alpha correction on final image
         corrpix = self._tile.load()
         for y in range(0, self._tile.height):
@@ -184,6 +168,29 @@ class mstr_photogen:
         os.remove(mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(self._ty) + "_" + str(self._tx) + ".png")
 
 
+        # Now generate the normal map for this ortho.
+        # But only if this is enabled.
+        if mstr_xp_genscenery and mstr_xp_scn_normalmaps:
+            # Generate the normal normal map first (hah)
+            nrm = mstr_xp_normalmap()
+            nrmimg = nrm.generate_normal_map_for_layer(self._tile, False)
+
+            # Now we need to walk through the water layers and generate a combined normal map
+            wtrlyr = Image.new("RGBA", (self._imgsize, self._imgsize))
+            for w in waterlayers:
+                wtrlyr.alpha_composite(w)
+            wtrlyr = wtrlyr.resize((int(mstr_photores/4), int(mstr_photores/4)), Image.Resampling.BILINEAR)
+            wtrimg = nrm.generate_normal_map_for_layer(wtrlyr, True)
+
+            # Blend
+            nrmimg.alpha_composite(wtrimg)
+
+            # Save
+            nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._ty) + "_" + str(
+                self._tx) + ".png"
+            nrmimg.save(nrmfln)
+
+
 
 
     # This checks the final image for empty patches. Should one be

repoinfo

@@ -82,6 +82,19 @@ Apart from that I am aware that the code is most likely not the best and can be
 - Current Python version (3.10 and up)
 - Python modules: Pillow (formerly PIL), requests, numpy
 
+IMPORTANT NOTE: Make sure that Pillow is at least version 11.0. If you have it installed already, you can check the version by doing a
+
+pip list
+
+in either your normal installation or your virtual environment (venv) and check the output. For proper functionality, the result should show like this:
+
+./pip list
+Package            Version
+------------------ ---------
+[...]
+pillow             11.0.0
+[...]
+
 
 [section]Configuration[/section]
 

xp_normalmap.py

@@ -27,27 +27,10 @@ from log import *
 class mstr_xp_normalmap:
 
     # Only a few params
-    def __init__(self, lat, lng, tag, value, tv, th, latlngfld):
-        self._lat = lat
-        self._lng = lng
-        self._tag = tag
-        self._value = value
-        self._latlngfld = latlngfld
-        self._tv = tv
-        self._th = th
+    def __init__(self):
         mstr_msg("xp_normalmap", "[X-Plane] Normal Map generator initialized")
 
 
-    # Load the layer image and resize it to 1/4th its size -
-    # then provide it
-    def load_layer(self):
-        qtr = int(mstr_photores / 4)
-        image = Image.open(mstr_datafolder + "_cache/" + str(self._lat) + "-" + str(self._tv) + "_" + str(self._lng) + "-" + str(self._th) + "_" + self._tag + "-" + self._value + "_layer.png")
-        image = image.resize((qtr,qtr), Image.Resampling.LANCZOS)
-        mstr_msg("xp_normalmap", "[X-Plane] Layer image loaded")
-        return image
-
-
     # A few mathematical calls we need
     # --------------------------------------------------------
     def intensity(self, pixel):
@@ -82,7 +65,7 @@ class mstr_xp_normalmap:
 
 
     # The Big Mac. Generate the normal map
-    def generate_normal_map_for_layer(self, image):
+    def generate_normal_map_for_layer(self, image, water=False):
         mstr_msg("xp_normalmap", "[X-Plane] Beginning normal map generation")
         # No specularity, no reflectivity - but standard color
         # Blue (reflectivity) and alpha (specularity) need to be 1 - but can be adjusted as needed
@@ -91,6 +74,9 @@ class mstr_xp_normalmap:
         image = image.resize((int(mstr_photores/4), int(mstr_photores/4)), Image.Resampling.BILINEAR)
 
         nmp = Image.new("RGBA", (image.width, image.height), (128,128,1,1))
+
+        if water: nmp = Image.new("RGBA", (image.width, image.height), (128, 128, 255, 0))
+
         org = image.load()
         nmp_pix = nmp.load()
 
@@ -134,7 +120,10 @@ class mstr_xp_normalmap:
                         nrm[1] = abs(nrm[1])
 
                     # Set pixel
-                    nmp_pix[x,y] = (int(self.map_component(nrm[0])), int(self.map_component(nrm[1])), 255 - int(self.map_component(nrm[2])), 1)
+                    if water:
+                        nmp_pix[x,y] = (int(self.map_component(nrm[0])), int(self.map_component(nrm[1])), int(self.map_component(nrm[2])), int(self.map_component(nrm[2])))
+                    if not water:
+                        nmp_pix[x,y] = (int(self.map_component(nrm[0])), int(self.map_component(nrm[1])), 255 - int(self.map_component(nrm[2])), 1)
 
         mstr_msg("xp_normalmap", "[X-Plane] Normal map generated")
         return nmp
@@ -143,33 +132,13 @@ class mstr_xp_normalmap:
     # The funnction to call. Blends with the existing map, or creates a new one
     def build_normalmap(self, layer):
         mstr_msg("xp_normalmap", "[X-Plane] Building normal map")
-        # The layer image
-        #lyr = self.load_layer()
 
         # Make the normal map for the layer
         nrm = self.generate_normal_map_for_layer(layer)
 
         # Normal map final file name
-        nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._tv) + "_" + str(self._th) + ".png"
-
-        # Check for existence of normal map file
-        ex = os.path.isfile(nrmfln)
-
-        # Does not exist? Just save
-        if ex == False:
-            nrm.save(nrmfln)
-
-        # Exists? Open it, composite both, save
-        if ex == True:
-            nrmmap = Image.open(nrmfln)
-            nrmmap.alpha_composite(nrm)
-
-            # Specularity blending correction
-            nrmmap_pix = nrmmap.load()
-            for y in range(nrmmap.height):
-                for x in range(nrmmap.width):
-                    c = nrmmap_pix[x,y]
-                    nrmmap_pix[x,y] = (c[0], c[1], c[2], 1)
-            nrmmap.save(nrmfln)
+        #nrmfln = mstr_datafolder + "z_orthographic/normals/" + self._latlngfld + "/" + str(self._tv) + "_" + str(self._th) + ".png"
             
-        mstr_msg("xp_normalmap", "[X-Plane] Normal map saved")
+        mstr_msg("xp_normalmap", "[X-Plane] Normal map generated")
+
+        return nrm

xp_scenery.py

@@ -15,8 +15,10 @@
 import os
 import math
 import urllib.request
+import numpy
 from defines import *
 from log import *
+from PIL import Image, ImageFilter, ImageEnhance
 
 class mstr_xp_scenery:
     # Set required variables
@@ -29,6 +31,11 @@ class mstr_xp_scenery:
         self._vstep = vstep
         self._latlngfld = latlngfld
         self._demfn = self.build_dem_filename()
+        self._dsfstring = ""
+        self._demdata = None  # To be populated when the mesh is built
+        self._demcoord = None # Also to be populated when mesh is built
+        self._waterdata = []  # So that we know where to implement water
+        #self.load_water_data()
 
 
     # Build the correct file name for the elevation model
@@ -60,49 +67,39 @@ class mstr_xp_scenery:
 
         return fn
 
-    # Generate the mesh script for the ortho photos
-    def build_mesh_script(self):
-        scr = mstr_datafolder + "z_orthographic/data/meshscript.txt"
-        # Before we blast all these lines into the file, we need to make sure they do not exist already
-        write_lines = True
 
-        if os.path.isfile(scr) == True:
-            fnlines = []
-            with open(scr) as textfile:
-                fnlines = textfile.readlines()
+    # Load the water data before we generate the mesh
+    def load_water_data(self):
+        fn = mstr_datafolder + "z_orthographic/data/" + self._latlngfld + "/wtrfile"
+        with open(fn) as file:
+            for line in file:
+                ln = line.replace(" ", "_")
+                ln = ln.replace("\n", "")
+                ln = ln.replace("\r", "")
+                self._waterdata.append(ln)
 
-            for line in fnlines:
-                l = line.split(" ")
-                if l[2] == str(self._lng) and l[3] == str(self._lat):
-                    write_lines = False
-                    break
-        else:
-            open(scr, 'a').close()
-        
-        # If we did not find the initial corner coordinate in the script, we can go ahead
-        if write_lines == True:
-            mstr_msg("xp_scenery", "[X-Plane] Writing mesh script file")
-            # We basically run through all tiles and note down the position of the orthos
-            # as needed by X-Plane. 
-            cur_lat = self._lat
-            cur_lng = self._lng
-            for lat in range(1, self._mlat+1):
-                for lng in range(1, self._mlng+1):
-                    # Write the line only if an ortho exists of course.
-                    if os.path.isfile(mstr_datafolder + "z_orthographic/" + self._latlngfld + "/orthos/" + str(lat) + "_" + str(lng) + ".dds" ) == True:
-                        # The '1' after 'ORTHOPHOTO' defines we want water underneath transparent parts of the DDS texture/ortho.
-                        # This ensures that even if the mesh does not include information for there being a water body,
-                        # we will get 100% correct representation of the water bodies.
-                        scrtxt = "ORTHOPHOTO 1 " + str(cur_lng) + " " + str(cur_lat) + " " + str(round(cur_lng+mstr_zl_18, 6)) + " " + str(cur_lat) + " " + str(round(cur_lng+mstr_zl_18, 6)) + " " + str(round(cur_lat+self._vstep, 6)) + " " + str(cur_lng) + " " + str(round(cur_lat+self._vstep, 6)) + " terrain/" + self._latlngfld + "/" + str(lat) + "_" + str(lng) + ".ter\n"
 
-                        with open(scr, 'a') as textfile:
-                            textfile.write(scrtxt)
+    # Check if ortho has water
+    def does_ortho_have_water(self, ortho):
+        wtr = False
+        if ortho in self._waterdata: wtr = True
+        return wtr
+                
 
-                    cur_lng = round(cur_lng + mstr_zl_18, 6)
 
-                cur_lng = self._lng
-                cur_lat = round(cur_lat + self._vstep, 6)
-            mstr_msg("xp_scenery", "[X-Plane] Mesh script completed")
+    # Build the DSF for the ortho photo overlay
+    def build_and_convert_dsf(self):
+        end = self.find_earthnavdata_number()
+        llf = self.xplane_latlng_folder(end)
+        meshtxt = mstr_datafolder + "_cache/mesh_"+self._latlngfld+".txt"
+        cmd = mstr_xp_dsftool + " --text2dsf " + meshtxt + " '" + mstr_datafolder + "z_orthographic/Earth nav data/" + llf + "/" + self._latlngfld + ".dsf'"
+        os.system(cmd)
+
+
+    # Find exact with of longitude
+    def find_width_of_longitude(self, lat):
+        dm = math.cos(math.radians(lat)) * 111.321 # <- 1 deg width at equator in km
+        return round(dm * 1000, 3)
 
 
     # Find the next "by-ten" numbers for the current latitude and longitude
@@ -161,24 +158,6 @@ class mstr_xp_scenery:
         mstr_msg("xp_scenery", "[X-Plane] XES data acquired")
 
 
-    # This builds the entire mesh in one go
-    def build_mesh(self):
-        mstr_msg("xp_scenery", "[X-Plane] Building DSF mesh")
-        end_bt = self.find_earthnavdata_number()
-        btlfn = str(self.xplane_latlng_folder(end_bt))
-        xp_folder = self.xplane_latlng_folder([self._lat, self._lng])
-        scr = mstr_datafolder + "z_orthographic/data/meshscript.txt"
-        wd = mstr_datafolder + "z_orthographic/data"
-        dsf = mstr_datafolder + "z_orthographic/Earth nav data/" + btlfn + "/" + xp_folder
-        xesfn = self.build_dem_filename(True)
-
-        # The main command to build the mesh
-        cmd = mstr_xp_meshtool + " \"" + scr + "\" \"" + mstr_datafolder + "_cache/" + xesfn + "\"" + " \"" + mstr_datafolder + "_cache/" + self._demfn + "\" \"" + wd + "\" \"" + dsf + ".dsf\""
-
-        os.system(cmd)
-
-        mstr_msg("xp_scenery", "[X-Plane] Mesh construction complete")
-
 
     # This generates all .ter files
     def build_ter_files(self):
@@ -188,19 +167,257 @@ class mstr_xp_scenery:
         xp_folder = self.xplane_latlng_folder([self._lat, self._lng])
         for lat in range(1, self._mlat+1):
             for lng in range(1, self._mlng+1):
-                terstr = ""
-                terstr = terstr + "A\n"
-                terstr = terstr + "800\n"
-                terstr = terstr + "TERRAIN\n"
-                terstr = terstr + "\n"
-                terstr = terstr + "BASE_TEX_NOWRAP ../orthos/"+xp_folder+"/"+str(lat)+"_"+str(lng)+".dds\n"
-                if mstr_xp_scn_normalmaps == True:
-                    terstr = terstr + "TEXTURE_NORMAL ../normals/"+xp_folder+"/"+str(lat)+"_"+str(lng)+".dds\n"
 
-                terfln = mstr_datafolder + "z_orthographic/terrain/"+xp_folder+"/"+str(lat)+"_"+str(lng)+".ter"
+                wdt = self.find_width_of_longitude(cur_lat)
+                dmt = wdt * mstr_zl_18
+
+                cnt_x = cur_lat + (self._vstep/2)
+                cnt_y = cur_lng + (mstr_zl_18/2)
+
+                terstr = ""
+                terstr = terstr + "A\r\n"
+                terstr = terstr + "800\r\n"
+                terstr = terstr + "TERRAIN\r\n"
+                terstr = terstr + "\r\n"
+                terstr = terstr + "LOAD_CENTER " + str(cnt_x) + " " + str(cnt_y) + " " + str(dmt) + " 2048\r\n"
+                terstr = terstr + "BASE_TEX_NOWRAP ../../orthos/" + self._latlngfld + "/" + str(lat)+"_"+str(lng)+".dds\r\n"
+                if mstr_xp_scn_normalmaps:
+                    terstr = terstr + "NORMAL_TEX 1.0 ../../normals/" + self._latlngfld + "/" + str(lat)+"_"+str(lng)+".png\r\n"
+
+                terfln = mstr_datafolder + "z_orthographic/terrain/" + self._latlngfld + "/" + str(lat)+"_"+str(lng)+".ter"
 
                 with open(terfln, 'w') as textfile:
                     textfile.write(terstr)
+                
+                cur_lng = round(cur_lng + mstr_zl_18, 6)
+
+            cur_lng = self._lng
+            cur_lat = round(cur_lat + self._vstep, 6)
 
         mstr_msg("xp_scenery", "[X-Plane] Terrain files written")
 
+
+    # This generates the entire terrain mesh
+    def generate_terrain_mesh(self):
+        # Get the DEM model file name, and acquire important info about the data
+        meshfn = mstr_datafolder + "_cache/" + self.build_dem_filename()
+        siz = os.path.getsize(meshfn)
+        dim = int(math.sqrt(siz/2))
+        assert dim*dim*2 == siz, 'Invalid file size'
+        self._demdata = numpy.fromfile(meshfn, numpy.dtype('>i2'), dim*dim).reshape((dim, dim))
+        self._demdata = self._demdata[::-1] # Invert order so that we can start from bottom left
+
+        # We want to achieve perfect stepping for each data point in the DEM.
+        demstep = round( 1 / len(self._demdata), 6)
+
+        # Generate an array which contains only the coordinates
+        self._demcoord = []
+        for r in range(0, len(self._demdata)):
+            row = []
+            for c in range(0, len(self._demdata)):
+                lat = round(self._lat + r * demstep, 6)
+                lng = round(self._lng + c * demstep, 6)
+                crd = [ lat, lng, self._demdata[r][c]]
+                #crd = [ lat, lng ]
+                row.append(crd)
+            self._demcoord.append(row)
+
+        mstr_msg("xp_scenery", "[X-Plane] Populating DSF information file")
+
+        # The complete string to write into the DSF txt file
+        dsf_str = ""
+
+        dsf_str = dsf_str + "PROPERTY sim/west " + str(int(self._lng)) + "\r\n"
+        dsf_str = dsf_str + "PROPERTY sim/east " + str((int(self._lng) + 1)) + "\r\n"
+        dsf_str = dsf_str + "PROPERTY sim/south " + str(int(self._lat)) + "\r\n"
+        dsf_str = dsf_str + "PROPERTY sim/north " + str((int(self._lat) + 1)) + "\r\n"
+        dsf_str = dsf_str + "PROPERTY sim/require_object 0/6\r\n"
+        dsf_str = dsf_str + "PROPERTY planet earth\r\n"
+        dsf_str = dsf_str + "PROPERTY sim/creation_agent Orthographic\r\n"
+        #dsf_str = dsf_str + "TERRAIN_DEF terrain_Water\r\n"
+
+        # The file to be converted into DSF later
+        meshtxt = mstr_datafolder + "_cache/mesh_"+self._latlngfld+".txt"
+
+        with open(meshtxt, 'w') as textfile:
+            textfile.write(dsf_str)
+
+        dsf_str = ""
+
+        # Orthos
+        for lat in range(1, self._mlat+1):
+            for lng in range(1, self._mlng+1):
+                # Write the line only if an ortho exists of course.
+                ddsf = mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(lat) + "_" + str(lng) + ".dds"
+                if os.path.isfile(ddsf):
+                    dsf_str = dsf_str + "TERRAIN_DEF terrain/" + self._latlngfld + "/" + str(lat) + "_" + str(lng) + ".ter\r\n"
+
+        with open(meshtxt, 'a') as textfile:
+            textfile.write(dsf_str)
+        
+
+        # OK. So. Let's build the mesh.
+        
+        # Current patch
+        curpatch = 0
+
+        for lat in range(1, self._mlat+1):
+            for lng in range(1, self._mlng+1):
+
+                # Create the patch only if the matching ortho exists.
+                # This way we make sure that we hit the same order as the .ter files.
+                # We can also detect which lat and lng coord we are on.
+
+                ddsf  = mstr_datafolder + "z_orthographic/orthos/" + self._latlngfld + "/" + str(lat) + "_" + str(lng) + ".dds"
+                if os.path.isfile(ddsf):
+
+                    # Base coords for this ortho
+                    base_lat = self._lat + ((lat-1) * self._vstep)
+                    base_lng = self._lng + ((lng-1) * mstr_zl_18)
+
+                    # Begin a new patch
+                    mstr_msg("xp_scenery", "[X-Plane] Processing ortho patch " + str(curpatch))
+                    with open(meshtxt, 'a') as textfile:
+                        textfile.write("BEGIN_PATCH " + str(curpatch) + " 0.000000 -1.000000 1 7\r\n")
+
+                    # Step for each ortho vertex
+                    odiv = 4
+                    latstep = self._vstep/odiv
+                    lngstep = mstr_zl_18 /odiv
+                    uv_step = 1 / odiv
+
+                    # Generate the ortho tile
+                    for y in range(0,odiv):
+                        for x in range(0,odiv):
+                            # Coordinates
+                            lat_b = round(base_lat + (y*latstep), 6)
+                            lat_t = round(base_lat + ((y+1)*latstep), 6)
+                            lng_l = round(base_lng + (x*lngstep), 6)
+                            lng_r = round(base_lng + ((x+1)*lngstep), 6)
+
+                            # Minimal adjustment
+                            if x == 0:
+                                lng_l = base_lng
+                            if y == 0:
+                                lat_b = base_lat
+                            if y == 3:
+                                lat_t = base_lat + self._vstep
+                            if x == 3:
+                                lng_r = base_lng + mstr_zl_18
+
+                            # Corrections, just in case
+                            if lat_b > self._lat + 1: lat_b = self._lat+1
+                            if lat_t > self._lat + 1: lat_t = self._lat+1
+                            if lng_l > self._lng + 1: lng_l = self._lng+1
+                            if lng_r > self._lng + 1: lng_r = self._lng+1
+
+                            # Height indexes
+                            hgt_bl_idx = self.find_height_for_coord([lat_b, lng_l])
+                            hgt_br_idx = self.find_height_for_coord([lat_b, lng_r])
+                            hgt_tr_idx = self.find_height_for_coord([lat_t, lng_r])
+                            hgt_tl_idx = self.find_height_for_coord([lat_t, lng_l])
+                            hgt_bl = round(self._demcoord[ hgt_bl_idx[0] ][ hgt_bl_idx[1] ][2], 6)
+                            hgt_br = round(self._demcoord[ hgt_br_idx[0] ][ hgt_br_idx[1] ][2], 6)
+                            hgt_tr = round(self._demcoord[ hgt_tr_idx[0] ][ hgt_tr_idx[1] ][2], 6)
+                            hgt_tl = round(self._demcoord[ hgt_tl_idx[0] ][ hgt_tl_idx[1] ][2], 6)
+
+                            # Coords of triangle vertices
+                            # 0 - Longitude
+                            # 1 - Latitude
+                            # 2 - Height in m
+                            t1_v1 = [ lng_r, lat_b, hgt_br ]
+                            t1_v2 = [ lng_l, lat_t, hgt_tl ]
+                            t1_v3 = [ lng_r, lat_t, hgt_tr ]
+                            t2_v1 = [ lng_l, lat_t, hgt_tl ]
+                            t2_v2 = [ lng_r, lat_b, hgt_br ]
+                            t2_v3 = [ lng_l, lat_b, hgt_bl ]
+
+                            # Write down the two triangles
+                            t_str = ""
+                            t_str = t_str + "BEGIN_PRIMITIVE 0\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t1_v1[0]) + " " + str(t1_v1[1]) + " " + str(t1_v1[2]) + " 0.000015 0.000015 " + str((x+1) * uv_step) + " " + str(y*uv_step) + "\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t1_v2[0]) + " " + str(t1_v2[1]) + " " + str(t1_v2[2]) + " 0.000015 0.000015 " + str(x * uv_step) + " " + str((y+1)*uv_step) + "\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t1_v3[0]) + " " + str(t1_v3[1]) + " " + str(t1_v3[2]) + " 0.000015 0.000015 " + str((x+1) * uv_step) + " " + str((y+1)*uv_step) + "\r\n"
+                            t_str = t_str + "END_PRIMITIVE 0\r\n"
+                            t_str = t_str + "BEGIN_PRIMITIVE 0\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t2_v1[0]) + " " + str(t2_v1[1]) + " " + str(t2_v1[2]) + " 0.000015 0.000015 " + str(x * uv_step) + " " + str((y+1)*uv_step) + "\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t2_v2[0]) + " " + str(t2_v2[1]) + " " + str(t2_v2[2]) + " 0.000015 0.000015 " + str((x+1) * uv_step) + " " + str(y*uv_step) + "\r\n"
+                            t_str = t_str + "PATCH_VERTEX " + str(t2_v3[0]) + " " + str(t2_v3[1]) + " " + str(t2_v3[2]) + " 0.000015 0.000015 " + str(x * uv_step) + " " + str(y*uv_step) + "\r\n"
+                            t_str = t_str + "END_PRIMITIVE 0\r\n"
+
+                            # Send to the file
+                            with open(meshtxt, 'a') as textfile:
+                                textfile.write(t_str)
+
+                            t_str = ""
+
+                    
+                    # End this patch
+                    with open(meshtxt, 'a') as textfile:
+                        textfile.write("END PATCH\r\n")
+
+                    # Increase patch number
+                    curpatch = curpatch + 1
+
+
+
+
+    # Find the next best matching height for a point
+    def find_height_for_coord(self, coord):
+        idx = [0,0]
+        dst = 99999
+        ste = self.find_height_scan_start_end_points(coord)
+
+        for r in range(ste[0], ste[1]+1):
+            for d in range(ste[2], ste[3]+1):
+                dist = math.dist(coord, [self._demcoord[r][d][0], self._demcoord[r][d][1]])
+                if dist < dst:
+                    dst = dist
+                    idx = [r,d]
+        return idx
+        
+
+
+    # Find the starting and end points to scan for heights in the DEM grid
+    def find_height_scan_start_end_points(self, stc):
+        startend = [0,0,0,0]
+        stp = 1 / len(self._demdata)
+
+        # Bottom
+        lt = self._lat
+        while lt < stc[0]:
+            lt = lt + stp
+            startend[0] = startend[0] + 1
+
+        # Top
+        lt = self._lat
+        while lt < stc[0]+self._vstep:
+            lt = lt+stp
+            startend[1] = startend[1] + 1
+
+        # Left
+        ln = self._lng
+        while ln < stc[1]:
+            ln = ln + stp
+            startend[2] = startend[2] + 1
+        
+        # Right
+        ln = self._lng
+        while ln < stc[1]+mstr_zl_18:
+            ln = ln + stp
+            startend[3] = startend[3] + 1
+
+        # Make sure we have everything
+        startend[0] = startend[0]-1
+        startend[1] = startend[1]+1
+        startend[2] = startend[2]-1
+        startend[3] = startend[3]+1
+
+        # Some corrections
+        if startend[0] < 0: startend[0] = 0
+        if startend[1] > len(self._demdata)-1: startend[1] = startend[1] = len(self._demdata)-1
+        if startend[2] < 0: startend[2] = 0
+        if startend[3] > len(self._demdata)-1: startend[3] = startend[3] = len(self._demdata)-1
+
+        return startend
+