diff --git a/defines.py b/defines.py index 13a76bb..0e5e024 100644 --- a/defines.py +++ b/defines.py @@ -15,11 +15,13 @@ # stored. #mstr_datafolder = "M:/Developer/Projects/orthographic/" mstr_datafolder = "D:/Developer/Projects/orthographic/" +#mstr_datafolder = "/home/marcus/Developer/Projects/orthographic/" # Switched to Linux, so path is amended # API endpoint to acquire OSM data (bonus: I have my own) -mstr_osm_endpoint = "https://marstr.online/osm/v1/" +#mstr_osm_endpoint = "https://marstr.online/osm/v1/" +mstr_osm_endpoint = "http://localhost/og.php" # Define the texture resolution you want to have your photos at. mstr_photores = 2048 @@ -100,7 +102,7 @@ mstr_xp_normal_maps = [ # How much of a tile we need for each zoom level. The higher # the zoom level, the smaller the area to generate a mask of - but also # higher detail. -mstr_zl_16 = 0.08 +mstr_zl_16 = 0.064 mstr_zl_17 = 0.048 mstr_zl_18 = 0.016 mstr_zl_19 = 0.008 diff --git a/maskgen.py b/maskgen.py index b8080c2..12f1061 100644 --- a/maskgen.py +++ b/maskgen.py @@ -37,6 +37,7 @@ class mstr_maskgen: self._vstep = vstep self._scale = 1 / math.cos(math.radians(self._box[0])) self._isline = isline + self._zoomlevel = mstr_zl_16 #mstr_msg("maskgen", "Intialized mask gen.") @@ -98,8 +99,8 @@ class mstr_maskgen: bbox.append(self._box[0] + ((self._box[1]-1) * self._vstep)) bbox.append(self._box[0] + ((self._box[1]-1) * self._vstep) + self._vstep) # Longitude - bbox.append(self._box[2] + ((self._box[3]-1) * mstr_zl_18)) - bbox.append(self._box[2] + ((self._box[3]-1) * mstr_zl_18) + mstr_zl_18) + bbox.append(self._box[2] + ((self._box[3]-1) * self._zoomlevel)) + bbox.append(self._box[2] + ((self._box[3]-1) * self._zoomlevel) + self._zoomlevel) # Building levels, if this is a building bld_levels = 0 @@ -213,7 +214,7 @@ class mstr_maskgen: # of the building in this loop. if mstr_shadow_enabled == True and is_prep == False: if self._tag == "building": - mpp = meters_per_pixel(self._tile_width) * mstr_zl_18 + mpp = meters_per_pixel(self._tile_width) * self._zoomlevel pix_per_floor = mstr_shadow_floor_h / mpp total_pix = pix_per_floor * bld_levels shift = int(total_pix) diff --git a/orthographic.py b/orthographic.py index defe6af..929bf95 100644 --- a/orthographic.py +++ b/orthographic.py @@ -30,6 +30,7 @@ class mstr_orthographic: # Constructor of class. Takes longitude and latitude. def __init__(self, lat, lng, outfolder, pwd, prep=False): + self._zoomlevel = mstr_zl_16 self._lat = lat self._long = lng self._output = outfolder @@ -68,7 +69,7 @@ class mstr_orthographic: # orthos can only be a power of 2, such as 2048x2048 def _findVerticalStepping(self): scale = 1 / math.cos(math.radians(self._lat)) - maxlat = (1 / scale) * mstr_zl_18 + maxlat = (1 / scale) * self._zoomlevel return maxlat @@ -90,7 +91,7 @@ class mstr_orthographic: bb_lat = self._lat bb_lng = self._long bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 + bb_lng_edge = self._long+self._zoomlevel cur_tile_x = 1 cur_tile_y = 1 osmxml = mstr_osmxml(0,0) @@ -109,7 +110,7 @@ class mstr_orthographic: bb_lat = bb_lat + self._vstep mlat = mlat+1 while bb_lng < self._long + 1: - bb_lng = bb_lng + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel mlng = mlng+1 mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) maxlatlng = [ mlat, mlng ] @@ -130,8 +131,8 @@ class mstr_orthographic: # Adjust longitude coordinates cur_tile_x = cur_tile_x+1 - bb_lng = bb_lng + mstr_zl_18 - bb_lng_edge = bb_lng_edge + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel + bb_lng_edge = bb_lng_edge + self._zoomlevel mstr_msg("orthographic", "Adjustment of longitude performed") # Adjust peak longitude tile number if cur_tile_x > top_lng: @@ -141,7 +142,7 @@ class mstr_orthographic: cur_tile_y = cur_tile_y+1 cur_tile_x = 1 bb_lng = self._long - bb_lng_edge = self._long + mstr_zl_18 + bb_lng_edge = self._long + self._zoomlevel bb_lat = bb_lat + self._vstep bb_lat_edge = bb_lat_edge + self._vstep mstr_msg("orthographic", "Adjustment of latitude performed") @@ -159,14 +160,14 @@ class mstr_orthographic: bb_lat = self._lat bb_lng = self._long bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 + bb_lng_edge = self._long+self._zoomlevel mlat = 1 mlng = 1 while bb_lat < self._lat + 1: bb_lat = bb_lat + self._vstep mlat = mlat+1 while bb_lng < self._long + 1: - bb_lng = bb_lng + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel mlng = mlng+1 mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) maxlatlng = [ mlat, mlng ] @@ -197,7 +198,7 @@ class mstr_orthographic: bb_lat = self._lat bb_lng = self._long bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 + bb_lng_edge = self._long+self._zoomlevel # We need to know the highest possible latitude and longitude tile numbers, # in case we render at the edge @@ -207,7 +208,7 @@ class mstr_orthographic: bb_lat = bb_lat + self._vstep mlat = mlat+1 while bb_lng < self._long + 1: - bb_lng = bb_lng + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel mlng = mlng+1 mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) maxlatlng = [ mlat, mlng ] @@ -217,9 +218,9 @@ class mstr_orthographic: if os.path.isfile(ddsf) == False: # Reset these two bb_lat = self._lat + ((grid_lat-1)*self._vstep) - bb_lng = self._long + ((grid_lng-1)*mstr_zl_18) + bb_lng = self._long + ((grid_lng-1)*self._zoomlevel) bb_lat_edge = self._lat + ((grid_lat-1)*self._vstep) + self._vstep - bb_lng_edge = self._long + ((grid_lng-1)*mstr_zl_18) + mstr_zl_18 + bb_lng_edge = self._long + ((grid_lng-1)*self._zoomlevel) + self._zoomlevel osmxml = mstr_osmxml() osmxml.adjust_bbox(bb_lat, bb_lng, bb_lat_edge, bb_lng_edge) @@ -355,7 +356,7 @@ class mstr_orthographic: bb_lat = self._lat bb_lng = self._long bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 + bb_lng_edge = self._long+self._zoomlevel cur_tile_x = 1 cur_tile_y = 1 #osmxml = mstr_osmxml(0,0) @@ -374,7 +375,7 @@ class mstr_orthographic: bb_lat = bb_lat + self._vstep mlat = mlat+1 while bb_lng < self._long + 1: - bb_lng = bb_lng + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel mlng = mlng+1 mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) maxlatlng = [ mlat, mlng ] @@ -387,8 +388,8 @@ class mstr_orthographic: contours = [] # Generate images for all resources - cnt_w = mlng * 20 - cnt_h = mlat * 20 + cnt_w = mlng * 80 + cnt_h = mlat * 80 for l in mstr_ortho_layers: if l[0] != "building": cntimg = Image.new("RGBA", (cnt_w, cnt_h), (255,255,255,255)) @@ -401,6 +402,11 @@ class mstr_orthographic: # stress on OSM servers and my server, we will do acquire the data # only for the current processed part of the tile. + # Total number of tiles to be generated + total_tiles = mlng * mlat + nextstep = 20 + mstr_important_msg("orthographic", "Generating map data required to build orthos") + for lat_grid in range(1, maxlatlng[0]+1): for lng_grid in range(1, maxlatlng[1]+1): # Adjust bounding box @@ -422,7 +428,7 @@ class mstr_orthographic: mg = mstr_maskgen( [self._lat, cur_tile_y, self._long, cur_tile_x], self._vstep, layer[0], layer[1], layer[2]) mask = mg._build_mask(osmxml, is_prep=True) # We need an object here - mask = mask.resize((20,20), Image.Resampling.BILINEAR) + mask = mask.resize((80,80), Image.Resampling.BILINEAR) idx = 0 for c in contours: if c[0] == layer[0] and c[1] == layer[1]: @@ -430,8 +436,8 @@ class mstr_orthographic: else: idx=idx+1 - cnx = (lng_grid-1) * 20 - cny = cnt_h - (lat_grid * 20) + cnx = (lng_grid-1) * 80 + cny = cnt_h - (lat_grid * 80) contours[idx][2].alpha_composite(mask, dest=(cnx,cny)) #tp = mstr_tileprep(self._lat, self._long, lat_grid, lng_grid, layer[0], layer[1], mask, False) @@ -441,18 +447,25 @@ class mstr_orthographic: # Adjust longitude coordinates cur_tile_x = cur_tile_x+1 - bb_lng = bb_lng + mstr_zl_18 - bb_lng_edge = bb_lng_edge + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel + bb_lng_edge = bb_lng_edge + self._zoomlevel mstr_msg("orthographic", "Adjustment of longitude performed") # Adjust peak longitude tile number if cur_tile_x > top_lng: top_lng = cur_tile_x + totaldata = glob.glob("D:/Developer/webserver/htdocs/server/osm/*.xml") + progress = int( (len(totaldata) / total_tiles) * 100 ) + + if progress >= nextstep: + mstr_important_msg("orthographic", str(progress) + " percent complete") + nextstep = nextstep + 20 + # Adjust latitude and all other values when we get here cur_tile_y = cur_tile_y+1 cur_tile_x = 1 bb_lng = self._long - bb_lng_edge = self._long + mstr_zl_18 + bb_lng_edge = self._long + self._zoomlevel bb_lat = bb_lat + self._vstep bb_lat_edge = bb_lat_edge + self._vstep mstr_msg("orthographic", "Adjustment of latitude performed") @@ -526,12 +539,12 @@ class mstr_orthographic: bb_lat = self._lat bb_lng = self._long bb_lat_edge = self._lat+self._vstep - bb_lng_edge = self._long+mstr_zl_18 + bb_lng_edge = self._long+self._zoomlevel while bb_lat < self._lat + 1: bb_lat = bb_lat + self._vstep mlat = mlat+1 while bb_lng < self._long + 1: - bb_lng = bb_lng + mstr_zl_18 + bb_lng = bb_lng + self._zoomlevel mlng = mlng+1 mstr_msg("orthographic", "Max lat tile: " + str(mlat) + " - max lng tile: " + str(mlng)) maxlatlng = [ mlat, mlng ] diff --git a/osmxml.py b/osmxml.py index 85ebdca..7c4c150 100644 --- a/osmxml.py +++ b/osmxml.py @@ -63,42 +63,56 @@ class mstr_osmxml: # Acquire XMLs in chunks, then store them def acquire_osm(self, v, h): mstr_msg("osmxml", "Acquiring OSM data for " + str(self._lat)+","+str(self._lng)+" - "+str(self._curB_lat)+","+str(self._curB_lng)) - - # We will use our self-hosted API for this. - parse = False - while parse == False: - data = { - "bbox": { - "lat": str(self._lat), - "lng": str(self._lng), - "lat_b": str(self._curB_lat), - "lng_b": str(self._curB_lng) - }, - "tile_lat": str(self._lat), - "tile_lng": str(self._lng), - "square_lat": str(v), - "square_lng": str(h) - } - r = requests.post(mstr_osm_endpoint, json=data) - try: - # Attempt to parse the XML string - dom = xml.dom.minidom.parseString(r.content) + # We switched to a local storage model to ease load on the server. We will check if we already have the data we need. + # If not, go an acquire it. + fn = mstr_datafolder + "z_orthographic/data/+++++/osm/" + str(self._lat) + "-" + str(v) + "_" + str(self._lng) + "-" + str(h) + ".xml" + if os.path.isfile(fn) == False: + # We will use our self-hosted API for this. + parse = False + while parse == False: + data = { + "bbox": { + "lat": str(self._lat), + "lng": str(self._lng), + "lat_b": str(self._curB_lat), + "lng_b": str(self._curB_lng) + }, + "tile_lat": str(self._lat), + "tile_lng": str(self._lng), + "square_lat": str(v), + "square_lng": str(h) + } + r = requests.post(mstr_osm_endpoint, json=data) - # Check if the DOM object has a document element - if dom.documentElement: - # Store the content in memory - self._xmlcontent = r.content - self._xmldata = xml.dom.minidom.parseString(self._xmlcontent) - self._xmlcontent = "" # Clear - parse = True + try: + # Attempt to parse the XML string + dom = xml.dom.minidom.parseString(r.content) - except ExpatError as e: - parse = False - time.sleep(1) - except Exception as e: - parse = False - time.sleep(1) + # Check if the DOM object has a document element + if dom.documentElement: + # Store the content in memory + self._xmlcontent = r.content + self._xmldata = xml.dom.minidom.parseString(self._xmlcontent) + self._xmlcontent = "" # Clear + parse = True + + except ExpatError as e: + parse = False + time.sleep(1) + except Exception as e: + parse = False + time.sleep(1) + else: + # Attempt to parse the XML string + dom = xml.dom.minidom.parse(fn) + + # Check if the DOM object has a document element + if dom.documentElement: + # Store the content in memory + self._xmlcontent = r.content + self._xmldata = xml.dom.minidom.parseString(self._xmlcontent) + self._xmlcontent = "" # Clear # Get all nodes from the specified OSM file