My PyQGIS script is designed to work on a set of roads, it uses a nested for loop to iterate through each road, prepare a raster based on the line segment's bounds, place points along the line segment, then perform a host of processes on the point, the raster, and the line to produce values that are put in a new shapefile managed as a geopandas geodatabase (I couldn't get QGIS to do it).
My problem is that upon every for-loop iteration, there seems to be an accumulation of RAM in the form of temporary layers that are not cleared after the loop is finished. Making my script fail in about 1.5 hours. It's not really possible for me to use more RAM on this problem, as I might need ~1TB if the script functions the way it does now. Is there any way to explicitly clear all temporary outputs, even those not displayed on the map itself?
Here is what some of my code looks like (abridged so ignore variable name inconsistencies):
def processAlgorithm(self, parameters, context, model_feedback):
# Use a multi-step feedback, so that individual child algorithm progress reports are adjusted for the
# overall progress through the model
results = {}
outputs = {}
road_layer = self.parameterAsVectorLayer(parameters, 'roads', context)
road_gdf = gpd.read_file(parameters['file_name_of_roads'])
feedback = QgsProcessingMultiStepFeedback(road_layer.featureCount(), model_feedback)
i = 0
for feature in road_layer.getFeatures():
i = i + 1
temp = QgsVectorLayer("LineString?crs=epsg:32145", "temp", "memory")
temp_pr = temp.dataProvider()
temp_pr.addAttributes(road_layer.fields())
temp_pr.addFeatures([feature])
# Bounding boxes
alg_params = {
'INPUT': temp, #parameters['roads']
'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
}
outputs['BoundingBoxes'] = processing.run('native:boundingboxes', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
# Points along geometry
alg_params = {
'DISTANCE': 10,
'END_OFFSET': 0,
'INPUT': temp,
'START_OFFSET': 0,
'OUTPUT': 'C://Users//jdahl//Downloads//temp_files//temp_points.shp'
}
outputs['PointsAlongGeometry'] = processing.run('native:pointsalonglines', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
# Clip dem by extent
alg_params = {
'DATA_TYPE': 0, # Use Input Layer Data Type
'EXTRA': '',
'INPUT': parameters['dem'],
'NODATA': None,
'OPTIONS': '',
'OVERCRS': False,
'PROJWIN': outputs['BoundingBoxes']['OUTPUT'],
'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
}
outputs['ClipDemByExtent'] = processing.run('gdal:cliprasterbyextent', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
if os.path.isfile('C://Users//jdahl//Downloads//temp_files//temp_points.shp'):
os.remove('C://Users//jdahl//Downloads//temp_files//temp_points.shp')
points_layer = gpd.read_file('C://Users//jdahl//Downloads//temp_files//temp_points.shp')
min_viz = 2.0
viz_sum = 0.0
for index, row in points_layer.iterrows():
geom = row['geometry']
qgs_point = QgsPointXY(geom.x, geom.y)
point_geom = QgsGeometry.fromPointXY(qgs_point)
point = point_geom.asPoint()
QgsMessageLog.logMessage("point " + str(index) + " started", 'debug')
alg_params = {
'BAND': 1,
'EXTRA': '',
'INPUT': outputs['ClipCompositeRasterByExtent']['OUTPUT'],
'MAX_DISTANCE': 250,
'OBSERVER': point,
'OBSERVER_HEIGHT': 2,
'OPTIONS': '',
'TARGET_HEIGHT': 1,
'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
}
outputs['Viewshed'] = processing.run('gdal:viewshed', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
if os.path.exists('C://Users//jdahl//Downloads//temp_files//visibility.shp'):
os.remove('C://Users//jdahl//Downloads//temp_files//visibility.shp')
# Field calculator
alg_params = {
'FIELD_LENGTH': 0,
'FIELD_NAME': 'lenrat',
'FIELD_PRECISION': 0,
'FIELD_TYPE': 0, # Decimal (double)
'FORMULA': '"lengthviz" / "lengthall"',
'INPUT': outputs['JoinAttributesByFieldValue']['OUTPUT'],
'OUTPUT': 'C://Users//jdahl//Downloads//temp_files//visibility.shp'
}
outputs['FieldCalculator'] = processing.run('native:fieldcalculator', alg_params, context=context, feedback=feedback, is_child_algorithm=True)
viz_val = gpd.read_file('C://Users//jdahl//Downloads//temp_files//visibility.shp').loc[0, 'lenrat']
if viz_val < min_viz:
min_viz = viz_val
viz_sum = viz_sum + viz_val
QgsMessageLog.logMessage("point " + str(index) + " finished", 'debug')
avg_viz = viz_sum / len(points_layer)
if 'min viz' not in road_gdf.columns:
road_gdf['min viz'] = None
if 'avg viz' not in road_gdf.columns:
road_gdf['avg viz'] = None
object_id = feature['OBJECTID']
road_gdf.loc[road_gdf['OBJECTID'] == object_id, 'min viz'] = min_viz
road_gdf.loc[road_gdf['OBJECTID'] == object_id, 'avg viz'] = avg_viz
feedback.setCurrentStep(i)
if feedback.isCanceled():
return {}
road_gdf.to_file(parameters['output'])
return results
