I am processing more than 5k NetCDF files in ArcPy, using ArcGIS Desktop 10.5. I am doing it using the Python console of ArcMap. The process runs well, but when around 113 files are done, the process breaks down. The ArcMap gives an ERROR(attached at the end).
Does anyone know if there is a limitation with the number of files to process?
The process is:
Read the netCDF files --> arcpy.MakeNetCDFRasterLayer_md
Apply Zonal Statistics as table -->arcpy.gp.ZonalStatisticsAsTable_sa
Export the results to a CSV
The code is:
in_fd="is the netCDF files directory. More than 5k files."
shape= "is the shapefile where the zonal statistics will be applied."
names=[]
path=[]
dic={}
for root,folder,files in os.walk(in_fd):
for file in files:
if os.path.isfile(os.path.join(root,file)) and file.endswith('.nc'):
path.append(os.path.join(root,file))
names.append(os.path.basename(file))
arcpy.env.workspace = r"Z:\EMSV-064_IM_for_EMSN071_ForestFireHazard_Germany\5_Process\3_Task2\1_Data\4_FWI\MIKEL\FWI_mv.gdb"
arcpy.env.overwriteOutput = True
for i in range (0,len(path)):
print("#############")
print("Empieza el: " + names[i])
ras=arcpy.MakeNetCDFRasterLayer_md(in_netCDF_file=path[i], variable="dc", x_dimension="longitude", y_dimension="latitude", out_raster_layer=names[i][:-3], band_dimension="", dimension_values="", value_selection_method="BY_VALUE")
table=arcpy.gp.ZonalStatisticsAsTable_sa(shape, "Id", ras, os.path.join(r"Z:\EMSV-064_IM_for_EMSN071_ForestFireHazard_Germany\5_Process\3_Task2\1_Data\4_FWI\MIKEL\Statistics_DC","id_"+str(names[i][13:21])), "DATA", "ALL")
with arcpy.da.SearchCursor(table,"*") as cursor:
for row in cursor:
dic[names[i][13:21]]=row[4],row[5],row[7],row[8]
ls=[]
vls=[(k,)+v for k,v in dic.items()]
for row in vls:
ls.append(row)
if os.path.exists(r"Z:\EMSV-064_IM_for_EMSN071_ForestFireHazard_Germany\5_Process\3_Task2\1_Data\4_FWI\MIKEL\CSV_mv\FWI_DC.csv"):
os.remove(r"Z:\EMSV-064_IM_for_EMSN071_ForestFireHazard_Germany\5_Process\3_Task2\1_Data\4_FWI\MIKEL\CSV_mv\FWI_DC.csv")
with open(r"Z:\EMSV-064_IM_for_EMSN071_ForestFireHazard_Germany\5_Process\3_Task2\1_Data\4_FWI\MIKEL\CSV_mv\FWI_DC.csv","wb")as out:
csv_out=csv.writer(out)
csv_out.writerow(['Fecha','MIN','MAX','MEAN','STD'])
for row in ls:
csv_out.writerow(row)
arcpy.Delete_management(ras)
arcpy.Delete_management(table)
The error is:
