Edit
Since first posting this answer I made pypi package https://pypi.org/project/jupyter-save-load-vars. It can be installed with
pip install jupyter-save-load-vars
Since dill.dump_session fails on any object that cannot be pickled and it apparently cannot be configured to simply ignore such objects, I wrote a pair of static functions savevars(file) and loadvars() that are in this ne1 library for use with our neuromorphic engineering jupyter notebook class chip exercises.
This approach is based on the useful post Get locals from calling namespace in Python
Example of use:
from jupyter_save_load_vars import savevars,loadvars
a=1
b=[2,3]
c='string'
o=(i for i in []) # make generator that cannot be pickled
savevars('testvars')
del a,b,c
loadvars('testvars')
print(a)
print(b)
print(c)
Output:
[INFO]: 2023-10-02 08:28:32,878 - NE1 - saved to testvars.dill variables [ a b c ] (File "/home/ne1/CoACH-labs/ne1.py", line 132, in savevars)
[WARNING]: 2023-10-02 08:28:32,879 - NE1 - could not pickle: ['o'] (File "/home/ne1/CoACH-labs/ne1.py", line 134, in savevars)
[INFO]: 2023-10-02 08:28:32,881 - NE1 - from testvars.dill loaded variables ['a', 'b', 'c'] (File "/home/ne1/CoACH-labs/ne1.py", line 158, in loadvars)
1
[2, 3]
string
For completeness to use the code below, I also put the custom Logger that formats the logging messages. If you don't want this logging output, replace log.XXX( with print(
The savevars(filename) function:
def savevars(filename):
"""
saves all local variables to a file with dill
:param filename: the name of the file. The suffix .dill is added if there is not a suffix already.
"""
if filename is None:
log.error('you must supply a filename')
return
from pathlib import Path
p=Path(filename)
if p.suffix=='': # if suffix is missing add .dill
p = p.parent / (p.name + _DILL)
import inspect,dill
locals=None
frame = inspect.currentframe().f_back
try:
locals = frame.f_locals
finally:
del frame
if locals is None: return
data={}
could_not_pickle=[]
from types import ModuleType
s=f'saved to {p} variables [ '
for k,v in locals.items():
# don't try to pickle any pyplane objects
if k.startswith('_') or k=='tmp' or k=='In' or k=='Out' or hasattr(v, '__call__') \
or isinstance(v,ModuleType) or isinstance(v,logging.Logger):
continue
try:
if not dill.pickles(v):
could_not_pickle.append(k)
continue
except:
could_not_pickle.append(k)
continue
s=s+k+' '
data[k]=v
s=s+']'
try:
with open(p,'wb') as f:
try:
dill.dump(data,f)
log.info(f'{s}')
if len(could_not_pickle)>0:
log.warning(f'could not pickle: {could_not_pickle}')
except TypeError as e:
log.error(f'\n Error: {e}')
except Exception as e:
log.error(f'could not save data to {p}')
The loadvars() function
def loadvars(filename):
""" Loads variables from file into the current workspace
:param filename: the dill file to load from, e.g. lab1. The suffix .dill is added automatically unless there is already a suffix.
This function loads the variables found in filename into the parent workspace.
"""
import dill
from pathlib import Path
p=Path(filename)
if p.suffix=='': # if suffix is missing add .dill
p = p.parent / (p.name + _DILL)
if not p.exists:
log.error(f'{p} does not exist')
return
try:
with open(p,'rb') as f:
data=dill.load(f)
log.info(f'from {p} loaded variables {list(data.keys())}')
import inspect
try:
frame = inspect.currentframe().f_back # get the workspace frame (jupyter workspace frame)
locals = frame.f_locals # get its local variable dict
for k in data:
try:
locals[k] = data[k] # set a value in it
except Exception as e:
log.error(f'could not set variable {k}')
finally:
del frame
except Exception as e:
log.error(f'could not load; got {e}')
The custom Logger with formatting and code line hyperlink that works in pycharm:
import logging
# general logger. Produces nice output format with live hyperlinks for pycharm users
# to use it, just call log=get_logger() at the top of your python file
# all these loggers share the same logger name 'Control_Toolkit'
_LOGGING_LEVEL = logging.DEBUG # usually INFO is good
class CustomFormatter(logging.Formatter):
"""Logging Formatter to add colors and count warning / errors"""
# see https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output/7995762#7995762
# \x1b[ (ESC[) is the CSI introductory sequence for ANSI https://en.wikipedia.org/wiki/ANSI_escape_code
# The control sequence CSI n m, named Select Graphic Rendition (SGR), sets display attributes.
grey = "\x1b[2;37m" # 2 faint, 37 gray
yellow = "\x1b[33;21m"
cyan = "\x1b[0;36m" # 0 normal 36 cyan
green = "\x1b[31;21m" # dark green
red = "\x1b[31;21m" # bold red
bold_red = "\x1b[31;1m"
light_blue = "\x1b[1;36m"
blue = "\x1b[1;34m"
reset = "\x1b[0m"
# File "{file}", line {max(line, 1)}'.replace("\\", "/")
format = '[%(levelname)s]: %(asctime)s - %(name)s - %(message)s (File "%(pathname)s", line %(lineno)d, in %(funcName)s)'
FORMATS = {
logging.DEBUG: grey + format + reset,
logging.INFO: cyan + format + reset,
logging.WARNING: red + format + reset,
logging.ERROR: bold_red + format + reset,
logging.CRITICAL: bold_red + format + reset
}
def format(self, record):
log_fmt = self.FORMATS.get(record.levelno)
formatter = logging.Formatter(log_fmt)
return formatter.format(record).replace("\\", "/") #replace \ with / for pycharm links
def get_logger():
""" Use get_logger to define a logger with useful color output and info and warning turned on according to the global LOGGING_LEVEL.
:returns: the logger.
"""
# logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger('NE1') # tobi changed so all have same name so we can uniformly affect all of them
logger.setLevel(_LOGGING_LEVEL)
# create console handler if this logger does not have handler yet
if len(logger.handlers)==0:
ch = logging.StreamHandler()
ch.setFormatter(CustomFormatter())
logger.addHandler(ch)
return logger
log=get_logger()
