feat: add psagentai agent with deployment and configuration

Initial project setup including the main agent script, configuration
files, profile JSON state, systemd service unit, startup script,
deployment documentation, and Python dependencies.

Includes .env, venv/, and __pycache__/ which should likely be
excluded via .gitignore to avoid committing secrets and artifacts.
This commit is contained in:
2026-10-25 17:05:13 -03:00
commit 6b3b6541a7
3792 changed files with 805563 additions and 0 deletions
@@ -0,0 +1,38 @@
from ._monitor import TMonitor, TqdmSynchronisationWarning
from ._tqdm_pandas import tqdm_pandas
from .cli import main # TODO: remove in v5.0.0
from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0
from .gui import trange as tgrange # TODO: remove in v5.0.0
from .std import (
TqdmDeprecationWarning, TqdmExperimentalWarning, TqdmKeyError, TqdmMonitorWarning,
TqdmTypeError, TqdmWarning, tqdm, trange)
from .version import __version__
__all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas',
'tqdm_notebook', 'tnrange', 'main', 'TMonitor',
'TqdmTypeError', 'TqdmKeyError',
'TqdmWarning', 'TqdmDeprecationWarning',
'TqdmExperimentalWarning',
'TqdmMonitorWarning', 'TqdmSynchronisationWarning',
'__version__']
def tqdm_notebook(*args, **kwargs): # pragma: no cover
"""See tqdm.notebook.tqdm for full documentation"""
from warnings import warn
from .notebook import tqdm as _tqdm_notebook
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`",
TqdmDeprecationWarning, stacklevel=2)
return _tqdm_notebook(*args, **kwargs)
def tnrange(*args, **kwargs): # pragma: no cover
"""Shortcut for `tqdm.notebook.tqdm(range(*args), **kwargs)`."""
from warnings import warn
from .notebook import trange as _tnrange
warn("Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`",
TqdmDeprecationWarning, stacklevel=2)
return _tnrange(*args, **kwargs)
@@ -0,0 +1,3 @@
from .cli import main
main()
@@ -0,0 +1,9 @@
from warnings import warn
from .cli import * # NOQA
from .cli import __all__ # NOQA
from .std import TqdmDeprecationWarning
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.cli.*` instead of `tqdm._main.*`",
TqdmDeprecationWarning, stacklevel=2)
@@ -0,0 +1,104 @@
import atexit
from threading import Event, Thread, current_thread
from time import time
from warnings import warn
__all__ = ["TMonitor", "TqdmSynchronisationWarning"]
class TqdmSynchronisationWarning(RuntimeWarning):
"""
tqdm multi-thread/-process errors which may cause incorrect nesting
but otherwise no adverse effects
"""
class TMonitor(Thread):
"""
Monitoring thread for tqdm bars.
Monitors if tqdm bars are taking too much time to display
and readjusts miniters automatically if necessary.
Parameters
----------
tqdm_cls : class
tqdm class to use (can be core tqdm or a submodule).
sleep_interval : float
Time to sleep between monitoring checks.
"""
_test = {} # internal vars for unit testing
def __init__(self, tqdm_cls, sleep_interval):
Thread.__init__(self, name="tqdm_monitor")
self.daemon = True # kill thread when main killed (KeyboardInterrupt)
self.woken = 0 # last time woken up, to sync with monitor
self.tqdm_cls = tqdm_cls
self.sleep_interval = sleep_interval
self._time = self._test.get("time", time)
self.was_killed = self._test.get("Event", Event)()
atexit.register(self._atexit_signal)
self.start()
def _atexit_signal(self):
"""
Non-joining shutdown signal.
Avoids deadlocks at interpreter exit from other threads, dead forks, etc.
This daemon thread is auto-reaped on shutdown without needing a join.
"""
self.was_killed.set()
def exit(self):
self.was_killed.set()
if self is not current_thread():
self.join()
return self.report()
def get_instances(self):
# returns a copy of started `tqdm_cls` instances
return [i for i in self.tqdm_cls._instances.copy()
# Avoid race by checking that the instance started
if hasattr(i, 'start_t')]
def run(self):
cur_t = self._time()
while True:
# After processing and before sleeping, notify that we woke
# Need to be done just before sleeping
self.woken = cur_t
# Sleep some time...
self.was_killed.wait(self.sleep_interval)
# Quit if killed
if self.was_killed.is_set():
return
# Then monitor!
# Acquire lock (to access _instances)
with self.tqdm_cls.get_lock():
cur_t = self._time()
# Check tqdm instances are waiting too long to print
instances = self.get_instances()
for instance in instances:
# Check event in loop to reduce blocking time on exit
if self.was_killed.is_set():
return
# Only if mininterval > 1 (else iterations are just slow)
# and last refresh exceeded maxinterval
if (
instance.miniters > 1
and (cur_t - instance.last_print_t) >= instance.maxinterval
):
# force bypassing miniters on next iteration
# (dynamic_miniters adjusts mininterval automatically)
instance.miniters = 1
# Refresh now! (works only for manual tqdm)
instance.refresh(nolock=True)
# Remove accidental long-lived strong reference
del instance
if instances != self.get_instances(): # pragma: nocover
warn("Set changed size during iteration" +
" (see https://github.com/tqdm/tqdm/issues/481)",
TqdmSynchronisationWarning, stacklevel=2)
# Remove accidental long-lived strong references
del instances
def report(self):
return not self.was_killed.is_set()
@@ -0,0 +1,9 @@
from warnings import warn
from .std import * # NOQA
from .std import __all__ # NOQA
from .std import TqdmDeprecationWarning
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.std.*` instead of `tqdm._tqdm.*`",
TqdmDeprecationWarning, stacklevel=2)
@@ -0,0 +1,9 @@
from warnings import warn
from .gui import * # NOQA
from .gui import __all__ # NOQA
from .std import TqdmDeprecationWarning
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.gui.*` instead of `tqdm._tqdm_gui.*`",
TqdmDeprecationWarning, stacklevel=2)
@@ -0,0 +1,9 @@
from warnings import warn
from .notebook import * # NOQA
from .notebook import __all__ # NOQA
from .std import TqdmDeprecationWarning
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`",
TqdmDeprecationWarning, stacklevel=2)
@@ -0,0 +1,24 @@
import sys
__author__ = "github.com/casperdcl"
__all__ = ['tqdm_pandas']
def tqdm_pandas(tclass, **tqdm_kwargs):
"""
Registers the given `tqdm` instance with
`pandas.core.groupby.DataFrameGroupBy.progress_apply`.
"""
from tqdm import TqdmDeprecationWarning
if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith(
'tqdm_')): # delayed adapter case
TqdmDeprecationWarning(
"Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`.",
fp_write=getattr(tqdm_kwargs.get('file', None), 'write', sys.stderr.write))
tclass.pandas(**tqdm_kwargs)
else:
TqdmDeprecationWarning(
"Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`.",
fp_write=getattr(tclass.fp, 'write', sys.stderr.write))
type(tclass).pandas(deprecated_t=tclass)
@@ -0,0 +1,11 @@
from warnings import warn
from .std import TqdmDeprecationWarning
from .utils import ( # noqa: F401, pylint: disable=unused-import
CUR_OS, IS_NIX, IS_WIN, RE_ANSI, Comparable, FormatReplace, SimpleTextIOWrapper,
_environ_cols_wrapper, _is_ascii, _is_utf, _screen_shape_linux, _screen_shape_tput,
_screen_shape_windows, _screen_shape_wrapper, _supports_unicode, _term_move_up, colorama)
warn("This function will be removed in tqdm==5.0.0\n"
"Please use `tqdm.utils.*` instead of `tqdm._utils.*`",
TqdmDeprecationWarning, stacklevel=2)
@@ -0,0 +1,103 @@
"""
Asynchronous progress bar decorator for iterators.
Includes a default `range` iterator printing to `stderr`.
Usage:
>>> from tqdm.asyncio import trange, tqdm
>>> async for i in trange(10):
... ...
"""
import asyncio
from sys import version_info
from .std import tqdm as std_tqdm
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange']
class tqdm_asyncio(std_tqdm):
"""
Asynchronous-friendly version of tqdm.
"""
def __init__(self, iterable=None, *args, **kwargs):
super().__init__(iterable, *args, **kwargs)
self.iterable_awaitable = False
if iterable is not None:
if hasattr(iterable, "__anext__"):
self.iterable_next = iterable.__anext__
self.iterable_awaitable = True
elif hasattr(iterable, "__aiter__"):
self.iterable_iterator = iterable.__aiter__()
self.iterable_next = self.iterable_iterator.__anext__
self.iterable_awaitable = True
elif hasattr(iterable, "__next__"):
self.iterable_next = iterable.__next__
else:
self.iterable_iterator = iter(iterable)
self.iterable_next = self.iterable_iterator.__next__
def __aiter__(self):
return self
async def __anext__(self):
try:
if self.iterable_awaitable:
res = await self.iterable_next()
else:
res = self.iterable_next()
self.update()
return res
except StopIteration:
self.close()
raise StopAsyncIteration
except BaseException:
self.close()
raise
def send(self, *args, **kwargs):
return self.iterable.send(*args, **kwargs)
@classmethod
def as_completed(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs):
"""
Wrapper for `asyncio.as_completed`.
"""
if total is None:
total = len(fs)
kwargs = {}
if version_info[:2] < (3, 10):
kwargs['loop'] = loop
yield from cls(asyncio.as_completed(fs, timeout=timeout, **kwargs),
total=total, **tqdm_kwargs)
@classmethod
async def gather(cls, *fs, loop=None, timeout=None, total=None, return_exceptions=False,
**tqdm_kwargs):
"""
Wrapper for `asyncio.gather`.
"""
async def wrap_awaitable(i, f):
try:
return i, await f
except Exception as e: # noqa: BLE001
if return_exceptions:
return i, e
raise
ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)]
res = [await f for f in cls.as_completed(ifs, loop=loop, timeout=timeout,
total=total, **tqdm_kwargs)]
return [i for _, i in sorted(res)]
def tarange(*args, **kwargs):
"""
A shortcut for `tqdm.asyncio.tqdm(range(*args), **kwargs)`.
"""
return tqdm_asyncio(range(*args), **kwargs)
# Aliases
tqdm = tqdm_asyncio
trange = tarange
@@ -0,0 +1,40 @@
"""
Enables multiple commonly used features.
Method resolution order:
- `tqdm.autonotebook` without import warnings
- `tqdm.asyncio`
- `tqdm.std` base class
Usage:
>>> from tqdm.auto import trange, tqdm
>>> for i in trange(10):
... ...
"""
import warnings
from .std import TqdmExperimentalWarning
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=TqdmExperimentalWarning)
from .autonotebook import tqdm as notebook_tqdm
from .asyncio import tqdm as asyncio_tqdm
from .std import tqdm as std_tqdm
if notebook_tqdm != std_tqdm:
class tqdm(notebook_tqdm, asyncio_tqdm): # pylint: disable=inconsistent-mro
pass
else:
tqdm = asyncio_tqdm
def trange(*args, **kwargs):
"""
A shortcut for `tqdm.auto.tqdm(range(*args), **kwargs)`.
"""
return tqdm(range(*args), **kwargs)
__all__ = ["tqdm", "trange"]
@@ -0,0 +1,38 @@
"""
Automatically choose between `tqdm.notebook` and `tqdm.std`.
Usage:
>>> from tqdm.autonotebook import trange, tqdm
>>> for i in trange(10):
... ...
"""
import os
import sys
from warnings import warn
try:
if 'ipykernel.zmqshell' in sys.modules:
if any(i == 'QT_API' or i.startswith('SPYDER') for i in os.environ):
raise ImportError("console") # jupyter-qtconsole/spyder
ipy = sys.modules['IPython'].get_ipython().__class__.__name__.lower()
if 'qt' in ipy or 'spyder' in ipy:
raise ImportError("console") # older jupyter-qtconsole/spyder
# jupyter-notebook/jupyterlab/vscode/binder/colab
elif 'IPython.utils._process_emscripten' in sys.modules:
pass # jupyterlite (pyodide)/jupyterlite-xeus
else:
raise ImportError("console") # ipython/jupyter-console
from .notebook import WARN_NOIPYW, IProgress
if IProgress is None:
from .std import TqdmWarning
warn(WARN_NOIPYW, TqdmWarning, stacklevel=2)
raise ImportError('ipywidgets')
except Exception:
from .std import tqdm, trange
else: # pragma: no cover
from .notebook import tqdm, trange
from .std import TqdmExperimentalWarning
warn("Using `tqdm.autonotebook.tqdm` in notebook mode."
" Use `tqdm.tqdm` instead to force console mode"
" (e.g. in jupyter console)", TqdmExperimentalWarning, stacklevel=2)
__all__ = ["tqdm", "trange"]
@@ -0,0 +1,324 @@
"""
Module version for monitoring CLI pipes (`... | python -m tqdm | ...`).
"""
import logging
import re
import sys
from ast import literal_eval
from textwrap import indent
from .std import TqdmKeyError, TqdmTypeError, tqdm
from .version import __version__
__all__ = ["main"]
log = logging.getLogger(__name__)
def cast(val, typ):
log.debug((val, typ))
if " or " in typ:
for t in typ.split(" or "):
try:
return cast(val, t)
except TqdmTypeError:
pass
raise TqdmTypeError(f"{val} : {typ}")
# sys.stderr.write('\ndebug | `val:type`: `' + val + ':' + typ + '`.\n')
if typ == 'bool':
if (val == 'True') or (val == ''):
return True
if val == 'False':
return False
raise TqdmTypeError(val + ' : ' + typ)
if typ == 'chr':
if len(val) == 1:
return val.encode()
if re.match(r"^\\\w+$", val):
return literal_eval(f'"{val}"').encode()
raise TqdmTypeError(f"{val} : {typ}")
if typ == 'str':
return val
if typ == 'int':
try:
return int(val)
except ValueError as exc:
raise TqdmTypeError(f"{val} : {typ}") from exc
if typ == 'float':
try:
return float(val)
except ValueError as exc:
raise TqdmTypeError(f"{val} : {typ}") from exc
raise TqdmTypeError(f"{val} : {typ}")
def posix_pipe(fin, fout, delim=b'\\n', buf_size=256,
callback=lambda float: None, callback_len=True):
"""
Params
------
fin : binary file with `read(buf_size : int)` method
fout : binary file with `write` (and optionally `flush`) methods.
callback : function(float), e.g.: `tqdm.update`
callback_len : If (default: True) do `callback(len(buffer))`.
Otherwise, do `callback(data) for data in buffer.split(delim)`.
"""
fp_write = fout.write
if not delim:
while True:
tmp = fin.read(buf_size)
# flush at EOF
if not tmp:
getattr(fout, 'flush', lambda: None)()
return
fp_write(tmp)
callback(len(tmp))
# return
buf = b''
len_delim = len(delim)
# n = 0
while True:
tmp = fin.read(buf_size)
# flush at EOF
if not tmp:
if buf:
fp_write(buf)
if callback_len:
# n += 1 + buf.count(delim)
callback(1 + buf.count(delim))
else:
for i in buf.split(delim):
callback(i)
getattr(fout, 'flush', lambda: None)()
return # n
while True:
i = tmp.find(delim)
if i < 0:
buf += tmp
break
fp_write(buf + tmp[:i + len(delim)])
# n += 1
callback(1 if callback_len else (buf + tmp[:i]))
buf = b''
tmp = tmp[i + len_delim:]
# ((opt, type), ... )
RE_OPTS = re.compile(r'\n {4}(\S+)\s{2,}:\s*([^,]+)')
# better split method assuming no positional args
RE_SHLEX = re.compile(r'\s*(?<!\S)--?([^\s=]+)(\s+|=|$)')
# TODO: add custom support for some of the following?
UNSUPPORTED_OPTS = ('iterable', 'gui', 'out', 'file')
# The 8 leading spaces are required for consistency
CLI_EXTRA_DOC = r"""
Extra CLI Options
-----------------
name : type, optional
TODO: find out why this is needed.
delim : chr, optional
Delimiting character [default: '\n']. Use '\0' for null.
N.B.: on Windows systems, Python converts '\n' to '\r\n'.
buf_size : int, optional
String buffer size in bytes [default: 256]
used when `delim` is specified.
bytes : bool, optional
If true, will count bytes, ignore `delim`, and default
`unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'.
tee : bool, optional
If true, passes `stdin` to both `stderr` and `stdout`.
update : bool, optional
If true, will treat input as newly elapsed iterations,
i.e. numbers to pass to `update()`. Note that this is slow
(~2e5 it/s) since every input must be decoded as a number.
update_to : bool, optional
If true, will treat input as total elapsed iterations,
i.e. numbers to assign to `self.n`. Note that this is slow
(~2e5 it/s) since every input must be decoded as a number.
null : bool, optional
If true, will discard input (no stdout).
manpath : str, optional
Directory in which to install tqdm man pages.
comppath : str, optional
Directory in which to place tqdm completion.
log : str, optional
CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET.
"""
def main(fp=sys.stderr, argv=None):
"""
Parameters (internal use only)
---------
fp : file-like object for tqdm
argv : list (default: sys.argv[1:])
"""
if argv is None:
argv = sys.argv[1:]
try:
log_idx = argv.index('--log')
except ValueError:
for i in argv:
if i.startswith('--log='):
logLevel = i[len('--log='):]
break
else:
logLevel = 'INFO'
else:
# argv.pop(log_idx)
# logLevel = argv.pop(log_idx)
logLevel = argv[log_idx + 1]
logging.basicConfig(level=getattr(logging, logLevel),
format="%(levelname)s:%(module)s:%(lineno)d:%(message)s")
# py<3.13 doesn't dedent docstrings
d = (tqdm.__doc__ if sys.version_info < (3, 13)
else indent(tqdm.__doc__, " ")) + CLI_EXTRA_DOC
opt_types = dict(RE_OPTS.findall(d))
# opt_types['delim'] = 'chr'
for o in UNSUPPORTED_OPTS:
opt_types.pop(o)
log.debug(sorted(opt_types.items()))
# d = RE_OPTS.sub(r' --\1=<\1> : \2', d)
split = RE_OPTS.split(d)
opt_types_desc = zip(split[1::3], split[2::3], split[3::3])
d = ''.join(('\n --{0} : {2}{3}' if otd[1] == 'bool' else
'\n --{0}=<{1}> : {2}{3}').format(
otd[0].replace('_', '-'), otd[0], *otd[1:])
for otd in opt_types_desc if otd[0] not in UNSUPPORTED_OPTS)
help_short = "Usage:\n tqdm [--help | options]\n"
d = help_short + """
Options:
-h, --help Print this help and exit.
-v, --version Print version and exit.
""" + d.strip('\n') + '\n'
# opts = docopt(d, version=__version__)
if any(v in argv for v in ('-v', '--version')):
sys.stdout.write(__version__ + '\n')
sys.exit(0)
elif any(v in argv for v in ('-h', '--help')):
sys.stdout.write(d + '\n')
sys.exit(0)
elif argv and argv[0][:2] != '--':
sys.stderr.write(f"Error:Unknown argument:{argv[0]}\n{help_short}")
argv = RE_SHLEX.split(' '.join(["tqdm"] + argv))
opts = dict(zip(argv[1::3], argv[3::3]))
log.debug(opts)
opts.pop('log', True)
tqdm_args = {'file': fp}
try:
for (o, v) in opts.items():
o = o.replace('-', '_')
try:
tqdm_args[o] = cast(v, opt_types[o])
except KeyError as e:
raise TqdmKeyError(str(e))
log.debug('args:' + str(tqdm_args))
delim_per_char = tqdm_args.pop('bytes', False)
update = tqdm_args.pop('update', False)
update_to = tqdm_args.pop('update_to', False)
if sum((delim_per_char, update, update_to)) > 1:
raise TqdmKeyError("Can only have one of --bytes --update --update_to")
except Exception:
fp.write("\nError:\n" + help_short)
stdin, stdout_write = sys.stdin, sys.stdout.write
for i in stdin:
stdout_write(i)
raise
else:
buf_size = tqdm_args.pop('buf_size', 256)
delim = tqdm_args.pop('delim', b'\\n')
tee = tqdm_args.pop('tee', False)
manpath = tqdm_args.pop('manpath', None)
comppath = tqdm_args.pop('comppath', None)
if tqdm_args.pop('null', False):
class stdout:
@staticmethod
def write(_):
pass
else:
stdout = sys.stdout
stdout = getattr(stdout, 'buffer', stdout)
stdin = getattr(sys.stdin, 'buffer', sys.stdin)
if manpath or comppath:
try: # py<3.9
import importlib_resources as resources
except ImportError:
from importlib import resources
from pathlib import Path
def cp(name, dst):
"""copy resource `name` to `dst`"""
fi = resources.files('tqdm') / name
dst.write_bytes(fi.read_bytes())
log.info("written:%s", dst)
if manpath is not None:
cp('tqdm.1', Path(manpath) / 'tqdm.1')
if comppath is not None:
cp('completion.sh', Path(comppath) / 'tqdm_completion.sh')
sys.exit(0)
if tee:
stdout_write = stdout.write
fp_write = getattr(fp, 'buffer', fp).write
class stdout: # pylint: disable=function-redefined
@staticmethod
def write(x):
with tqdm.external_write_mode(file=fp):
fp_write(x)
stdout_write(x)
if delim_per_char:
tqdm_args.setdefault('unit', 'B')
tqdm_args.setdefault('unit_scale', True)
tqdm_args.setdefault('unit_divisor', 1024)
log.debug(tqdm_args)
with tqdm(**tqdm_args) as t:
posix_pipe(stdin, stdout, '', buf_size, t.update)
elif delim == b'\\n':
log.debug(tqdm_args)
write = stdout.write
if update or update_to:
with tqdm(**tqdm_args) as t:
if update:
def callback(i):
t.update(literal_eval(i.decode()))
else: # update_to
def callback(i):
t.update(literal_eval(i.decode()) - t.n)
for i in stdin:
write(i)
callback(i)
else:
for i in tqdm(stdin, **tqdm_args):
write(i)
else:
log.debug(tqdm_args)
with tqdm(**tqdm_args) as t:
callback_len = False
if update:
def callback(i):
t.update(literal_eval(i.decode()))
elif update_to:
def callback(i):
t.update(literal_eval(i.decode()) - t.n)
else:
callback = t.update
callback_len = True
posix_pipe(stdin, stdout, delim, buf_size, callback, callback_len)
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
_tqdm(){
local cur prv
cur="${COMP_WORDS[COMP_CWORD]}"
prv="${COMP_WORDS[COMP_CWORD - 1]}"
case ${prv} in
--bar_format|--buf_size|--colour|--comppath|--delay|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor)
# await user input
;;
"--log")
COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur}))
;;
*)
COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --colour --comppath --delay --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --null --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --update --update_to --version --write_bytes -h -v' -- ${cur}))
;;
esac
}
complete -F _tqdm tqdm
@@ -0,0 +1,91 @@
"""
Thin wrappers around common functions.
Subpackages contain potentially unstable extensions.
"""
from warnings import warn
from ..auto import tqdm as tqdm_auto
from ..std import TqdmDeprecationWarning, tqdm
from ..utils import ObjectWrapper
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['tenumerate', 'tzip', 'tmap']
class DummyTqdmFile(ObjectWrapper):
"""Dummy file-like that will write to tqdm"""
def __init__(self, wrapped):
super().__init__(wrapped)
self._buf = []
def write(self, x, nolock=False):
nl = b"\n" if isinstance(x, bytes) else "\n"
pre, sep, post = x.rpartition(nl)
if sep:
blank = type(nl)()
tqdm.write(blank.join(self._buf + [pre, sep]),
end=blank, file=self._wrapped, nolock=nolock)
self._buf = [post]
else:
self._buf.append(x)
def __del__(self):
if self._buf:
blank = type(self._buf[0])()
try:
tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped)
except (OSError, ValueError):
pass
def builtin_iterable(func):
"""Returns `func`"""
warn("This function has no effect, and will be removed in tqdm==5.0.0",
TqdmDeprecationWarning, stacklevel=2)
return func
def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs):
"""
Equivalent of `numpy.ndenumerate` or builtin `enumerate`.
Parameters
----------
tqdm_class : [default: tqdm.auto.tqdm].
"""
try:
import numpy as np
except ImportError:
pass
else:
if isinstance(iterable, np.ndarray):
return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size,
**tqdm_kwargs)
return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start)
def tzip(iter1, *iter2plus, **tqdm_kwargs):
"""
Equivalent of builtin `zip`.
Parameters
----------
tqdm_class : [default: tqdm.auto.tqdm].
"""
kwargs = tqdm_kwargs.copy()
tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
yield from zip(tqdm_class(iter1, **kwargs), *iter2plus)
def tmap(function, *sequences, **tqdm_kwargs):
"""
Equivalent of builtin `map`.
Parameters
----------
tqdm_class : [default: tqdm.auto.tqdm].
"""
for i in tzip(*sequences, **tqdm_kwargs):
yield function(*i)
@@ -0,0 +1,28 @@
"""
Even more features than `tqdm.auto` (all the bells & whistles):
- `tqdm.auto`
- `tqdm.tqdm.pandas`
- `tqdm.contrib.slack`
+ uses `${TQDM_SLACK_TOKEN}` and `${TQDM_SLACK_CHANNEL}`
- `tqdm.contrib.telegram`
+ uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}`
- `tqdm.contrib.discord`
+ uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}`
"""
__all__ = ['tqdm', 'trange']
import warnings
from os import getenv
if getenv("TQDM_SLACK_TOKEN") and getenv("TQDM_SLACK_CHANNEL"):
from .slack import tqdm, trange
elif getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"):
from .telegram import tqdm, trange
elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"):
from .discord import tqdm, trange
else:
from ..auto import tqdm, trange
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=FutureWarning)
tqdm.pandas()
@@ -0,0 +1,259 @@
"""
Thin wrappers around `concurrent.futures`.
"""
import sys
from contextlib import contextmanager
from operator import length_hint
from ..auto import tqdm as tqdm_auto
from ..std import TqdmWarning
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['thread_map', 'process_map', 'interpreter_map']
class _InterpreterLock:
"""Reentrant lock backed by a cross-interpreter queue."""
from threading import get_ident
from time import monotonic as _time
def __init__(self, queue):
from threading import RLock
self._queue = queue
self._lock = RLock()
self._owner = None
self._depth = 0
def acquire(self, blocking=True, timeout=-1):
from queue import Empty
start = self._time()
if timeout == -1:
acquired = self._lock.acquire(blocking)
else:
acquired = self._lock.acquire(blocking, timeout)
if not acquired:
return False
if self._depth:
self._depth += 1
return True
try:
if not blocking:
self._queue.get_nowait()
elif timeout == -1:
self._queue.get()
else:
remaining = max(0, timeout - (self._time() - start))
self._queue.get(timeout=remaining)
except Empty:
self._lock.release()
return False
self._owner = self.get_ident()
self._depth = 1
return True
def release(self):
if self._owner != self.get_ident():
raise RuntimeError("cannot release un-acquired lock")
self._depth -= 1
if not self._depth:
self._owner = None
self._queue.put(None)
self._lock.release()
def __enter__(self):
self.acquire()
return self
def __exit__(self, *exc):
self.release()
@contextmanager
def ensure_lock(tqdm_class, lock_name="", lock=None):
"""get (create if necessary) and then restore `tqdm_class`'s lock"""
old_lock = getattr(tqdm_class, '_lock', None) # don't create a new lock
if lock is None:
lock = old_lock or tqdm_class.get_lock() # maybe create a new lock
lock = getattr(lock, lock_name, lock) # maybe subtype
tqdm_class.set_lock(lock)
yield lock
if old_lock is None:
del tqdm_class._lock
else:
tqdm_class.set_lock(old_lock)
def _get_interpreter_init(tqdm_class, lock_queue_id):
"""Return an initializer which bootstraps the parent import path and lock."""
code = (
"import sys\n"
f"sys.path[:] = {sys.path!r}\n"
"from concurrent import interpreters\n"
"from importlib import import_module\n"
"from tqdm.contrib.concurrent import _InterpreterLock\n"
f"tqdm_class = import_module({tqdm_class.__module__!r})\n"
f"for name in {tqdm_class.__qualname__.split('.')!r}:\n"
" tqdm_class = getattr(tqdm_class, name)\n"
"tqdm_class.monitor_interval = 0\n"
f"tqdm_class.set_lock(_InterpreterLock(interpreters.Queue({lock_queue_id!r})))")
return exec, (code,)
def _min_map_len(iterables):
"""min(map(length_hint, iterables), default=0)"""
return min((n for it in iterables if (n := length_hint(it, -1)) >= 0), default=0)
def _executor_map(
PoolExecutor, fn, *iterables, max_workers=None, timeout=None, chunksize=1, lock_name="",
tqdm_class=tqdm_auto, smoothing=0.0, _lock=None, _initializer=None, _initargs=None,
**tqdm_kwargs
):
"""
Implementation of `thread_map`, `process_map` and `interpreter_map`.
Parameters
----------
max_workers : int
timeout : int
buffersize : int
Requires Python>=3.14.
thread_name_prefix : str
max_tasks_per_child : int
mp_context : str
"""
kwargs = tqdm_kwargs.copy()
if 'total' not in kwargs:
kwargs['total'] = _min_map_len(iterables)
map_kwargs = {}
if 'buffersize' in kwargs:
map_kwargs['buffersize'] = kwargs.pop('buffersize')
pool_kwargs = {}
for k in ('thread_name_prefix', 'max_tasks_per_child', 'mp_context'):
if k in kwargs:
pool_kwargs[k] = kwargs.pop(k)
dynamic_miniters = None
if kwargs['total'] and 'miniters' not in kwargs:
try:
from os import process_cpu_count as cpu_count
except ImportError:
from os import cpu_count
# thread & process pools have different default workers, but we KISS here
rough_max = max_workers or min(32, (cpu_count() or 1) + 4)
if kwargs['total'] > rough_max:
kwargs['miniters'] = rough_max
dynamic_miniters = True
with ensure_lock(tqdm_class, lock_name=lock_name, lock=_lock) as lk:
# share lock in case workers are already using `tqdm`
if _initializer is None:
_initializer = tqdm_class.set_lock
_initargs = (lk,)
with PoolExecutor(max_workers=max_workers, initializer=_initializer, initargs=_initargs,
**pool_kwargs) as ex:
with tqdm_class(smoothing=smoothing, **kwargs) as pbar:
if dynamic_miniters is not None:
pbar.dynamic_miniters = True
orisubmit = ex.submit
def patchsubmit(*args, **kwargs):
fut = orisubmit(*args, **kwargs)
fut.add_done_callback(lambda _: pbar.update())
return fut
ex.submit = patchsubmit
return list(ex.map(
fn, *iterables, timeout=timeout, chunksize=chunksize, **map_kwargs))
def thread_map(fn, *iterables, **tqdm_kwargs):
"""
Equivalent of `list(map(fn, *iterables))`
driven by `concurrent.futures.ThreadPoolExecutor`.
Parameters
----------
max_workers : int, optional
Maximum number of workers to spawn; passed to `concurrent.futures.ThreadPoolExecutor`.
thread_name_prefix : str, optional
Passed to `concurrent.futures.ThreadPoolExecutor` [default: ''].
timeout : int or float, optional
Seconds to wait before raising `TimeoutError` if `__next__` is called and the
result isn't available. [default: None].
buffersize : int, optional
Requires Python>=3.14 [default: None].
tqdm_class : optional
`tqdm` class to use for bars [default: tqdm.auto.tqdm].
smoothing : float, optional
Passed to `tqdm_class`; the [default: 0] is average (due to erratic update frequency).
lock_name : str, optional
Member of `tqdm_class.get_lock()` to use [default: ''].
"""
from concurrent.futures import ThreadPoolExecutor
return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
def interpreter_map(fn, *iterables, **tqdm_kwargs):
"""
Equivalent of `list(map(fn, *iterables))`
driven by `concurrent.futures.InterpreterPoolExecutor` (Python 3.14+).
Parameters
----------
Same as `thread_map`.
Notes
-----
`fn`, its arguments, and its return values must be pickleable.
Worker progress bars using the same `tqdm_class` share a cross-interpreter write lock.
"""
from concurrent import interpreters
from concurrent.futures import InterpreterPoolExecutor
lock_queue = interpreters.create_queue()
lock_queue.put(None)
tqdm_class = tqdm_kwargs.get("tqdm_class", tqdm_auto)
initializer, initargs = _get_interpreter_init(tqdm_class, lock_queue.id)
return _executor_map(
InterpreterPoolExecutor, fn, *iterables, _lock=_InterpreterLock(lock_queue),
_initializer=initializer, _initargs=initargs, **tqdm_kwargs)
def process_map(fn, *iterables, lock_name="mp_lock", **tqdm_kwargs):
"""
Equivalent of `list(map(fn, *iterables))`
driven by `concurrent.futures.ProcessPoolExecutor`.
Parameters
----------
max_workers : int, optional
Maximum number of workers to spawn; passed to `concurrent.futures.ProcessPoolExecutor`.
timeout : int or float, optional
Seconds to wait before raising `TimeoutError` if `__next__` is called and the
result isn't available. [default: None].
chunksize : int, optional
Approximate size of chunks sent to worker processes; passed to
`concurrent.futures.ProcessPoolExecutor.map`. [default: 1].
buffersize : int, optional
Requires Python>=3.14 [default: None].
max_tasks_per_child : int, optional
Maximum number of tasks a worker process can complete before being replaced
with a new process; passed to `concurrent.futures.ProcessPoolExecutor`.
mp_context : multiprocessing.BaseContext, optional
Multiprocessing context to use, e.g. `multiprocessing.get_context('fork')`.
lock_name : str, optional
Member of `tqdm_class.get_lock()` to use [default: mp_lock].
tqdm_class : optional
`tqdm` class to use for bars [default: tqdm.auto.tqdm].
smoothing : float, optional
Passed to `tqdm_class`; the [default: 0] is average (due to erratic update frequency).
"""
from concurrent.futures import ProcessPoolExecutor
if iterables and 'chunksize' not in tqdm_kwargs:
# default `chunksize=1` has poor performance for large iterables
# (most time spent dispatching items to workers).
shortest_iterable_len = _min_map_len(iterables)
if shortest_iterable_len > 1000:
from warnings import warn
warn("Iterable length %d > 1000 but `chunksize` is not set."
" This may seriously degrade multiprocess performance."
" Set `chunksize=1` or more." % shortest_iterable_len,
TqdmWarning, stacklevel=2)
return _executor_map(ProcessPoolExecutor, fn, *iterables, lock_name=lock_name, **tqdm_kwargs)
@@ -0,0 +1,155 @@
"""
Sends updates to a Discord bot.
Usage:
>>> from tqdm.contrib.discord import tqdm, trange
>>> for i in trange(10, token='{token}', channel_id='{channel_id}'):
... ...
![screenshot](https://tqdm.github.io/img/screenshot-discord.png)
"""
from warnings import warn
from requests import Session
from requests.utils import default_user_agent
from ..auto import tqdm as tqdm_auto
from ..std import TqdmWarning
from ..utils import envwrap
from ..version import __version__
from .utils_worker import MonoWorker
__author__ = {"github.com/": ["casperdcl", "guigoruiz1"]}
__all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange']
class DiscordIO(MonoWorker):
"""Non-blocking file-like IO using a Discord Bot."""
API = 'https://discord.com/api/v10'
UA = f"tqdm (https://tqdm.github.io, {__version__}) {default_user_agent()}"
def __init__(self, token, channel_id):
"""Creates a new message in the given `channel_id`."""
super().__init__()
self.token = token
self.channel_id = channel_id
self.session = Session()
self.text = self.__class__.__name__
self.message_id # pylint: disable=pointless-statement
@property
def message_id(self):
if hasattr(self, '_message_id'):
return self._message_id # pylint: disable=access-member-before-definition
try:
req = self.session.post(
f'{self.API}/channels/{self.channel_id}/messages',
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA},
json={'content': f"`{self.text}`"})
res = req.json()
req.raise_for_status()
except Exception as e:
if req.status_code == 429:
warn("Creation rate limit: try increasing `mininterval`.",
TqdmWarning, stacklevel=2)
else:
tqdm_auto.write(str(e))
else:
self._message_id = res['id']
return self._message_id
def write(self, s):
"""Replaces internal `message_id`'s text with `s`."""
if not s:
s = "..."
s = s.replace('\r', '').strip()
if s == self.text:
return # avoid duplicate message Bot error
message_id = self.message_id
if message_id is None:
return
self.text = s
try:
future = self.submit(
self.session.patch,
f'{self.API}/channels/{self.channel_id}/messages/{message_id}',
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA},
json={'content': f"`{self.text}`"})
except Exception as e:
tqdm_auto.write(str(e))
else:
return future
def delete(self):
"""Deletes internal `message_id`."""
try:
future = self.submit(
self.session.delete,
f'{self.API}/channels/{self.channel_id}/messages/{self.message_id}',
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA})
except Exception as e:
tqdm_auto.write(str(e))
else:
return future
class tqdm_discord(tqdm_auto): # pylint: disable=inconsistent-mro
"""
Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot.
May take a few seconds to create (`__init__`).
- create a discord bot (not public, no requirement of OAuth2 code
grant, only send message permissions) & invite it to a channel:
<https://discordpy.readthedocs.io/en/latest/discord.html>
- copy the bot `{token}` & `{channel_id}` and paste below
>>> from tqdm.contrib.discord import tqdm, trange
>>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'):
... ...
"""
@envwrap("tqdm", "discord", is_method=True)
def __init__(self, *args, token=None, channel_id=None, **kwargs):
"""
Parameters
----------
token : str, required. Discord bot token
[default: ${TQDM_DISCORD_TOKEN}].
channel_id : int, required. Discord channel ID
[default: ${TQDM_DISCORD_CHANNEL_ID}].
See `tqdm.auto.tqdm.__init__` for other parameters.
"""
if not kwargs.get('disable'):
kwargs = kwargs.copy()
self.dio = DiscordIO(token, channel_id)
super().__init__(*args, **kwargs)
def display(self, **kwargs): # pylint: disable=arguments-differ
super().display(**kwargs)
fmt = self.format_dict
if fmt.get('bar_format', None):
fmt['bar_format'] = fmt['bar_format'].replace(
'<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}')
self.dio.write(self.format_meter(**fmt))
def clear(self, *args, **kwargs):
super().clear(*args, **kwargs)
if not self.disable:
self.dio.write("")
def close(self):
if self.disable:
return
super().close()
if not (self.leave or (self.leave is None and self.pos == 0)):
self.dio.delete()
def tdrange(*args, **kwargs):
"""Shortcut for `tqdm.contrib.discord.tqdm(range(*args), **kwargs)`."""
return tqdm_discord(range(*args), **kwargs)
# Aliases
tqdm = tqdm_discord
trange = tdrange
@@ -0,0 +1,92 @@
"""
Thin wrappers around `itertools`.
"""
import itertools
import math
from ..auto import tqdm as tqdm_auto
__author__ = {"github.com/": ["casperdcl"]}
__all__ = [
'chain', 'product', 'permutations', 'combinations', 'combinations_with_replacement', 'batched']
def chain(*iterables, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.chain`."""
if total is None:
try:
total = sum(map(len, iterables))
except (TypeError, AttributeError):
pass
return tqdm_class(itertools.chain(*iterables), total=total, **kwargs)
def product(*iterables, repeat=1, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.product`."""
if total is None:
try:
lens = list(map(len, iterables))
except (TypeError, AttributeError):
pass
else:
total = math.prod(lens) ** repeat
yield from tqdm_class(itertools.product(*iterables, repeat=repeat), total=total, **kwargs)
def permutations(iterable, r=None, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.permutations`."""
if total is None:
try:
n = len(iterable)
except (TypeError, AttributeError):
pass
else:
r = n if r is None else r
if r > n:
total = 0
else:
total = math.perm(n, r)
return tqdm_class(itertools.permutations(iterable, r), total=total, **kwargs)
def combinations(iterable, r, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.combinations`."""
if total is None:
try:
n = len(iterable)
except (TypeError, AttributeError):
pass
else:
if r > n:
total = 0
else:
total = math.comb(n, r)
return tqdm_class(itertools.combinations(iterable, r), total=total, **kwargs)
def combinations_with_replacement(iterable, r, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.combinations_with_replacement`."""
if total is None:
try:
n = len(iterable)
except (TypeError, AttributeError):
pass
else:
total = 1
for i in range(n+r-1, n-1, -1):
total *= i
for i in range(1, r+1):
total //= i
return tqdm_class(itertools.combinations_with_replacement(iterable, r), total=total, **kwargs)
def batched(iterable, n, total=None, tqdm_class=tqdm_auto, **kwargs):
"""Equivalent of `itertools.batched`."""
if total is None:
try:
total = len(iterable)
except (TypeError, AttributeError):
pass
return tqdm_class(itertools.batched(iterable, n), unit_scale=n,
total=(total+n-1) // n if total is not None else None,
**kwargs)
@@ -0,0 +1,128 @@
"""
Helper functionality for interoperability with stdlib `logging`.
"""
import logging
import sys
from contextlib import contextmanager
try:
from typing import Iterator, List, Optional, Type # noqa: F401, pylint: disable=unused-import
except ImportError:
pass
from ..std import tqdm as std_tqdm
class _TqdmLoggingHandler(logging.StreamHandler):
def __init__(
self,
tqdm_class=std_tqdm # type: Type[std_tqdm]
):
super().__init__()
self.tqdm_class = tqdm_class
def emit(self, record):
try:
msg = self.format(record)
self.tqdm_class.write(msg, file=self.stream)
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except: # noqa pylint: disable=bare-except
self.handleError(record)
def _is_console_logging_handler(handler):
return (isinstance(handler, logging.StreamHandler)
and handler.stream in {sys.stdout, sys.stderr})
def _get_first_found_console_logging_handler(handlers):
for handler in handlers:
if _is_console_logging_handler(handler):
return handler
@contextmanager
def logging_redirect_tqdm(
loggers=None, # type: Optional[List[logging.Logger]],
tqdm_class=std_tqdm # type: Type[std_tqdm]
):
# type: (...) -> Iterator[None]
"""
Context manager redirecting console logging to `tqdm.write()`, leaving
other logging handlers (e.g. log files) unaffected.
Parameters
----------
loggers : list, optional
Which handlers to redirect (default: [logging.root]).
tqdm_class : optional
Example
-------
```python
import logging
from tqdm import trange
from tqdm.contrib.logging import logging_redirect_tqdm
LOG = logging.getLogger(__name__)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
with logging_redirect_tqdm():
for i in trange(9):
if i == 4:
LOG.info("console logging redirected to `tqdm.write()`")
# logging restored
```
"""
if loggers is None:
loggers = [logging.root]
original_handlers_list = [logger.handlers for logger in loggers]
try:
for logger in loggers:
tqdm_handler = _TqdmLoggingHandler(tqdm_class)
orig_handler = _get_first_found_console_logging_handler(logger.handlers)
if orig_handler is not None:
tqdm_handler.setFormatter(orig_handler.formatter)
tqdm_handler.stream = orig_handler.stream
for log_filter in orig_handler.filters:
tqdm_handler.addFilter(log_filter)
logger.handlers = [
handler for handler in logger.handlers
if not _is_console_logging_handler(handler)] + [tqdm_handler]
yield
finally:
for logger, original_handlers in zip(loggers, original_handlers_list):
logger.handlers = original_handlers
@contextmanager
def tqdm_logging_redirect(
*args,
# loggers=None, # type: Optional[List[logging.Logger]]
# tqdm=None, # type: Optional[Type[tqdm.tqdm]]
**kwargs
):
# type: (...) -> Iterator[None]
"""
Convenience shortcut for:
```python
with tqdm_class(*args, **tqdm_kwargs) as pbar:
with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class):
yield pbar
```
Parameters
----------
tqdm_class : optional, (default: tqdm.std.tqdm).
loggers : optional, list.
**tqdm_kwargs : passed to `tqdm_class`.
"""
tqdm_kwargs = kwargs.copy()
loggers = tqdm_kwargs.pop('loggers', None)
tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm)
with tqdm_class(*args, **tqdm_kwargs) as pbar:
with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class):
yield pbar
@@ -0,0 +1,119 @@
"""
Sends updates to a Slack app.
Usage:
>>> from tqdm.contrib.slack import tqdm, trange
>>> for i in trange(10, token='{token}', channel='{channel}'):
... ...
![screenshot](https://tqdm.github.io/img/screenshot-slack.png)
"""
import logging
try:
from slack_sdk import WebClient
except ImportError:
raise ImportError("Please `pip install slack-sdk`")
from ..auto import tqdm as tqdm_auto
from ..utils import envwrap
from .utils_worker import MonoWorker
__author__ = {"github.com/": ["0x2b3bfa0", "casperdcl"]}
__all__ = ['SlackIO', 'tqdm_slack', 'tsrange', 'tqdm', 'trange']
class SlackIO(MonoWorker):
"""Non-blocking file-like IO using a Slack app."""
def __init__(self, token, channel):
"""Creates a new message in the given `channel`."""
super().__init__()
self.client = WebClient(token=token)
self.text = self.__class__.__name__
try:
self.message = self.client.chat_postMessage(channel=channel, text=self.text)
except Exception as e:
tqdm_auto.write(str(e))
self.message = None
def write(self, s):
"""Replaces internal `message`'s text with `s`."""
if not s:
s = "..."
s = s.replace('\r', '').strip()
if s == self.text:
return # skip duplicate message
message = self.message
if message is None:
return
self.text = s
try:
future = self.submit(self.client.chat_update, channel=message['channel'],
ts=message['ts'], text='`' + s + '`')
except Exception as e:
tqdm_auto.write(str(e))
else:
return future
class tqdm_slack(tqdm_auto): # pylint: disable=inconsistent-mro
"""
Standard `tqdm.auto.tqdm` but also sends updates to a Slack app.
May take a few seconds to create (`__init__`).
- create a Slack app with the `chat:write` scope & invite it to a
channel: <https://api.slack.com/authentication/basics>
- copy the bot `{token}` & `{channel}` and paste below
>>> from tqdm.contrib.slack import tqdm, trange
>>> for i in tqdm(iterable, token='{token}', channel='{channel}'):
... ...
"""
@envwrap("tqdm", "slack", is_method=True)
def __init__(self, *args, token=None, channel=None, **kwargs):
"""
Parameters
----------
token : str, required. Slack token
[default: ${TQDM_SLACK_TOKEN}].
channel : int, required. Slack channel
[default: ${TQDM_SLACK_CHANNEL}].
mininterval : float, optional.
Minimum of [default: 1.5] to avoid rate limit.
See `tqdm.auto.tqdm.__init__` for other parameters.
"""
if not kwargs.get('disable'):
kwargs = kwargs.copy()
logging.getLogger("HTTPClient").setLevel(logging.WARNING)
self.sio = SlackIO(token, channel)
kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5))
super().__init__(*args, **kwargs)
def display(self, **kwargs): # pylint: disable=arguments-differ
super().display(**kwargs)
fmt = self.format_dict
if fmt.get('bar_format', None):
fmt['bar_format'] = fmt['bar_format'].replace(
'<bar/>', '`{bar:10}`').replace('{bar}', '`{bar:10u}`')
elif self.total:
fmt['bar_format'] = '{l_bar}`{bar:10}`{r_bar}'
if fmt['ascii'] is False:
fmt['ascii'] = [":black_square:", ":small_blue_diamond:", ":large_blue_diamond:",
":large_blue_square:"]
fmt['ncols'] = 336
self.sio.write(self.format_meter(**fmt))
def clear(self, *args, **kwargs):
super().clear(*args, **kwargs)
if not self.disable:
self.sio.write("")
def tsrange(*args, **kwargs):
"""Shortcut for `tqdm.contrib.slack.tqdm(range(*args), **kwargs)`."""
return tqdm_slack(range(*args), **kwargs)
# Aliases
tqdm = tqdm_slack
trange = tsrange
@@ -0,0 +1,152 @@
"""
Sends updates to a Telegram bot.
Usage:
>>> from tqdm.contrib.telegram import tqdm, trange
>>> for i in trange(10, token='{token}', chat_id='{chat_id}'):
... ...
![screenshot](https://tqdm.github.io/img/screenshot-telegram.gif)
"""
from warnings import warn
from requests import Session
from ..auto import tqdm as tqdm_auto
from ..std import TqdmWarning
from ..utils import envwrap
from .utils_worker import MonoWorker
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange']
class TelegramIO(MonoWorker):
"""Non-blocking file-like IO using a Telegram Bot."""
API = 'https://api.telegram.org/bot'
def __init__(self, token, chat_id):
"""Creates a new message in the given `chat_id`."""
super().__init__()
self.token = token
self.chat_id = chat_id
self.session = Session()
self.text = self.__class__.__name__
self.message_id # pylint: disable=pointless-statement
@property
def message_id(self):
if hasattr(self, '_message_id'):
return self._message_id # pylint: disable=access-member-before-definition
try:
req = self.session.post(
f'{self.API}{self.token}/sendMessage',
data={'text': f"`{self.text}`", 'chat_id': self.chat_id,
'parse_mode': 'MarkdownV2'})
res = req.json()
req.raise_for_status()
except Exception as e:
if req.status_code == 429:
warn("Creation rate limit: try increasing `mininterval`.",
TqdmWarning, stacklevel=2)
else:
tqdm_auto.write(str(e))
else:
self._message_id = res['result']['message_id']
return self._message_id
def write(self, s):
"""Replaces internal `message_id`'s text with `s`."""
if not s:
s = "..."
s = s.replace('\r', '').strip()
if s == self.text:
return # avoid duplicate message Bot error
message_id = self.message_id
if message_id is None:
return
self.text = s
try:
future = self.submit(
self.session.post, f'{self.API}{self.token}/editMessageText',
data={'text': f"`{s}`", 'chat_id': self.chat_id,
'message_id': message_id, 'parse_mode': 'MarkdownV2'})
except Exception as e:
tqdm_auto.write(str(e))
else:
return future
def delete(self):
"""Deletes internal `message_id`."""
try:
future = self.submit(
self.session.post, '{self.API}{self.token}/deleteMessage',
data={'chat_id': self.chat_id, 'message_id': self.message_id})
except Exception as e:
tqdm_auto.write(str(e))
else:
return future
class tqdm_telegram(tqdm_auto): # pylint: disable=inconsistent-mro
"""
Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot.
May take a few seconds to create (`__init__`).
- create a bot <https://core.telegram.org/bots#6-botfather>
- copy its `{token}`
- add the bot to a chat and send it a message such as `/start`
- go to <https://api.telegram.org/bot`{token}`/getUpdates> to find out
the `{chat_id}`
- paste the `{token}` & `{chat_id}` below
>>> from tqdm.contrib.telegram import tqdm, trange
>>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'):
... ...
"""
@envwrap("tqdm", "telegram", is_method=True)
def __init__(self, *args, token=None, chat_id=None, **kwargs):
"""
Parameters
----------
token : str, required. Telegram token
[default: ${TQDM_TELEGRAM_TOKEN}].
chat_id : str, required. Telegram chat ID
[default: ${TQDM_TELEGRAM_CHAT_ID}].
See `tqdm.auto.tqdm.__init__` for other parameters.
"""
if not kwargs.get('disable'):
kwargs = kwargs.copy()
self.tgio = TelegramIO(token, chat_id)
super().__init__(*args, **kwargs)
def display(self, **kwargs): # pylint: disable=arguments-differ
super().display(**kwargs)
fmt = self.format_dict
if fmt.get('bar_format', None):
fmt['bar_format'] = fmt['bar_format'].replace(
'<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}')
self.tgio.write(self.format_meter(**fmt))
def clear(self, *args, **kwargs):
super().clear(*args, **kwargs)
if not self.disable:
self.tgio.write("")
def close(self):
if self.disable:
return
super().close()
if not (self.leave or (self.leave is None and self.pos == 0)):
self.tgio.delete()
def ttgrange(*args, **kwargs):
"""Shortcut for `tqdm.contrib.telegram.tqdm(range(*args), **kwargs)`."""
return tqdm_telegram(range(*args), **kwargs)
# Aliases
tqdm = tqdm_telegram
trange = ttgrange
@@ -0,0 +1,38 @@
"""
IO/concurrency helpers for `tqdm.contrib`.
"""
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from ..auto import tqdm as tqdm_auto
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['MonoWorker']
class MonoWorker:
"""
Supports one running task and one waiting task.
The waiting task is the most recent submitted (others are discarded).
"""
def __init__(self):
self.pool = ThreadPoolExecutor(max_workers=1)
self.futures = deque([], 2)
def submit(self, func, *args, **kwargs):
"""`func(*args, **kwargs)` may replace currently waiting task."""
futures = self.futures
if len(futures) == futures.maxlen:
running = futures.popleft()
if not running.done():
if len(futures): # clear waiting
waiting = futures.pop()
waiting.cancel()
futures.appendleft(running) # re-insert running
try:
waiting = self.pool.submit(func, *args, **kwargs)
except Exception as e:
tqdm_auto.write(str(e))
else:
futures.append(waiting)
return waiting
@@ -0,0 +1,44 @@
from functools import partial
from dask.callbacks import Callback
from .auto import tqdm as tqdm_auto
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['TqdmCallback']
class TqdmCallback(Callback):
"""Dask callback for task progress."""
def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto,
**tqdm_kwargs):
"""
Parameters
----------
tqdm_class : optional
`tqdm` class to use for bars [default: `tqdm.auto.tqdm`].
tqdm_kwargs : optional
Any other arguments used for all bars.
"""
super().__init__(start=start, pretask=pretask)
if tqdm_kwargs:
tqdm_class = partial(tqdm_class, **tqdm_kwargs)
self.tqdm_class = tqdm_class
def _start_state(self, _, state):
self.pbar = self.tqdm_class(total=sum(
len(state[k]) for k in ['ready', 'waiting', 'running', 'finished']))
def _posttask(self, *_, **__):
self.pbar.update()
def _finish(self, *_, **__):
self.pbar.close()
def display(self):
"""Displays in the current cell in Notebooks."""
container = getattr(self.bar, 'container', None)
if container is None:
return
from .notebook import display
display(container)
@@ -0,0 +1,179 @@
"""
Matplotlib GUI progress bar decorator for iterators.
Usage:
>>> from tqdm.gui import trange, tqdm
>>> for i in trange(10):
... ...
"""
# future division is important to divide integers and get as
# a result precise floating numbers (instead of truncated int)
import re
from warnings import warn
# to inherit from the tqdm class
from .std import TqdmExperimentalWarning
from .std import tqdm as std_tqdm
# import compatibility functions and utilities
__author__ = {"github.com/": ["casperdcl", "lrq3000"]}
__all__ = ['tqdm_gui', 'tgrange', 'tqdm', 'trange']
class tqdm_gui(std_tqdm): # pragma: no cover
"""Experimental Matplotlib GUI version of tqdm!"""
# TODO: @classmethod: write() on GUI?
def __init__(self, *args, **kwargs):
from collections import deque
import matplotlib as mpl
import matplotlib.pyplot as plt
kwargs = kwargs.copy()
kwargs['gui'] = True
colour = kwargs.pop('colour', 'g')
super().__init__(*args, **kwargs)
if self.disable:
return
warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2)
self.mpl = mpl
self.plt = plt
# Remember if external environment uses toolbars
self.toolbar = self.mpl.rcParams['toolbar']
self.mpl.rcParams['toolbar'] = 'None'
self.mininterval = max(self.mininterval, 0.5)
self.fig, ax = plt.subplots(figsize=(9, 2.2))
# self.fig.subplots_adjust(bottom=0.2)
total = self.__len__() # avoids TypeError on None #971
if total is not None:
self.xdata = []
self.ydata = []
self.zdata = []
else:
self.xdata = deque([])
self.ydata = deque([])
self.zdata = deque([])
self.line1, = ax.plot(self.xdata, self.ydata, color='b')
self.line2, = ax.plot(self.xdata, self.zdata, color='k')
ax.set_ylim(0, 0.001)
if total is not None:
ax.set_xlim(0, 100)
ax.set_xlabel("percent")
self.fig.legend((self.line1, self.line2), ("cur", "est"),
loc='center right')
# progress bar
self.hspan = plt.axhspan(0, 0.001, xmin=0, xmax=0, color=colour)
else:
# ax.set_xlim(-60, 0)
ax.set_xlim(0, 60)
ax.invert_xaxis()
ax.set_xlabel("seconds")
ax.legend(("cur", "est"), loc='lower left')
ax.grid()
# ax.set_xlabel('seconds')
ax.set_ylabel((self.unit if self.unit else "it") + "/s")
if self.unit_scale:
plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
ax.yaxis.get_offset_text().set_x(-0.15)
# Remember if external environment is interactive
self.wasion = plt.isinteractive()
plt.ion()
self.ax = ax
def close(self):
if self.disable:
return
self.disable = True
with self.get_lock():
self._instances.remove(self)
# Restore toolbars
self.mpl.rcParams['toolbar'] = self.toolbar
# Return to non-interactive mode
if not self.wasion:
self.plt.ioff()
if self.leave:
self.display()
else:
self.plt.close(self.fig)
def clear(self, *_, **__):
pass
def display(self, *_, **__):
n = self.n
cur_t = self._time()
elapsed = cur_t - self.start_t
delta_it = n - self.last_print_n
delta_t = cur_t - self.last_print_t
# Inline due to multiple calls
total = self.total
xdata = self.xdata
ydata = self.ydata
zdata = self.zdata
ax = self.ax
line1 = self.line1
line2 = self.line2
hspan = getattr(self, 'hspan', None)
# instantaneous rate
y = delta_it / delta_t
# overall rate
z = n / elapsed
# update line data
xdata.append(n * 100.0 / total if total else cur_t)
ydata.append(y)
zdata.append(z)
# Discard old values
# xmin, xmax = ax.get_xlim()
# if (not total) and elapsed > xmin * 1.1:
if (not total) and elapsed > 66:
xdata.popleft()
ydata.popleft()
zdata.popleft()
ymin, ymax = ax.get_ylim()
if y > ymax or z > ymax:
ymax = 1.1 * y
ax.set_ylim(ymin, ymax)
ax.figure.canvas.draw()
if total:
line1.set_data(xdata, ydata)
line2.set_data(xdata, zdata)
if hspan:
hspan.set_xy((0, ymin))
hspan.set_height(ymax - ymin)
hspan.set_width(n / total)
else:
t_ago = [cur_t - i for i in xdata]
line1.set_data(t_ago, ydata)
line2.set_data(t_ago, zdata)
d = self.format_dict
# remove {bar}
d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace(
"{bar}", "<bar/>")
msg = self.format_meter(**d)
if '<bar/>' in msg:
msg = "".join(re.split(r'\|?<bar/>\|?', msg, maxsplit=1))
ax.set_title(msg, fontname="DejaVu Sans Mono", fontsize=11)
self.plt.pause(1e-9)
def tgrange(*args, **kwargs):
"""Shortcut for `tqdm.gui.tqdm(range(*args), **kwargs)`."""
return tqdm_gui(range(*args), **kwargs)
# Aliases
tqdm = tqdm_gui
trange = tgrange
@@ -0,0 +1,123 @@
from copy import copy
from functools import partial
from .auto import tqdm as tqdm_auto
try:
import keras
except (ImportError, AttributeError) as e:
try:
from tensorflow import keras
except ImportError:
raise e
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['TqdmCallback']
class TqdmCallback(keras.callbacks.Callback):
"""Keras callback for epoch and batch progress."""
@staticmethod
def bar2callback(bar, pop=None, delta=(lambda logs: 1)):
def callback(_, logs=None):
n = delta(logs)
if logs:
if pop:
logs = copy(logs)
for i in pop:
logs.pop(i, 0)
bar.set_postfix(logs, refresh=False)
bar.update(n)
return callback
def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1,
tqdm_class=tqdm_auto, **tqdm_kwargs):
"""
Parameters
----------
epochs : int, optional
data_size : int, optional
Number of training pairs.
batch_size : int, optional
Number of training pairs per batch.
verbose : int
0: epoch, 1: batch (transient), 2: batch. [default: 1].
Will be set to `0` unless both `data_size` and `batch_size`
are given.
tqdm_class : optional
`tqdm` class to use for bars [default: `tqdm.auto.tqdm`].
tqdm_kwargs : optional
Any other arguments used for all bars.
"""
if tqdm_kwargs:
tqdm_class = partial(tqdm_class, **tqdm_kwargs)
self.tqdm_class = tqdm_class
self.epoch_bar = tqdm_class(total=epochs, unit='epoch')
self.on_epoch_end = self.bar2callback(self.epoch_bar)
if data_size and batch_size:
self.batches = batches = (data_size + batch_size - 1) // batch_size
else:
self.batches = batches = None
self.verbose = verbose
if verbose == 1:
self.batch_bar = tqdm_class(total=batches, unit='batch', leave=False)
self.on_batch_end = self.bar2callback(
self.batch_bar, pop=['batch', 'size'],
delta=lambda logs: logs.get('size', 1))
def on_train_begin(self, *_, **__):
params = self.params.get
auto_total = params('epochs', params('nb_epoch', None))
if auto_total is not None and auto_total != self.epoch_bar.total:
self.epoch_bar.reset(total=auto_total)
def on_epoch_begin(self, epoch, *_, **__):
if self.epoch_bar.n < epoch:
ebar = self.epoch_bar
ebar.n = ebar.last_print_n = ebar.initial = epoch
if self.verbose:
params = self.params.get
total = params('samples', params(
'nb_sample', params('steps', None))) or self.batches
if self.verbose == 2:
if hasattr(self, 'batch_bar'):
self.batch_bar.close()
self.batch_bar = self.tqdm_class(
total=total, unit='batch', leave=True,
unit_scale=1 / (params('batch_size', 1) or 1))
self.on_batch_end = self.bar2callback(
self.batch_bar, pop=['batch', 'size'],
delta=lambda logs: logs.get('size', 1))
elif self.verbose == 1:
self.batch_bar.unit_scale = 1 / (params('batch_size', 1) or 1)
self.batch_bar.reset(total=total)
else:
raise KeyError('Unknown verbosity')
def on_train_end(self, *_, **__):
if hasattr(self, 'batch_bar'):
self.batch_bar.close()
self.epoch_bar.close()
def display(self):
"""Displays in the current cell in Notebooks."""
container = getattr(self.epoch_bar, 'container', None)
if container is None:
return
from .notebook import display
display(container)
batch_bar = getattr(self, 'batch_bar', None)
if batch_bar is not None:
display(batch_bar.container)
@staticmethod
def _implements_train_batch_hooks():
return True
@staticmethod
def _implements_test_batch_hooks():
return True
@staticmethod
def _implements_predict_batch_hooks():
return True
@@ -0,0 +1,317 @@
"""
IPython/Jupyter Notebook progress bar decorator for iterators.
Includes a default `range` iterator printing to `stderr`.
Usage:
>>> from tqdm.notebook import trange, tqdm
>>> for i in trange(10):
... ...
"""
# import compatibility functions and utilities
import re
import sys
from html import escape
from weakref import proxy
# to inherit from the tqdm class
from .std import tqdm as std_tqdm
if True: # pragma: no cover
# import IPython/Jupyter base widget and display utilities
IPY = 0
try: # IPython 4.x
import ipywidgets # noqa: F401, pylint: disable=unused-import
IPY = 4
except ImportError: # IPython 3.x / 2.x
IPY = 32
import warnings
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore', message=".*The `IPython.html` package has been deprecated.*")
try:
import IPython.html.widgets
except ImportError:
pass
else:
ipywidgets = IPython.html.widgets
try: # IPython 4.x / 3.x
if IPY == 32:
from IPython.html.widgets import HTML
from IPython.html.widgets import FloatProgress as IProgress
from IPython.html.widgets import HBox
IPY = 3
else:
from ipywidgets import HTML
from ipywidgets import FloatProgress as IProgress
from ipywidgets import HBox
except ImportError:
try: # IPython 2.x
from IPython.html.widgets import HTML
from IPython.html.widgets import ContainerWidget as HBox
from IPython.html.widgets import FloatProgressWidget as IProgress
IPY = 2
except ImportError:
IPY = 0
IProgress = None
HBox = object
try:
from IPython.display import display # , clear_output
except ImportError:
pass
__author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]}
__all__ = ['tqdm_notebook', 'tnrange', 'tqdm', 'trange']
WARN_NOIPYW = ("IProgress not found. Please update jupyter and ipywidgets."
" See https://ipywidgets.readthedocs.io/en/stable"
"/user_install.html")
class TqdmHBox(HBox):
"""`ipywidgets.HBox` with a pretty representation"""
def _json_(self, pretty=None):
pbar = getattr(self, 'pbar', None)
if pbar is None:
return {}
d = pbar.format_dict
if pretty is not None:
d["ascii"] = not pretty
return d
def __repr__(self, pretty=False):
pbar = getattr(self, 'pbar', None)
if pbar is None:
return super().__repr__()
return pbar.format_meter(**self._json_(pretty))
def _repr_pretty_(self, pp, *_, **__):
pp.text(self.__repr__(True))
class tqdm_notebook(std_tqdm):
"""
Experimental IPython/Jupyter Notebook widget using tqdm!
"""
@staticmethod
def status_printer(_, total=None, desc=None, ncols=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
# fp = file
# Prepare IPython progress bar
if IProgress is None: # #187 #451 #558 #872
raise ImportError(WARN_NOIPYW)
if total:
pbar = IProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if ncols is None:
pbar.layout.width = "20px"
ltext = HTML()
rtext = HTML()
if desc:
ltext.value = desc
container = TqdmHBox(children=[ltext, pbar, rtext])
# Prepare layout
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
try:
if int(ncols) > 0: # isnumeric and positive
ncols += 'px'
except ValueError:
pass
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
return container
def display(self, msg=None, pos=None,
# additional signals
close=False, bar_style=None, check_delay=True):
# Note: contrary to native tqdm, msg='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
if not msg and not close:
d = self.format_dict
# remove {bar}
d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace(
"{bar}", "<bar/>")
msg = self.format_meter(**d)
ltext, pbar, rtext = self.container.children
pbar.value = self.n
if msg:
msg = msg.replace(' ', '\u2007') # fix html space padding
# html escape special characters (like '&')
if '<bar/>' in msg:
left, right = map(escape, re.split(r'\|?<bar/>\|?', msg, maxsplit=1))
else:
left, right = '', escape(msg)
# Update description
ltext.value = left
# never clear the bar (signal: msg='')
if right:
rtext.value = right
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overridden by
# success because the bar gets closed after the error...
if pbar.bar_style != 'danger' or bar_style != 'success':
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
self.container.close()
except AttributeError:
self.container.visible = False
self.container.layout.visibility = 'hidden' # IPYW>=8
if check_delay and self.delay > 0 and not self.displayed:
display(self.container)
self.displayed = True
@property
def colour(self):
if hasattr(self, 'container'):
return self.container.children[-2].style.bar_color
@colour.setter
def colour(self, bar_color):
if hasattr(self, 'container'):
self.container.children[-2].style.bar_color = bar_color
def __init__(self, *args, **kwargs):
"""
Supports the usual `tqdm.tqdm` parameters as well as those listed below.
Parameters
----------
display : Whether to call `display(self.container)` immediately
[default: True].
"""
kwargs = kwargs.copy()
# Setup default output
file_kwarg = kwargs.get('file', sys.stderr)
if file_kwarg is sys.stderr or file_kwarg is None:
kwargs['file'] = sys.stdout # avoid the red block in IPython
# Initialize parent class + avoid printing by using gui=True
kwargs['gui'] = True
# convert disable = None to False
kwargs['disable'] = bool(kwargs.get('disable', False))
colour = kwargs.pop('colour', None)
display_here = kwargs.pop('display', True)
super().__init__(*args, **kwargs)
if self.disable or not kwargs['gui']:
self.disp = lambda *_, **__: None
return
# Get bar width
self.ncols = '100%' if self.dynamic_ncols else kwargs.get("ncols", None)
# Replace with IPython progress bar display (with correct total)
unit_scale = 1 if self.unit_scale is True else self.unit_scale or 1
total = self.total * unit_scale if self.total else self.total
self.container = self.status_printer(self.fp, total, self.desc, self.ncols)
self.container.pbar = proxy(self)
self.displayed = False
if display_here and self.delay <= 0:
display(self.container)
self.displayed = True
self.disp = self.display
self.colour = colour
# Print initial bar state
if not self.disable:
self.display(check_delay=False)
def __iter__(self):
try:
it = super().__iter__()
yield from it
# NB: except ... [ as ...] breaks IPython async KeyboardInterrupt
except: # NOQA
self.disp(bar_style='danger')
raise
# NB: don't `finally: close()`
# since this could be a shared bar which the user will `reset()`
def update(self, n=1):
try:
return super().update(n=n)
# NB: except ... [ as ...] breaks IPython async KeyboardInterrupt
except: # NOQA
# cannot catch KeyboardInterrupt when using manual tqdm
# as the interrupt will most likely happen on another statement
self.disp(bar_style='danger')
raise
# NB: don't `finally: close()`
# since this could be a shared bar which the user will `reset()`
def close(self):
if self.disable:
return
super().close()
# Try to detect if there was an error or KeyboardInterrupt
# in manual mode: if n < total, things probably got wrong
if self.total and self.n < self.total:
self.disp(bar_style='danger', check_delay=False)
else:
if self.leave:
self.disp(bar_style='success', check_delay=False)
else:
self.disp(close=True, check_delay=False)
def clear(self, *_, **__):
pass
def reset(self, total=None):
"""
Resets to 0 iterations for repeated use.
Consider combining with `leave=True`.
Parameters
----------
total : int or float, optional. Total to use for the new bar.
"""
if self.disable:
return super().reset(total=total)
_, pbar, _ = self.container.children
pbar.bar_style = ''
if total is not None:
pbar.max = total
if not self.total and self.ncols is None: # no longer unknown total
pbar.layout.width = None # reset width
return super().reset(total=total)
def tnrange(*args, **kwargs):
"""Shortcut for `tqdm.notebook.tqdm(range(*args), **kwargs)`."""
return tqdm_notebook(range(*args), **kwargs)
# Aliases
tqdm = tqdm_notebook
trange = tnrange
@@ -0,0 +1,154 @@
"""
`rich.progress` decorator for iterators.
Usage:
>>> from tqdm.rich import trange, tqdm
>>> for i in trange(10):
... ...
"""
from warnings import warn
from rich.progress import (
BarColumn, Progress, ProgressColumn, Text, TimeElapsedColumn, TimeRemainingColumn, filesize)
from .std import TqdmExperimentalWarning
from .std import tqdm as std_tqdm
__author__ = {"github.com/": ["casperdcl"]}
__all__ = ['tqdm_rich', 'trrange', 'tqdm', 'trange']
class FractionColumn(ProgressColumn):
"""Renders completed/total, e.g. '0.5/2.3 G'."""
def __init__(self, unit_scale=False, unit_divisor=1000):
self.unit_scale = unit_scale
self.unit_divisor = unit_divisor
super().__init__()
def render(self, task):
"""Calculate common unit for completed and total."""
completed = int(task.completed)
# total is None when the iterable has no length (e.g. a generator)
total = int(task.total) if task.total is not None else None
base = total if total is not None else completed
if self.unit_scale:
unit, suffix = filesize.pick_unit_and_suffix(
base,
["", "K", "M", "G", "T", "P", "E", "Z", "Y"],
self.unit_divisor,
)
else:
unit, suffix = filesize.pick_unit_and_suffix(base, [""], 1)
precision = 0 if unit == 1 else 1
total_str = f"{total/unit:,.{precision}f}" if total is not None else "?"
return Text(
f"{completed/unit:,.{precision}f}/{total_str} {suffix}",
style="progress.download")
class RateColumn(ProgressColumn):
"""Renders human readable transfer speed."""
def __init__(self, unit="", unit_scale=False, unit_divisor=1000):
self.unit = unit
self.unit_scale = unit_scale
self.unit_divisor = unit_divisor
super().__init__()
def render(self, task):
"""Show data transfer speed."""
speed = task.speed
if speed is None:
return Text(f"? {self.unit}/s", style="progress.data.speed")
if self.unit_scale:
unit, suffix = filesize.pick_unit_and_suffix(
speed,
["", "K", "M", "G", "T", "P", "E", "Z", "Y"],
self.unit_divisor,
)
else:
unit, suffix = filesize.pick_unit_and_suffix(speed, [""], 1)
precision = 0 if unit == 1 else 1
return Text(f"{speed/unit:,.{precision}f} {suffix}{self.unit}/s",
style="progress.data.speed")
class tqdm_rich(std_tqdm): # pragma: no cover
"""Experimental rich.progress GUI version of tqdm!"""
# TODO: @classmethod: write()?
def __init__(self, *args, **kwargs):
"""
This class accepts the following parameters *in addition* to
the parameters accepted by `tqdm`.
Parameters
----------
progress : tuple, optional
arguments for `rich.progress.Progress()`.
options : dict, optional
keyword arguments for `rich.progress.Progress()`.
"""
kwargs = kwargs.copy()
kwargs['gui'] = True
# convert disable = None to False
kwargs['disable'] = bool(kwargs.get('disable', False))
progress = kwargs.pop('progress', None)
options = kwargs.pop('options', {}).copy()
super().__init__(*args, **kwargs)
if self.disable:
return
warn("rich is experimental/alpha", TqdmExperimentalWarning, stacklevel=2)
d = self.format_dict
if progress is None:
progress = (
"[progress.description]{task.description}"
"[progress.percentage]{task.percentage:>4.0f}%",
BarColumn(bar_width=None),
FractionColumn(
unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']),
"[", TimeElapsedColumn(), "<", TimeRemainingColumn(),
",", RateColumn(unit=d['unit'], unit_scale=d['unit_scale'],
unit_divisor=d['unit_divisor']), "]"
)
options.setdefault('transient', not self.leave)
self._prog = Progress(*progress, **options)
self._prog.__enter__()
self._task_id = self._prog.add_task(self.desc or "", **d)
def close(self):
if self.disable:
return
self.display() # print 100%, vis #1306
super().close()
self._prog.__exit__(None, None, None)
def clear(self, *_, **__):
pass
def display(self, *_, **__):
if not hasattr(self, '_task_id'):
return
self._prog.update(self._task_id, completed=self.n, description=self.desc)
def reset(self, total=None):
"""
Resets to 0 iterations for repeated use.
Parameters
----------
total : int or float, optional. Total to use for the new bar.
"""
if hasattr(self, '_prog'):
self._prog.reset(total=total)
super().reset(total=total)
def trrange(*args, **kwargs):
"""Shortcut for `tqdm.rich.tqdm(range(*args), **kwargs)`."""
return tqdm_rich(range(*args), **kwargs)
# Aliases
tqdm = tqdm_rich
trange = trrange
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,196 @@
"""
Tkinter GUI progress bar decorator for iterators.
Usage:
>>> from tqdm.tk import trange, tqdm
>>> for i in trange(10):
... ...
"""
import re
import sys
import tkinter
import tkinter.ttk as ttk
from warnings import warn
from .std import TqdmExperimentalWarning, TqdmWarning
from .std import tqdm as std_tqdm
__author__ = {"github.com/": ["richardsheridan", "casperdcl"]}
__all__ = ['tqdm_tk', 'ttkrange', 'tqdm', 'trange']
class tqdm_tk(std_tqdm): # pragma: no cover
"""
Experimental Tkinter GUI version of tqdm!
Note: Window interactivity suffers if `tqdm_tk` is not running within
a Tkinter mainloop and values are generated infrequently. In this case,
consider calling `tqdm_tk.refresh()` frequently in the Tk thread.
"""
# TODO: @classmethod: write()?
def __init__(self, *args, **kwargs):
"""
This class accepts the following parameters *in addition* to
the parameters accepted by `tqdm`.
Parameters
----------
grab : bool, optional
Grab the input across all windows of the process.
tk_parent : `tkinter.Wm`, optional
Parent Tk window.
cancel_callback : Callable, optional
Create a cancel button and set `cancel_callback` to be called
when the cancel or window close button is clicked.
"""
kwargs = kwargs.copy()
kwargs['gui'] = True
# convert disable = None to False
kwargs['disable'] = bool(kwargs.get('disable', False))
self._warn_leave = 'leave' in kwargs
grab = kwargs.pop('grab', False)
tk_parent = kwargs.pop('tk_parent', None)
self._cancel_callback = kwargs.pop('cancel_callback', None)
super().__init__(*args, **kwargs)
if self.disable:
return
if tk_parent is None: # Discover parent widget
try:
tk_parent = tkinter._default_root
except AttributeError:
raise AttributeError(
"`tk_parent` required when using `tkinter.NoDefaultRoot()`")
if tk_parent is None: # use new default root window as display
self._tk_window = tkinter.Tk()
else: # some other windows already exist
self._tk_window = tkinter.Toplevel()
else:
self._tk_window = tkinter.Toplevel(tk_parent)
warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2)
self._tk_dispatching = self._tk_dispatching_helper()
self._tk_window.protocol("WM_DELETE_WINDOW", self.cancel)
self._tk_window.wm_title(self.desc)
self._tk_window.wm_attributes("-topmost", 1)
self._tk_window.after(0, lambda: self._tk_window.wm_attributes("-topmost", 0))
self._tk_n_var = tkinter.DoubleVar(self._tk_window, value=0)
self._tk_text_var = tkinter.StringVar(self._tk_window)
pbar_frame = ttk.Frame(self._tk_window, padding=5)
pbar_frame.pack()
_tk_label = ttk.Label(pbar_frame, textvariable=self._tk_text_var,
wraplength=600, anchor="center", justify="center")
_tk_label.pack()
self._tk_pbar = ttk.Progressbar(
pbar_frame, variable=self._tk_n_var, length=450)
if self.total is not None:
self._tk_pbar.configure(maximum=self.total)
else:
self._tk_pbar.configure(mode="indeterminate")
self._tk_pbar.pack()
if self._cancel_callback is not None:
_tk_button = ttk.Button(pbar_frame, text="Cancel", command=self.cancel)
_tk_button.pack()
if grab:
self._tk_window.grab_set()
def close(self):
if self.disable:
return
self.disable = True
with self.get_lock():
self._instances.remove(self)
def _close():
self._tk_window.after('idle', self._tk_window.destroy)
if not self._tk_dispatching:
self._tk_window.update()
self._tk_window.protocol("WM_DELETE_WINDOW", _close)
# if leave is set but we are self-dispatching, the left window is
# totally unresponsive unless the user manually dispatches
if not self.leave:
_close()
elif not self._tk_dispatching:
if self._warn_leave:
warn("leave flag ignored if not in tkinter mainloop",
TqdmWarning, stacklevel=2)
_close()
def clear(self, *_, **__):
pass
def display(self, *_, **__):
self._tk_n_var.set(self.n)
d = self.format_dict
# remove {bar}
d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace(
"{bar}", "<bar/>")
msg = self.format_meter(**d)
if '<bar/>' in msg:
msg = "".join(re.split(r'\|?<bar/>\|?', msg, maxsplit=1))
self._tk_text_var.set(msg)
if not self._tk_dispatching:
self._tk_window.update()
def set_description(self, desc=None, refresh=True):
self.set_description_str(desc, refresh)
def set_description_str(self, desc=None, refresh=True):
self.desc = desc
if not self.disable:
self._tk_window.wm_title(desc)
if refresh and not self._tk_dispatching:
self._tk_window.update()
def cancel(self):
"""
`cancel_callback()` followed by `close()`
when close/cancel buttons clicked.
"""
if self._cancel_callback is not None:
self._cancel_callback()
self.close()
def reset(self, total=None):
"""
Resets to 0 iterations for repeated use.
Parameters
----------
total : int or float, optional. Total to use for the new bar.
"""
if hasattr(self, '_tk_pbar'):
if total is None:
self._tk_pbar.configure(maximum=100, mode="indeterminate")
else:
self._tk_pbar.configure(maximum=total, mode="determinate")
super().reset(total=total)
@staticmethod
def _tk_dispatching_helper():
"""determine if Tkinter mainloop is dispatching events"""
codes = {tkinter.mainloop.__code__, tkinter.Misc.mainloop.__code__}
for frame in sys._current_frames().values():
while frame:
if frame.f_code in codes:
return True
frame = frame.f_back
return False
def ttkrange(*args, **kwargs):
"""Shortcut for `tqdm.tk.tqdm(range(*args), **kwargs)`."""
return tqdm_tk(range(*args), **kwargs)
# Aliases
tqdm = tqdm_tk
trange = ttkrange
@@ -0,0 +1,241 @@
.TH "TQDM" "1" "2015\-2026" "tqdm User Manuals"
.SH NAME
tqdm \- fast, extensible progress bar for Python and CLI
.SH SYNOPSIS
tqdm [\f[I]options\f[R]]
.SH DESCRIPTION
See \c
.UR https://github.com/tqdm/tqdm
.UE \c
\&.
Can be used as a pipe:
.IP
.EX
$ \f[I]# count lines of code\f[R]
$ cat *.py \f[B]|\f[R] tqdm \f[B]|\f[R] wc \-l
327it [00:00, 981773.38it/s]
327
$ \f[I]# find all files\f[R]
$ find . \-name \(dq*.py\(dq \f[B]|\f[R] tqdm \f[B]|\f[R] wc \-l
432it [00:00, 833842.30it/s]
432
\f[I]# ... and more info\f[R]
$ find . \-name \(aq*.py\(aq \-exec wc \-l \(rs{} \(rs; \(rs
\f[B]|\f[R] tqdm \-\-total 432 \-\-unit files \-\-desc counting \(rs
\f[B]|\f[R] awk \(aq{ sum += $1 }; END { print sum }\(aq
counting: 100%\f[B]|\f[R]█████████\f[B]|\f[R] 432/432 [00:00<00:00, 794361.83files/s]
131998
.EE
.SH OPTIONS
.TP
\-h, \-\-help
Print this help and exit.
.TP
\-v, \-\-version
Print version and exit.
.TP
\-\-desc=\f[I]desc\f[R]
str, optional.
Prefix for the progress bar.
.TP
\-\-total=\f[I]total\f[R]
int or float, optional.
The number of expected iterations.
If unspecified, len(iterable) is used if possible.
If float(\(lqinf\(rq) or as a last resort, only basic progress
statistics are displayed (no ETA, no progress bar).
If \f[CR]gui\f[R] is True and this parameter needs subsequent updating,
specify an initial arbitrary large positive number, e.g.\ 9e9.
.TP
\-\-leave
bool, optional.
If [default: True], keeps all traces of the progress bar upon
termination of iteration.
If \f[CR]None\f[R], will leave only if \f[CR]position\f[R] is
\f[CR]0\f[R].
.TP
\-\-ncols=\f[I]ncols\f[R]
int, optional.
The width of the entire output message.
If specified, dynamically resizes the progress bar to stay within this
bound.
If unspecified, attempts to use environment width.
The fallback is a meter width of 10 and no limit for the counter and
statistics.
If 0, will not print any meter (only stats).
.TP
\-\-mininterval=\f[I]mininterval\f[R]
float, optional.
Minimum progress display update interval [default: 0.1] seconds.
.TP
\-\-maxinterval=\f[I]maxinterval\f[R]
float, optional.
Maximum progress display update interval [default: 10] seconds.
Automatically adjusts \f[CR]miniters\f[R] to correspond to
\f[CR]mininterval\f[R] after long display update lag.
Only works if \f[CR]dynamic_miniters\f[R] or monitor thread is enabled.
.TP
\-\-miniters=\f[I]miniters\f[R]
int or float, optional.
Minimum progress display update interval, in iterations.
If 0 and \f[CR]dynamic_miniters\f[R], will automatically adjust to equal
\f[CR]mininterval\f[R] (more CPU efficient, good for tight loops).
If > 0, will skip display of specified number of iterations.
Tweak this and \f[CR]mininterval\f[R] to get very efficient loops.
If your progress is erratic with both fast and slow iterations (network,
skipping items, etc) you should set miniters=1.
.TP
\-\-ascii=\f[I]ascii\f[R]
bool or str, optional.
If unspecified or False, use unicode (smooth blocks) to fill the meter.
The fallback is to use ASCII characters \(rq 123456789#\(lq.
.TP
\-\-disable
bool, optional.
Whether to disable the entire progress bar wrapper [default: False].
If set to None, disable on non\-TTY.
.TP
\-\-unit=\f[I]unit\f[R]
str, optional.
String that will be used to define the unit of each iteration [default:
it].
.TP
\-\-unit\-scale=\f[I]unit_scale\f[R]
bool or int or float, optional.
If 1 or True, the number of iterations will be reduced/scaled
automatically and a metric prefix following the International System of
Units standard will be added (kilo, mega, etc.)
[default: False].
If any other non\-zero number, will scale \f[CR]total\f[R] and
\f[CR]n\f[R].
.TP
\-\-dynamic\-ncols
bool, optional.
If set, constantly alters \f[CR]ncols\f[R] and \f[CR]nrows\f[R] to the
environment (allowing for window resizes) [default: False].
.TP
\-\-smoothing=\f[I]smoothing\f[R]
float, optional.
Exponential moving average smoothing factor for speed estimates (ignored
in GUI mode).
Ranges from 0 (average speed) to 1 (current/instantaneous speed)
[default: 0.3].
.TP
\-\-bar\-format=\f[I]bar_format\f[R]
str, optional.
Specify a custom bar string formatting.
May impact performance.
[default: `{l_bar}{bar}{r_bar}'], where l_bar=`{desc}:
{percentage:3.0f}%|' and r_bar=`| {n_fmt}/{total_fmt}
[{elapsed}<{remaining}, \(cq \(cq{rate_fmt}{postfix}]' Possible vars:
l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, elapsed,
elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, rate_noinv,
rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, unit_divisor,
remaining, remaining_s, eta.
Note that a trailing \(lq:\(rq is automatically removed after {desc} if
the latter is empty.
.TP
\-\-initial=\f[I]initial\f[R]
int or float, optional.
The initial counter value.
Useful when restarting a progress bar [default: 0].
If using float, consider specifying \f[CR]{n:.3f}\f[R] or similar in
\f[CR]bar_format\f[R], or specifying \f[CR]unit_scale\f[R].
.TP
\-\-position=\f[I]position\f[R]
int, optional.
Specify the line offset to print this bar (starting from 0) Automatic if
unspecified.
Useful to manage multiple bars at once (eg, from threads).
.TP
\-\-postfix=\f[I]postfix\f[R]
dict or *, optional.
Specify additional stats to display at the end of the bar.
Calls \f[CR]set_postfix(**postfix)\f[R] if possible (dict).
.TP
\-\-unit\-divisor=\f[I]unit_divisor\f[R]
float, optional.
[default: 1000], ignored unless \f[CR]unit_scale\f[R] is True.
.TP
\-\-write\-bytes
bool, optional.
Whether to write bytes.
If (default: False) will write unicode.
.TP
\-\-lock\-args=\f[I]lock_args\f[R]
tuple, optional.
Passed to \f[CR]refresh\f[R] for intermediate output (initialisation,
iterating, and updating).
.TP
\-\-nrows=\f[I]nrows\f[R]
int, optional.
The screen height.
If specified, hides nested bars outside this bound.
If unspecified, attempts to use environment height.
The fallback is 20.
.TP
\-\-colour=\f[I]colour\f[R]
str, optional.
Bar colour (e.g.\ `green', `#00ff00').
.TP
\-\-delay=\f[I]delay\f[R]
float, optional.
Don\(cqt display until [default: 0] seconds have elapsed.
.TP
\-\-delim=\f[I]delim\f[R]
chr, optional.
Delimiting character [default: `\(rsn'].
Use `\(rs0' for null.
N.B.: on Windows systems, Python converts `\(rsn' to `\(rsr\(rsn'.
.TP
\-\-buf\-size=\f[I]buf_size\f[R]
int, optional.
String buffer size in bytes [default: 256] used when \f[CR]delim\f[R] is
specified.
.TP
\-\-bytes
bool, optional.
If true, will count bytes, ignore \f[CR]delim\f[R], and default
\f[CR]unit_scale\f[R] to True, \f[CR]unit_divisor\f[R] to 1024, and
\f[CR]unit\f[R] to `B'.
.TP
\-\-tee
bool, optional.
If true, passes \f[CR]stdin\f[R] to both \f[CR]stderr\f[R] and
\f[CR]stdout\f[R].
.TP
\-\-update
bool, optional.
If true, will treat input as newly elapsed iterations, i.e.\ numbers to
pass to \f[CR]update()\f[R].
Note that this is slow (\(ti2e5 it/s) since every input must be decoded
as a number.
.TP
\-\-update\-to
bool, optional.
If true, will treat input as total elapsed iterations, i.e.\ numbers to
assign to \f[CR]self.n\f[R].
Note that this is slow (\(ti2e5 it/s) since every input must be decoded
as a number.
.TP
\-\-null
bool, optional.
If true, will discard input (no stdout).
.TP
\-\-manpath=\f[I]manpath\f[R]
str, optional.
Directory in which to install tqdm man pages.
.TP
\-\-comppath=\f[I]comppath\f[R]
str, optional.
Directory in which to place tqdm completion.
.TP
\-\-log=\f[I]log\f[R]
str, optional.
CRITICAL|FATAL|ERROR|WARN(ING)|[default: `INFO']|DEBUG|NOTSET.
.SH AUTHORS
tqdm developers \c
.UR https://github.com/tqdm
.UE \c.
@@ -0,0 +1,328 @@
"""
General helpers required for `tqdm.std`.
"""
import os
import re
import sys
from functools import partial, partialmethod, wraps
from inspect import signature
# TODO consider using wcswidth third-party package for 0-width characters
from unicodedata import east_asian_width
from warnings import warn
from weakref import proxy
_range, _unich, _unicode, _basestring = range, chr, str, str
CUR_OS = sys.platform
IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin'])
IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin', 'freebsd'])
RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]")
try:
if IS_WIN:
import colorama
else:
raise ImportError
except ImportError:
colorama = None
else:
try:
colorama.init(strip=False)
except TypeError:
colorama.init()
def envwrap(name, app="", types=None, is_method=False):
"""
Basic (env-only) version of [envwrap](https://github.com/tqdm/envwrap).
Install `envwrap` for config file support.
"""
if types is None:
types = {}
if name[-1] == "_":
name = name[:-1]
warn("Trailing underscore in `name` is automatic", DeprecationWarning, stacklevel=2)
prefixes = (name, f"{name}_{app}") if app else (name,)
env_overrides = {}
for prefix in prefixes:
prefix = prefix.upper() + "_"
i = len(prefix)
env_overrides.update(
(k[i:].lower(), v) for k, v in os.environ.items() if k.startswith(prefix))
part = partialmethod if is_method else partial
def wrap(func):
params = signature(func).parameters
# ignore unknown env vars
overrides = {k: v for k, v in env_overrides.items() if k in params}
# infer overrides' `type`s
for k in overrides:
param = params[k]
if param.annotation is not param.empty: # typehints
for typ in getattr(param.annotation, '__args__', (param.annotation,)):
try:
overrides[k] = typ(overrides[k])
except Exception: # nosec B110
pass
else:
break
elif param.default is not None: # type of default value
overrides[k] = type(param.default)(overrides[k])
else:
try: # `types` fallback
overrides[k] = types[k](overrides[k])
except KeyError: # keep unconverted (`str`)
pass
return part(func, **overrides)
return wrap
try:
from envwrap import envwrap # noqa: F401, F811, pylint: disable=unused-import
except ModuleNotFoundError:
pass
class FormatReplace:
"""
>>> a = FormatReplace('something')
>>> f"{a:5d}"
'something'
""" # NOQA: P102
def __init__(self, replace=''):
self.replace = replace
self.format_called = 0
def __format__(self, _):
self.format_called += 1
return self.replace
class Comparable:
"""Assumes child has self._comparable attr/@property"""
def __lt__(self, other):
return self._comparable < other._comparable
def __le__(self, other):
return (self < other) or (self == other)
def __eq__(self, other):
return self._comparable == other._comparable
def __ne__(self, other):
return not self == other
def __gt__(self, other):
return not self <= other
def __ge__(self, other):
return not self < other
class ObjectWrapper:
def __getattr__(self, name):
return getattr(self._wrapped, name)
def __setattr__(self, name, value):
return setattr(self._wrapped, name, value)
def wrapper_getattr(self, name):
"""Actual `self.getattr` rather than self._wrapped.getattr"""
try:
return object.__getattr__(self, name)
except AttributeError: # py2
return getattr(self, name)
def wrapper_setattr(self, name, value):
"""Actual `self.setattr` rather than self._wrapped.setattr"""
return object.__setattr__(self, name, value)
def __init__(self, wrapped):
"""
Thin wrapper around a given object
"""
self.wrapper_setattr('_wrapped', wrapped)
class SimpleTextIOWrapper(ObjectWrapper):
"""
Change only `.write()` of the wrapped object by encoding the passed
value and passing the result to the wrapped object's `.write()` method.
"""
# pylint: disable=too-few-public-methods
def __init__(self, wrapped, encoding):
super().__init__(wrapped)
self.wrapper_setattr('encoding', encoding)
def write(self, s):
"""
Encode `s` and pass to the wrapped object's `.write()` method.
"""
return self._wrapped.write(s.encode(self.wrapper_getattr('encoding')))
def __eq__(self, other):
return self._wrapped == getattr(other, '_wrapped', other)
class DisableOnWriteError(ObjectWrapper):
"""
Disable the given `tqdm_instance` upon `write()` or `flush()` errors.
"""
@staticmethod
def disable_on_exception(tqdm_instance, func):
"""
Quietly set `tqdm_instance.miniters=inf` if `func` raises `errno=5`.
"""
tqdm_instance = proxy(tqdm_instance)
def inner(*args, **kwargs):
try:
return func(*args, **kwargs)
except OSError as e:
if e.errno != 5:
raise
try:
tqdm_instance.miniters = float('inf')
except ReferenceError:
pass
except ValueError as e:
if 'closed' not in str(e):
raise
try:
tqdm_instance.miniters = float('inf')
except ReferenceError:
pass
return inner
def __init__(self, wrapped, tqdm_instance): # noqa: B042
super().__init__(wrapped)
if hasattr(wrapped, 'write'):
self.wrapper_setattr(
'write', self.disable_on_exception(tqdm_instance, wrapped.write))
if hasattr(wrapped, 'flush'):
self.wrapper_setattr(
'flush', self.disable_on_exception(tqdm_instance, wrapped.flush))
def __eq__(self, other):
return self._wrapped == getattr(other, '_wrapped', other)
class CallbackIOWrapper(ObjectWrapper):
def __init__(self, callback, stream, method="read"):
"""
Wrap a given `file`-like object's `read()` or `write()` to report
lengths to the given `callback`
"""
super().__init__(stream)
func = getattr(stream, method)
if method == "write":
@wraps(func)
def write(data, *args, **kwargs):
res = func(data, *args, **kwargs)
callback(len(data))
return res
self.wrapper_setattr('write', write)
elif method == "read":
@wraps(func)
def read(*args, **kwargs):
data = func(*args, **kwargs)
callback(len(data))
return data
self.wrapper_setattr('read', read)
else:
raise KeyError("Can only wrap read/write methods")
def _is_utf(encoding):
try:
'\u2588\u2589'.encode(encoding)
except UnicodeEncodeError:
return False
except Exception:
try:
return encoding.lower().startswith('utf-') or ('U8' == encoding)
except Exception:
return False
else:
return True
def _supports_unicode(fp):
try:
return _is_utf(fp.encoding)
except AttributeError:
return False
def _is_ascii(s):
if isinstance(s, str):
for c in s:
if ord(c) > 255:
return False
return True
return _supports_unicode(s)
def _screen_shape_wrapper(): # pragma: no cover
"""
Return a function which returns console dimensions (width, height).
Supported: linux, osx, windows, cygwin.
"""
def inner(fp):
try:
from os import get_terminal_size
cols, lines = get_terminal_size(getattr(fp, 'fileno', lambda: None)())
return cols - 1, lines - 1
except Exception:
return None, None
return inner
def _environ_cols_wrapper(): # pragma: no cover
"""
Return a function which returns console width.
Supported: linux, osx, windows, cygwin.
"""
warn("Use `_screen_shape_wrapper()(file)[0]` instead of"
" `_environ_cols_wrapper()(file)`", DeprecationWarning, stacklevel=2)
shape = _screen_shape_wrapper()
if not shape:
return None
@wraps(shape)
def inner(fp):
return shape(fp)[0]
return inner
def _term_move_up(): # pragma: no cover
return '' if (os.name == 'nt') and (colorama is None) else '\x1b[A'
def _text_width(s):
return sum(2 if east_asian_width(ch) in 'FW' else 1 for ch in str(s))
def disp_len(data):
"""
Returns the real on-screen length of a string which may contain
ANSI control codes and wide chars.
"""
return _text_width(RE_ANSI.sub('', data))
def disp_trim(data, length):
"""
Trim a string which may contain ANSI control characters.
"""
if len(data) == disp_len(data):
return data[:length]
ansi_present = bool(RE_ANSI.search(data))
while disp_len(data) > length: # carefully delete one char at a time
data = data[:-1]
if ansi_present and bool(RE_ANSI.search(data)):
# assume ANSI reset is required
return data if data.endswith("\033[0m") else data + "\033[0m"
return data
@@ -0,0 +1,7 @@
"""`tqdm` version detector. Precedence: installed dist, git, 'UNKNOWN'."""
from importlib.metadata import PackageNotFoundError, version
try:
__version__ = version('tqdm')
except PackageNotFoundError:
__version__ = "UNKNOWN"