feat: add psagentai agent with deployment and configuration
Initial project setup including the main agent script, configuration files, profile JSON state, systemd service unit, startup script, deployment documentation, and Python dependencies. Includes .env, venv/, and __pycache__/ which should likely be excluded via .gitignore to avoid committing secrets and artifacts.
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Thin wrappers around common functions.
|
||||
|
||||
Subpackages contain potentially unstable extensions.
|
||||
"""
|
||||
from warnings import warn
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
from ..std import TqdmDeprecationWarning, tqdm
|
||||
from ..utils import ObjectWrapper
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl"]}
|
||||
__all__ = ['tenumerate', 'tzip', 'tmap']
|
||||
|
||||
|
||||
class DummyTqdmFile(ObjectWrapper):
|
||||
"""Dummy file-like that will write to tqdm"""
|
||||
|
||||
def __init__(self, wrapped):
|
||||
super().__init__(wrapped)
|
||||
self._buf = []
|
||||
|
||||
def write(self, x, nolock=False):
|
||||
nl = b"\n" if isinstance(x, bytes) else "\n"
|
||||
pre, sep, post = x.rpartition(nl)
|
||||
if sep:
|
||||
blank = type(nl)()
|
||||
tqdm.write(blank.join(self._buf + [pre, sep]),
|
||||
end=blank, file=self._wrapped, nolock=nolock)
|
||||
self._buf = [post]
|
||||
else:
|
||||
self._buf.append(x)
|
||||
|
||||
def __del__(self):
|
||||
if self._buf:
|
||||
blank = type(self._buf[0])()
|
||||
try:
|
||||
tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped)
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def builtin_iterable(func):
|
||||
"""Returns `func`"""
|
||||
warn("This function has no effect, and will be removed in tqdm==5.0.0",
|
||||
TqdmDeprecationWarning, stacklevel=2)
|
||||
return func
|
||||
|
||||
|
||||
def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of `numpy.ndenumerate` or builtin `enumerate`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tqdm_class : [default: tqdm.auto.tqdm].
|
||||
"""
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(iterable, np.ndarray):
|
||||
return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size,
|
||||
**tqdm_kwargs)
|
||||
return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start)
|
||||
|
||||
|
||||
def tzip(iter1, *iter2plus, **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of builtin `zip`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tqdm_class : [default: tqdm.auto.tqdm].
|
||||
"""
|
||||
kwargs = tqdm_kwargs.copy()
|
||||
tqdm_class = kwargs.pop("tqdm_class", tqdm_auto)
|
||||
yield from zip(tqdm_class(iter1, **kwargs), *iter2plus)
|
||||
|
||||
|
||||
def tmap(function, *sequences, **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of builtin `map`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tqdm_class : [default: tqdm.auto.tqdm].
|
||||
"""
|
||||
for i in tzip(*sequences, **tqdm_kwargs):
|
||||
yield function(*i)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
Even more features than `tqdm.auto` (all the bells & whistles):
|
||||
|
||||
- `tqdm.auto`
|
||||
- `tqdm.tqdm.pandas`
|
||||
- `tqdm.contrib.slack`
|
||||
+ uses `${TQDM_SLACK_TOKEN}` and `${TQDM_SLACK_CHANNEL}`
|
||||
- `tqdm.contrib.telegram`
|
||||
+ uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}`
|
||||
- `tqdm.contrib.discord`
|
||||
+ uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}`
|
||||
"""
|
||||
__all__ = ['tqdm', 'trange']
|
||||
import warnings
|
||||
from os import getenv
|
||||
|
||||
if getenv("TQDM_SLACK_TOKEN") and getenv("TQDM_SLACK_CHANNEL"):
|
||||
from .slack import tqdm, trange
|
||||
elif getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"):
|
||||
from .telegram import tqdm, trange
|
||||
elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"):
|
||||
from .discord import tqdm, trange
|
||||
else:
|
||||
from ..auto import tqdm, trange
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", category=FutureWarning)
|
||||
tqdm.pandas()
|
||||
@@ -0,0 +1,259 @@
|
||||
"""
|
||||
Thin wrappers around `concurrent.futures`.
|
||||
"""
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from operator import length_hint
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
from ..std import TqdmWarning
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl"]}
|
||||
__all__ = ['thread_map', 'process_map', 'interpreter_map']
|
||||
|
||||
|
||||
class _InterpreterLock:
|
||||
"""Reentrant lock backed by a cross-interpreter queue."""
|
||||
from threading import get_ident
|
||||
from time import monotonic as _time
|
||||
|
||||
def __init__(self, queue):
|
||||
from threading import RLock
|
||||
self._queue = queue
|
||||
self._lock = RLock()
|
||||
self._owner = None
|
||||
self._depth = 0
|
||||
|
||||
def acquire(self, blocking=True, timeout=-1):
|
||||
from queue import Empty
|
||||
start = self._time()
|
||||
if timeout == -1:
|
||||
acquired = self._lock.acquire(blocking)
|
||||
else:
|
||||
acquired = self._lock.acquire(blocking, timeout)
|
||||
if not acquired:
|
||||
return False
|
||||
if self._depth:
|
||||
self._depth += 1
|
||||
return True
|
||||
try:
|
||||
if not blocking:
|
||||
self._queue.get_nowait()
|
||||
elif timeout == -1:
|
||||
self._queue.get()
|
||||
else:
|
||||
remaining = max(0, timeout - (self._time() - start))
|
||||
self._queue.get(timeout=remaining)
|
||||
except Empty:
|
||||
self._lock.release()
|
||||
return False
|
||||
self._owner = self.get_ident()
|
||||
self._depth = 1
|
||||
return True
|
||||
|
||||
def release(self):
|
||||
if self._owner != self.get_ident():
|
||||
raise RuntimeError("cannot release un-acquired lock")
|
||||
self._depth -= 1
|
||||
if not self._depth:
|
||||
self._owner = None
|
||||
self._queue.put(None)
|
||||
self._lock.release()
|
||||
|
||||
def __enter__(self):
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.release()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def ensure_lock(tqdm_class, lock_name="", lock=None):
|
||||
"""get (create if necessary) and then restore `tqdm_class`'s lock"""
|
||||
old_lock = getattr(tqdm_class, '_lock', None) # don't create a new lock
|
||||
if lock is None:
|
||||
lock = old_lock or tqdm_class.get_lock() # maybe create a new lock
|
||||
lock = getattr(lock, lock_name, lock) # maybe subtype
|
||||
tqdm_class.set_lock(lock)
|
||||
yield lock
|
||||
if old_lock is None:
|
||||
del tqdm_class._lock
|
||||
else:
|
||||
tqdm_class.set_lock(old_lock)
|
||||
|
||||
|
||||
def _get_interpreter_init(tqdm_class, lock_queue_id):
|
||||
"""Return an initializer which bootstraps the parent import path and lock."""
|
||||
code = (
|
||||
"import sys\n"
|
||||
f"sys.path[:] = {sys.path!r}\n"
|
||||
"from concurrent import interpreters\n"
|
||||
"from importlib import import_module\n"
|
||||
"from tqdm.contrib.concurrent import _InterpreterLock\n"
|
||||
f"tqdm_class = import_module({tqdm_class.__module__!r})\n"
|
||||
f"for name in {tqdm_class.__qualname__.split('.')!r}:\n"
|
||||
" tqdm_class = getattr(tqdm_class, name)\n"
|
||||
"tqdm_class.monitor_interval = 0\n"
|
||||
f"tqdm_class.set_lock(_InterpreterLock(interpreters.Queue({lock_queue_id!r})))")
|
||||
return exec, (code,)
|
||||
|
||||
|
||||
def _min_map_len(iterables):
|
||||
"""min(map(length_hint, iterables), default=0)"""
|
||||
return min((n for it in iterables if (n := length_hint(it, -1)) >= 0), default=0)
|
||||
|
||||
|
||||
def _executor_map(
|
||||
PoolExecutor, fn, *iterables, max_workers=None, timeout=None, chunksize=1, lock_name="",
|
||||
tqdm_class=tqdm_auto, smoothing=0.0, _lock=None, _initializer=None, _initargs=None,
|
||||
**tqdm_kwargs
|
||||
):
|
||||
"""
|
||||
Implementation of `thread_map`, `process_map` and `interpreter_map`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
max_workers : int
|
||||
timeout : int
|
||||
buffersize : int
|
||||
Requires Python>=3.14.
|
||||
thread_name_prefix : str
|
||||
max_tasks_per_child : int
|
||||
mp_context : str
|
||||
"""
|
||||
kwargs = tqdm_kwargs.copy()
|
||||
if 'total' not in kwargs:
|
||||
kwargs['total'] = _min_map_len(iterables)
|
||||
map_kwargs = {}
|
||||
if 'buffersize' in kwargs:
|
||||
map_kwargs['buffersize'] = kwargs.pop('buffersize')
|
||||
pool_kwargs = {}
|
||||
for k in ('thread_name_prefix', 'max_tasks_per_child', 'mp_context'):
|
||||
if k in kwargs:
|
||||
pool_kwargs[k] = kwargs.pop(k)
|
||||
dynamic_miniters = None
|
||||
if kwargs['total'] and 'miniters' not in kwargs:
|
||||
try:
|
||||
from os import process_cpu_count as cpu_count
|
||||
except ImportError:
|
||||
from os import cpu_count
|
||||
# thread & process pools have different default workers, but we KISS here
|
||||
rough_max = max_workers or min(32, (cpu_count() or 1) + 4)
|
||||
if kwargs['total'] > rough_max:
|
||||
kwargs['miniters'] = rough_max
|
||||
dynamic_miniters = True
|
||||
with ensure_lock(tqdm_class, lock_name=lock_name, lock=_lock) as lk:
|
||||
# share lock in case workers are already using `tqdm`
|
||||
if _initializer is None:
|
||||
_initializer = tqdm_class.set_lock
|
||||
_initargs = (lk,)
|
||||
with PoolExecutor(max_workers=max_workers, initializer=_initializer, initargs=_initargs,
|
||||
**pool_kwargs) as ex:
|
||||
with tqdm_class(smoothing=smoothing, **kwargs) as pbar:
|
||||
if dynamic_miniters is not None:
|
||||
pbar.dynamic_miniters = True
|
||||
orisubmit = ex.submit
|
||||
|
||||
def patchsubmit(*args, **kwargs):
|
||||
fut = orisubmit(*args, **kwargs)
|
||||
fut.add_done_callback(lambda _: pbar.update())
|
||||
return fut
|
||||
ex.submit = patchsubmit
|
||||
return list(ex.map(
|
||||
fn, *iterables, timeout=timeout, chunksize=chunksize, **map_kwargs))
|
||||
|
||||
|
||||
def thread_map(fn, *iterables, **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of `list(map(fn, *iterables))`
|
||||
driven by `concurrent.futures.ThreadPoolExecutor`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
max_workers : int, optional
|
||||
Maximum number of workers to spawn; passed to `concurrent.futures.ThreadPoolExecutor`.
|
||||
thread_name_prefix : str, optional
|
||||
Passed to `concurrent.futures.ThreadPoolExecutor` [default: ''].
|
||||
timeout : int or float, optional
|
||||
Seconds to wait before raising `TimeoutError` if `__next__` is called and the
|
||||
result isn't available. [default: None].
|
||||
buffersize : int, optional
|
||||
Requires Python>=3.14 [default: None].
|
||||
tqdm_class : optional
|
||||
`tqdm` class to use for bars [default: tqdm.auto.tqdm].
|
||||
smoothing : float, optional
|
||||
Passed to `tqdm_class`; the [default: 0] is average (due to erratic update frequency).
|
||||
lock_name : str, optional
|
||||
Member of `tqdm_class.get_lock()` to use [default: ''].
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
|
||||
|
||||
|
||||
def interpreter_map(fn, *iterables, **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of `list(map(fn, *iterables))`
|
||||
driven by `concurrent.futures.InterpreterPoolExecutor` (Python 3.14+).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
Same as `thread_map`.
|
||||
|
||||
Notes
|
||||
-----
|
||||
`fn`, its arguments, and its return values must be pickleable.
|
||||
Worker progress bars using the same `tqdm_class` share a cross-interpreter write lock.
|
||||
"""
|
||||
from concurrent import interpreters
|
||||
from concurrent.futures import InterpreterPoolExecutor
|
||||
lock_queue = interpreters.create_queue()
|
||||
lock_queue.put(None)
|
||||
tqdm_class = tqdm_kwargs.get("tqdm_class", tqdm_auto)
|
||||
initializer, initargs = _get_interpreter_init(tqdm_class, lock_queue.id)
|
||||
return _executor_map(
|
||||
InterpreterPoolExecutor, fn, *iterables, _lock=_InterpreterLock(lock_queue),
|
||||
_initializer=initializer, _initargs=initargs, **tqdm_kwargs)
|
||||
|
||||
|
||||
def process_map(fn, *iterables, lock_name="mp_lock", **tqdm_kwargs):
|
||||
"""
|
||||
Equivalent of `list(map(fn, *iterables))`
|
||||
driven by `concurrent.futures.ProcessPoolExecutor`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
max_workers : int, optional
|
||||
Maximum number of workers to spawn; passed to `concurrent.futures.ProcessPoolExecutor`.
|
||||
timeout : int or float, optional
|
||||
Seconds to wait before raising `TimeoutError` if `__next__` is called and the
|
||||
result isn't available. [default: None].
|
||||
chunksize : int, optional
|
||||
Approximate size of chunks sent to worker processes; passed to
|
||||
`concurrent.futures.ProcessPoolExecutor.map`. [default: 1].
|
||||
buffersize : int, optional
|
||||
Requires Python>=3.14 [default: None].
|
||||
max_tasks_per_child : int, optional
|
||||
Maximum number of tasks a worker process can complete before being replaced
|
||||
with a new process; passed to `concurrent.futures.ProcessPoolExecutor`.
|
||||
mp_context : multiprocessing.BaseContext, optional
|
||||
Multiprocessing context to use, e.g. `multiprocessing.get_context('fork')`.
|
||||
lock_name : str, optional
|
||||
Member of `tqdm_class.get_lock()` to use [default: mp_lock].
|
||||
tqdm_class : optional
|
||||
`tqdm` class to use for bars [default: tqdm.auto.tqdm].
|
||||
smoothing : float, optional
|
||||
Passed to `tqdm_class`; the [default: 0] is average (due to erratic update frequency).
|
||||
"""
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
if iterables and 'chunksize' not in tqdm_kwargs:
|
||||
# default `chunksize=1` has poor performance for large iterables
|
||||
# (most time spent dispatching items to workers).
|
||||
shortest_iterable_len = _min_map_len(iterables)
|
||||
if shortest_iterable_len > 1000:
|
||||
from warnings import warn
|
||||
warn("Iterable length %d > 1000 but `chunksize` is not set."
|
||||
" This may seriously degrade multiprocess performance."
|
||||
" Set `chunksize=1` or more." % shortest_iterable_len,
|
||||
TqdmWarning, stacklevel=2)
|
||||
return _executor_map(ProcessPoolExecutor, fn, *iterables, lock_name=lock_name, **tqdm_kwargs)
|
||||
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Sends updates to a Discord bot.
|
||||
|
||||
Usage:
|
||||
>>> from tqdm.contrib.discord import tqdm, trange
|
||||
>>> for i in trange(10, token='{token}', channel_id='{channel_id}'):
|
||||
... ...
|
||||
|
||||

|
||||
"""
|
||||
from warnings import warn
|
||||
|
||||
from requests import Session
|
||||
from requests.utils import default_user_agent
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
from ..std import TqdmWarning
|
||||
from ..utils import envwrap
|
||||
from ..version import __version__
|
||||
from .utils_worker import MonoWorker
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl", "guigoruiz1"]}
|
||||
__all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange']
|
||||
|
||||
|
||||
class DiscordIO(MonoWorker):
|
||||
"""Non-blocking file-like IO using a Discord Bot."""
|
||||
API = 'https://discord.com/api/v10'
|
||||
UA = f"tqdm (https://tqdm.github.io, {__version__}) {default_user_agent()}"
|
||||
|
||||
def __init__(self, token, channel_id):
|
||||
"""Creates a new message in the given `channel_id`."""
|
||||
super().__init__()
|
||||
self.token = token
|
||||
self.channel_id = channel_id
|
||||
self.session = Session()
|
||||
self.text = self.__class__.__name__
|
||||
self.message_id # pylint: disable=pointless-statement
|
||||
|
||||
@property
|
||||
def message_id(self):
|
||||
if hasattr(self, '_message_id'):
|
||||
return self._message_id # pylint: disable=access-member-before-definition
|
||||
try:
|
||||
req = self.session.post(
|
||||
f'{self.API}/channels/{self.channel_id}/messages',
|
||||
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA},
|
||||
json={'content': f"`{self.text}`"})
|
||||
res = req.json()
|
||||
req.raise_for_status()
|
||||
except Exception as e:
|
||||
if req.status_code == 429:
|
||||
warn("Creation rate limit: try increasing `mininterval`.",
|
||||
TqdmWarning, stacklevel=2)
|
||||
else:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
self._message_id = res['id']
|
||||
return self._message_id
|
||||
|
||||
def write(self, s):
|
||||
"""Replaces internal `message_id`'s text with `s`."""
|
||||
if not s:
|
||||
s = "..."
|
||||
s = s.replace('\r', '').strip()
|
||||
if s == self.text:
|
||||
return # avoid duplicate message Bot error
|
||||
message_id = self.message_id
|
||||
if message_id is None:
|
||||
return
|
||||
self.text = s
|
||||
try:
|
||||
future = self.submit(
|
||||
self.session.patch,
|
||||
f'{self.API}/channels/{self.channel_id}/messages/{message_id}',
|
||||
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA},
|
||||
json={'content': f"`{self.text}`"})
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
return future
|
||||
|
||||
def delete(self):
|
||||
"""Deletes internal `message_id`."""
|
||||
try:
|
||||
future = self.submit(
|
||||
self.session.delete,
|
||||
f'{self.API}/channels/{self.channel_id}/messages/{self.message_id}',
|
||||
headers={'Authorization': f'Bot {self.token}', 'User-Agent': self.UA})
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
return future
|
||||
|
||||
|
||||
class tqdm_discord(tqdm_auto): # pylint: disable=inconsistent-mro
|
||||
"""
|
||||
Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot.
|
||||
May take a few seconds to create (`__init__`).
|
||||
|
||||
- create a discord bot (not public, no requirement of OAuth2 code
|
||||
grant, only send message permissions) & invite it to a channel:
|
||||
<https://discordpy.readthedocs.io/en/latest/discord.html>
|
||||
- copy the bot `{token}` & `{channel_id}` and paste below
|
||||
|
||||
>>> from tqdm.contrib.discord import tqdm, trange
|
||||
>>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'):
|
||||
... ...
|
||||
"""
|
||||
@envwrap("tqdm", "discord", is_method=True)
|
||||
def __init__(self, *args, token=None, channel_id=None, **kwargs):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
token : str, required. Discord bot token
|
||||
[default: ${TQDM_DISCORD_TOKEN}].
|
||||
channel_id : int, required. Discord channel ID
|
||||
[default: ${TQDM_DISCORD_CHANNEL_ID}].
|
||||
|
||||
See `tqdm.auto.tqdm.__init__` for other parameters.
|
||||
"""
|
||||
if not kwargs.get('disable'):
|
||||
kwargs = kwargs.copy()
|
||||
self.dio = DiscordIO(token, channel_id)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def display(self, **kwargs): # pylint: disable=arguments-differ
|
||||
super().display(**kwargs)
|
||||
fmt = self.format_dict
|
||||
if fmt.get('bar_format', None):
|
||||
fmt['bar_format'] = fmt['bar_format'].replace(
|
||||
'<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}')
|
||||
self.dio.write(self.format_meter(**fmt))
|
||||
|
||||
def clear(self, *args, **kwargs):
|
||||
super().clear(*args, **kwargs)
|
||||
if not self.disable:
|
||||
self.dio.write("")
|
||||
|
||||
def close(self):
|
||||
if self.disable:
|
||||
return
|
||||
super().close()
|
||||
if not (self.leave or (self.leave is None and self.pos == 0)):
|
||||
self.dio.delete()
|
||||
|
||||
|
||||
def tdrange(*args, **kwargs):
|
||||
"""Shortcut for `tqdm.contrib.discord.tqdm(range(*args), **kwargs)`."""
|
||||
return tqdm_discord(range(*args), **kwargs)
|
||||
|
||||
|
||||
# Aliases
|
||||
tqdm = tqdm_discord
|
||||
trange = tdrange
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Thin wrappers around `itertools`.
|
||||
"""
|
||||
import itertools
|
||||
import math
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl"]}
|
||||
__all__ = [
|
||||
'chain', 'product', 'permutations', 'combinations', 'combinations_with_replacement', 'batched']
|
||||
|
||||
|
||||
def chain(*iterables, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.chain`."""
|
||||
if total is None:
|
||||
try:
|
||||
total = sum(map(len, iterables))
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
return tqdm_class(itertools.chain(*iterables), total=total, **kwargs)
|
||||
|
||||
|
||||
def product(*iterables, repeat=1, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.product`."""
|
||||
if total is None:
|
||||
try:
|
||||
lens = list(map(len, iterables))
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
total = math.prod(lens) ** repeat
|
||||
yield from tqdm_class(itertools.product(*iterables, repeat=repeat), total=total, **kwargs)
|
||||
|
||||
|
||||
def permutations(iterable, r=None, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.permutations`."""
|
||||
if total is None:
|
||||
try:
|
||||
n = len(iterable)
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
r = n if r is None else r
|
||||
if r > n:
|
||||
total = 0
|
||||
else:
|
||||
total = math.perm(n, r)
|
||||
return tqdm_class(itertools.permutations(iterable, r), total=total, **kwargs)
|
||||
|
||||
|
||||
def combinations(iterable, r, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.combinations`."""
|
||||
if total is None:
|
||||
try:
|
||||
n = len(iterable)
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
if r > n:
|
||||
total = 0
|
||||
else:
|
||||
total = math.comb(n, r)
|
||||
return tqdm_class(itertools.combinations(iterable, r), total=total, **kwargs)
|
||||
|
||||
|
||||
def combinations_with_replacement(iterable, r, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.combinations_with_replacement`."""
|
||||
if total is None:
|
||||
try:
|
||||
n = len(iterable)
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
total = 1
|
||||
for i in range(n+r-1, n-1, -1):
|
||||
total *= i
|
||||
for i in range(1, r+1):
|
||||
total //= i
|
||||
return tqdm_class(itertools.combinations_with_replacement(iterable, r), total=total, **kwargs)
|
||||
|
||||
|
||||
def batched(iterable, n, total=None, tqdm_class=tqdm_auto, **kwargs):
|
||||
"""Equivalent of `itertools.batched`."""
|
||||
if total is None:
|
||||
try:
|
||||
total = len(iterable)
|
||||
except (TypeError, AttributeError):
|
||||
pass
|
||||
return tqdm_class(itertools.batched(iterable, n), unit_scale=n,
|
||||
total=(total+n-1) // n if total is not None else None,
|
||||
**kwargs)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
Helper functionality for interoperability with stdlib `logging`.
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
|
||||
try:
|
||||
from typing import Iterator, List, Optional, Type # noqa: F401, pylint: disable=unused-import
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from ..std import tqdm as std_tqdm
|
||||
|
||||
|
||||
class _TqdmLoggingHandler(logging.StreamHandler):
|
||||
def __init__(
|
||||
self,
|
||||
tqdm_class=std_tqdm # type: Type[std_tqdm]
|
||||
):
|
||||
super().__init__()
|
||||
self.tqdm_class = tqdm_class
|
||||
|
||||
def emit(self, record):
|
||||
try:
|
||||
msg = self.format(record)
|
||||
self.tqdm_class.write(msg, file=self.stream)
|
||||
self.flush()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except: # noqa pylint: disable=bare-except
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
def _is_console_logging_handler(handler):
|
||||
return (isinstance(handler, logging.StreamHandler)
|
||||
and handler.stream in {sys.stdout, sys.stderr})
|
||||
|
||||
|
||||
def _get_first_found_console_logging_handler(handlers):
|
||||
for handler in handlers:
|
||||
if _is_console_logging_handler(handler):
|
||||
return handler
|
||||
|
||||
|
||||
@contextmanager
|
||||
def logging_redirect_tqdm(
|
||||
loggers=None, # type: Optional[List[logging.Logger]],
|
||||
tqdm_class=std_tqdm # type: Type[std_tqdm]
|
||||
):
|
||||
# type: (...) -> Iterator[None]
|
||||
"""
|
||||
Context manager redirecting console logging to `tqdm.write()`, leaving
|
||||
other logging handlers (e.g. log files) unaffected.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
loggers : list, optional
|
||||
Which handlers to redirect (default: [logging.root]).
|
||||
tqdm_class : optional
|
||||
|
||||
Example
|
||||
-------
|
||||
```python
|
||||
import logging
|
||||
from tqdm import trange
|
||||
from tqdm.contrib.logging import logging_redirect_tqdm
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with logging_redirect_tqdm():
|
||||
for i in trange(9):
|
||||
if i == 4:
|
||||
LOG.info("console logging redirected to `tqdm.write()`")
|
||||
# logging restored
|
||||
```
|
||||
"""
|
||||
if loggers is None:
|
||||
loggers = [logging.root]
|
||||
original_handlers_list = [logger.handlers for logger in loggers]
|
||||
try:
|
||||
for logger in loggers:
|
||||
tqdm_handler = _TqdmLoggingHandler(tqdm_class)
|
||||
orig_handler = _get_first_found_console_logging_handler(logger.handlers)
|
||||
if orig_handler is not None:
|
||||
tqdm_handler.setFormatter(orig_handler.formatter)
|
||||
tqdm_handler.stream = orig_handler.stream
|
||||
for log_filter in orig_handler.filters:
|
||||
tqdm_handler.addFilter(log_filter)
|
||||
logger.handlers = [
|
||||
handler for handler in logger.handlers
|
||||
if not _is_console_logging_handler(handler)] + [tqdm_handler]
|
||||
yield
|
||||
finally:
|
||||
for logger, original_handlers in zip(loggers, original_handlers_list):
|
||||
logger.handlers = original_handlers
|
||||
|
||||
|
||||
@contextmanager
|
||||
def tqdm_logging_redirect(
|
||||
*args,
|
||||
# loggers=None, # type: Optional[List[logging.Logger]]
|
||||
# tqdm=None, # type: Optional[Type[tqdm.tqdm]]
|
||||
**kwargs
|
||||
):
|
||||
# type: (...) -> Iterator[None]
|
||||
"""
|
||||
Convenience shortcut for:
|
||||
```python
|
||||
with tqdm_class(*args, **tqdm_kwargs) as pbar:
|
||||
with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class):
|
||||
yield pbar
|
||||
```
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tqdm_class : optional, (default: tqdm.std.tqdm).
|
||||
loggers : optional, list.
|
||||
**tqdm_kwargs : passed to `tqdm_class`.
|
||||
"""
|
||||
tqdm_kwargs = kwargs.copy()
|
||||
loggers = tqdm_kwargs.pop('loggers', None)
|
||||
tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm)
|
||||
with tqdm_class(*args, **tqdm_kwargs) as pbar:
|
||||
with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class):
|
||||
yield pbar
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Sends updates to a Slack app.
|
||||
|
||||
Usage:
|
||||
>>> from tqdm.contrib.slack import tqdm, trange
|
||||
>>> for i in trange(10, token='{token}', channel='{channel}'):
|
||||
... ...
|
||||
|
||||

|
||||
"""
|
||||
import logging
|
||||
|
||||
try:
|
||||
from slack_sdk import WebClient
|
||||
except ImportError:
|
||||
raise ImportError("Please `pip install slack-sdk`")
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
from ..utils import envwrap
|
||||
from .utils_worker import MonoWorker
|
||||
|
||||
__author__ = {"github.com/": ["0x2b3bfa0", "casperdcl"]}
|
||||
__all__ = ['SlackIO', 'tqdm_slack', 'tsrange', 'tqdm', 'trange']
|
||||
|
||||
|
||||
class SlackIO(MonoWorker):
|
||||
"""Non-blocking file-like IO using a Slack app."""
|
||||
def __init__(self, token, channel):
|
||||
"""Creates a new message in the given `channel`."""
|
||||
super().__init__()
|
||||
self.client = WebClient(token=token)
|
||||
self.text = self.__class__.__name__
|
||||
try:
|
||||
self.message = self.client.chat_postMessage(channel=channel, text=self.text)
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
self.message = None
|
||||
|
||||
def write(self, s):
|
||||
"""Replaces internal `message`'s text with `s`."""
|
||||
if not s:
|
||||
s = "..."
|
||||
s = s.replace('\r', '').strip()
|
||||
if s == self.text:
|
||||
return # skip duplicate message
|
||||
message = self.message
|
||||
if message is None:
|
||||
return
|
||||
self.text = s
|
||||
try:
|
||||
future = self.submit(self.client.chat_update, channel=message['channel'],
|
||||
ts=message['ts'], text='`' + s + '`')
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
return future
|
||||
|
||||
|
||||
class tqdm_slack(tqdm_auto): # pylint: disable=inconsistent-mro
|
||||
"""
|
||||
Standard `tqdm.auto.tqdm` but also sends updates to a Slack app.
|
||||
May take a few seconds to create (`__init__`).
|
||||
|
||||
- create a Slack app with the `chat:write` scope & invite it to a
|
||||
channel: <https://api.slack.com/authentication/basics>
|
||||
- copy the bot `{token}` & `{channel}` and paste below
|
||||
>>> from tqdm.contrib.slack import tqdm, trange
|
||||
>>> for i in tqdm(iterable, token='{token}', channel='{channel}'):
|
||||
... ...
|
||||
"""
|
||||
@envwrap("tqdm", "slack", is_method=True)
|
||||
def __init__(self, *args, token=None, channel=None, **kwargs):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
token : str, required. Slack token
|
||||
[default: ${TQDM_SLACK_TOKEN}].
|
||||
channel : int, required. Slack channel
|
||||
[default: ${TQDM_SLACK_CHANNEL}].
|
||||
mininterval : float, optional.
|
||||
Minimum of [default: 1.5] to avoid rate limit.
|
||||
|
||||
See `tqdm.auto.tqdm.__init__` for other parameters.
|
||||
"""
|
||||
if not kwargs.get('disable'):
|
||||
kwargs = kwargs.copy()
|
||||
logging.getLogger("HTTPClient").setLevel(logging.WARNING)
|
||||
self.sio = SlackIO(token, channel)
|
||||
kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5))
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def display(self, **kwargs): # pylint: disable=arguments-differ
|
||||
super().display(**kwargs)
|
||||
fmt = self.format_dict
|
||||
if fmt.get('bar_format', None):
|
||||
fmt['bar_format'] = fmt['bar_format'].replace(
|
||||
'<bar/>', '`{bar:10}`').replace('{bar}', '`{bar:10u}`')
|
||||
elif self.total:
|
||||
fmt['bar_format'] = '{l_bar}`{bar:10}`{r_bar}'
|
||||
if fmt['ascii'] is False:
|
||||
fmt['ascii'] = [":black_square:", ":small_blue_diamond:", ":large_blue_diamond:",
|
||||
":large_blue_square:"]
|
||||
fmt['ncols'] = 336
|
||||
self.sio.write(self.format_meter(**fmt))
|
||||
|
||||
def clear(self, *args, **kwargs):
|
||||
super().clear(*args, **kwargs)
|
||||
if not self.disable:
|
||||
self.sio.write("")
|
||||
|
||||
|
||||
def tsrange(*args, **kwargs):
|
||||
"""Shortcut for `tqdm.contrib.slack.tqdm(range(*args), **kwargs)`."""
|
||||
return tqdm_slack(range(*args), **kwargs)
|
||||
|
||||
|
||||
# Aliases
|
||||
tqdm = tqdm_slack
|
||||
trange = tsrange
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Sends updates to a Telegram bot.
|
||||
|
||||
Usage:
|
||||
>>> from tqdm.contrib.telegram import tqdm, trange
|
||||
>>> for i in trange(10, token='{token}', chat_id='{chat_id}'):
|
||||
... ...
|
||||
|
||||

|
||||
"""
|
||||
from warnings import warn
|
||||
|
||||
from requests import Session
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
from ..std import TqdmWarning
|
||||
from ..utils import envwrap
|
||||
from .utils_worker import MonoWorker
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl"]}
|
||||
__all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange']
|
||||
|
||||
|
||||
class TelegramIO(MonoWorker):
|
||||
"""Non-blocking file-like IO using a Telegram Bot."""
|
||||
API = 'https://api.telegram.org/bot'
|
||||
|
||||
def __init__(self, token, chat_id):
|
||||
"""Creates a new message in the given `chat_id`."""
|
||||
super().__init__()
|
||||
self.token = token
|
||||
self.chat_id = chat_id
|
||||
self.session = Session()
|
||||
self.text = self.__class__.__name__
|
||||
self.message_id # pylint: disable=pointless-statement
|
||||
|
||||
@property
|
||||
def message_id(self):
|
||||
if hasattr(self, '_message_id'):
|
||||
return self._message_id # pylint: disable=access-member-before-definition
|
||||
try:
|
||||
req = self.session.post(
|
||||
f'{self.API}{self.token}/sendMessage',
|
||||
data={'text': f"`{self.text}`", 'chat_id': self.chat_id,
|
||||
'parse_mode': 'MarkdownV2'})
|
||||
res = req.json()
|
||||
req.raise_for_status()
|
||||
except Exception as e:
|
||||
if req.status_code == 429:
|
||||
warn("Creation rate limit: try increasing `mininterval`.",
|
||||
TqdmWarning, stacklevel=2)
|
||||
else:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
self._message_id = res['result']['message_id']
|
||||
return self._message_id
|
||||
|
||||
def write(self, s):
|
||||
"""Replaces internal `message_id`'s text with `s`."""
|
||||
if not s:
|
||||
s = "..."
|
||||
s = s.replace('\r', '').strip()
|
||||
if s == self.text:
|
||||
return # avoid duplicate message Bot error
|
||||
message_id = self.message_id
|
||||
if message_id is None:
|
||||
return
|
||||
self.text = s
|
||||
try:
|
||||
future = self.submit(
|
||||
self.session.post, f'{self.API}{self.token}/editMessageText',
|
||||
data={'text': f"`{s}`", 'chat_id': self.chat_id,
|
||||
'message_id': message_id, 'parse_mode': 'MarkdownV2'})
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
return future
|
||||
|
||||
def delete(self):
|
||||
"""Deletes internal `message_id`."""
|
||||
try:
|
||||
future = self.submit(
|
||||
self.session.post, '{self.API}{self.token}/deleteMessage',
|
||||
data={'chat_id': self.chat_id, 'message_id': self.message_id})
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
return future
|
||||
|
||||
|
||||
class tqdm_telegram(tqdm_auto): # pylint: disable=inconsistent-mro
|
||||
"""
|
||||
Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot.
|
||||
May take a few seconds to create (`__init__`).
|
||||
|
||||
- create a bot <https://core.telegram.org/bots#6-botfather>
|
||||
- copy its `{token}`
|
||||
- add the bot to a chat and send it a message such as `/start`
|
||||
- go to <https://api.telegram.org/bot`{token}`/getUpdates> to find out
|
||||
the `{chat_id}`
|
||||
- paste the `{token}` & `{chat_id}` below
|
||||
|
||||
>>> from tqdm.contrib.telegram import tqdm, trange
|
||||
>>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'):
|
||||
... ...
|
||||
"""
|
||||
@envwrap("tqdm", "telegram", is_method=True)
|
||||
def __init__(self, *args, token=None, chat_id=None, **kwargs):
|
||||
"""
|
||||
Parameters
|
||||
----------
|
||||
token : str, required. Telegram token
|
||||
[default: ${TQDM_TELEGRAM_TOKEN}].
|
||||
chat_id : str, required. Telegram chat ID
|
||||
[default: ${TQDM_TELEGRAM_CHAT_ID}].
|
||||
|
||||
See `tqdm.auto.tqdm.__init__` for other parameters.
|
||||
"""
|
||||
if not kwargs.get('disable'):
|
||||
kwargs = kwargs.copy()
|
||||
self.tgio = TelegramIO(token, chat_id)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def display(self, **kwargs): # pylint: disable=arguments-differ
|
||||
super().display(**kwargs)
|
||||
fmt = self.format_dict
|
||||
if fmt.get('bar_format', None):
|
||||
fmt['bar_format'] = fmt['bar_format'].replace(
|
||||
'<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}')
|
||||
self.tgio.write(self.format_meter(**fmt))
|
||||
|
||||
def clear(self, *args, **kwargs):
|
||||
super().clear(*args, **kwargs)
|
||||
if not self.disable:
|
||||
self.tgio.write("")
|
||||
|
||||
def close(self):
|
||||
if self.disable:
|
||||
return
|
||||
super().close()
|
||||
if not (self.leave or (self.leave is None and self.pos == 0)):
|
||||
self.tgio.delete()
|
||||
|
||||
|
||||
def ttgrange(*args, **kwargs):
|
||||
"""Shortcut for `tqdm.contrib.telegram.tqdm(range(*args), **kwargs)`."""
|
||||
return tqdm_telegram(range(*args), **kwargs)
|
||||
|
||||
|
||||
# Aliases
|
||||
tqdm = tqdm_telegram
|
||||
trange = ttgrange
|
||||
@@ -0,0 +1,38 @@
|
||||
"""
|
||||
IO/concurrency helpers for `tqdm.contrib`.
|
||||
"""
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from ..auto import tqdm as tqdm_auto
|
||||
|
||||
__author__ = {"github.com/": ["casperdcl"]}
|
||||
__all__ = ['MonoWorker']
|
||||
|
||||
|
||||
class MonoWorker:
|
||||
"""
|
||||
Supports one running task and one waiting task.
|
||||
The waiting task is the most recent submitted (others are discarded).
|
||||
"""
|
||||
def __init__(self):
|
||||
self.pool = ThreadPoolExecutor(max_workers=1)
|
||||
self.futures = deque([], 2)
|
||||
|
||||
def submit(self, func, *args, **kwargs):
|
||||
"""`func(*args, **kwargs)` may replace currently waiting task."""
|
||||
futures = self.futures
|
||||
if len(futures) == futures.maxlen:
|
||||
running = futures.popleft()
|
||||
if not running.done():
|
||||
if len(futures): # clear waiting
|
||||
waiting = futures.pop()
|
||||
waiting.cancel()
|
||||
futures.appendleft(running) # re-insert running
|
||||
try:
|
||||
waiting = self.pool.submit(func, *args, **kwargs)
|
||||
except Exception as e:
|
||||
tqdm_auto.write(str(e))
|
||||
else:
|
||||
futures.append(waiting)
|
||||
return waiting
|
||||
Reference in New Issue
Block a user