From 7d263b0d2271947003907a2191d5589e11a3320e Mon Sep 17 00:00:00 2001 From: Ruben van Eldik Date: Wed, 23 Oct 2024 14:50:04 +0200 Subject: [PATCH 1/4] Add typing (cherry picked from commit f113e8f55cc47f392f69e8d30c6a5fca1b9b5833) --- .gitignore | 1 + logtail/compat.py | 2 +- mypy.ini | 2 ++ py.typed | 0 test-requirements.txt | 3 +++ tox.ini | 1 + 6 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 mypy.ini create mode 100644 py.typed diff --git a/.gitignore b/.gitignore index f2ee7f9..384189a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__ htmlcov .coverage .tox +.idea *.egg-info build/* dist/* diff --git a/logtail/compat.py b/logtail/compat.py index 1a4f794..4ff47e8 100644 --- a/logtail/compat.py +++ b/logtail/compat.py @@ -4,4 +4,4 @@ try: import queue except ImportError: - import Queue as queue + import Queue as queue # type: ignore[import-not-found, no-redef] diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..515be77 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +files = logtail diff --git a/py.typed b/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/test-requirements.txt b/test-requirements.txt index 5dbaaae..f044253 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,6 @@ httpretty>=0.9.4 nose-py3 mock>=1.0.1 rq +mypy>=1.13.0 +types-setuptools>=75.2.0.20241019 +msgpack-types>=0.5.0 diff --git a/tox.ini b/tox.ini index 7a659b4..e816774 100644 --- a/tox.ini +++ b/tox.ini @@ -15,3 +15,4 @@ deps = -rrequirements.txt commands = nosetests --with-coverage --cover-branches --cover-package=logtail + mypy logtail From 9c46735e8457ca37f90803e5d510e0093cd1485d Mon Sep 17 00:00:00 2001 From: Ruben van Eldik Date: Wed, 23 Oct 2024 14:55:12 +0200 Subject: [PATCH 2/4] Add types-requests (cherry picked from commit 20fe01c72b9aec00606138afc741ac841feebcb0) --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index f044253..ca7f75c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,4 @@ rq mypy>=1.13.0 types-setuptools>=75.2.0.20241019 msgpack-types>=0.5.0 +types-requests>=2.32.0.20241016 From 9465de016730e9e236fc5e260fdc95f4a8d540c4 Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 25 Sep 2026 15:10:13 +0200 Subject: [PATCH 3/4] Ship py.typed inside the package, annotate the public API and type-check in CI py.typed only counts when it is installed next to the package's modules, so it moves into logtail/ and setup.py packages it. Every function in the package is annotated and mypy runs as its own CI job with disallow_untyped_defs, so strict type checkers see real signatures instead of untyped calls. LogtailContext.__exit__ now returns None instead of the context object; it only ever returned that when there was no exception to swallow, so nothing changes at runtime. Co-Authored-By: Ruben van Eldik Co-Authored-By: Claude Fable 5.1 --- .github/workflows/main.yml | 15 +++++++++++++ logtail/flusher.py | 25 +++++++++++----------- logtail/formatter.py | 11 +++++----- logtail/frame.py | 30 +++++++++++++++----------- logtail/handler.py | 41 ++++++++++++++++++------------------ logtail/helpers.py | 26 +++++++++++------------ py.typed => logtail/py.typed | 0 logtail/rq.py | 4 ++-- logtail/uploader.py | 7 +++--- mypy.ini | 2 ++ setup.py | 1 + test-requirements.txt | 1 - tox.ini | 1 - 13 files changed, 95 insertions(+), 69 deletions(-) rename py.typed => logtail/py.typed (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc7b349..d912957 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,3 +27,18 @@ jobs: pip install tox tox-gh-actions - name: Test with tox run: tox + + typecheck: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r test-requirements.txt + - name: Type-check with mypy + run: mypy diff --git a/logtail/flusher.py b/logtail/flusher.py index d80d2e1..d8196f4 100644 --- a/logtail/flusher.py +++ b/logtail/flusher.py @@ -3,13 +3,14 @@ import threading import time +from typing import Any, Callable from .compat import queue RETRY_SCHEDULE = (1, 10, 60) # seconds -class TransportFrame(dict): +class TransportFrame(dict[str, Any]): """A record logged by a flush worker itself, i.e. urllib3 describing our own upload. It rides along with the next batch of regular records but never triggers an upload @@ -18,7 +19,7 @@ class TransportFrame(dict): class FlushWorker(threading.Thread): - def __init__(self, upload, pipe, buffer_capacity, flush_interval, check_interval): + def __init__(self, upload: Callable[[list[Any]], Any], pipe: 'queue.Queue[Any]', buffer_capacity: int, flush_interval: float, check_interval: float) -> None: threading.Thread.__init__(self) self.parent_thread = threading.current_thread() self.upload = upload @@ -30,11 +31,11 @@ def __init__(self, upload, pipe, buffer_capacity, flush_interval, check_interval self._flushing = False self._clean = True - def run(self): + def run(self) -> None: while self.should_run: self.step() - def _is_parent_alive(self): + def _is_parent_alive(self) -> bool: try: return self.parent_thread.is_alive() except RuntimeError: @@ -44,10 +45,10 @@ def _is_parent_alive(self): # interpreter shutdown). Treat that as "no longer alive". return False - def step(self): + def step(self) -> None: last_flush = time.time() time_remaining = _initial_time_remaining(self.flush_interval) - frame = [] + frame: list[Any] = [] self._clean = True # If the parent thread has exited but there are still outstanding @@ -80,7 +81,7 @@ def step(self): # request fails in a way that can be retried, it is retried with an # exponential backoff in between attempts. if frame and not all(isinstance(entry, TransportFrame) for entry in frame): - response = None + response: Any = None for delay in RETRY_SCHEDULE + (None, ): response = self.upload(frame) if not _should_retry(response.status_code): @@ -100,25 +101,25 @@ def step(self): if shutdown and self.pipe.empty(): self.should_run = False - def flush(self): + def flush(self) -> None: self._flushing = True while not self._clean or not self.pipe.empty(): time.sleep(self.check_interval) self._flushing = False -def in_flush_worker(): +def in_flush_worker() -> bool: return isinstance(threading.current_thread(), FlushWorker) -def _initial_time_remaining(flush_interval): +def _initial_time_remaining(flush_interval: float) -> float: return flush_interval -def _calculate_time_remaining(last_flush, flush_interval): +def _calculate_time_remaining(last_flush: float, flush_interval: float) -> float: elapsed = time.time() - last_flush time_remaining = max(flush_interval - elapsed, 0) return time_remaining -def _should_retry(status_code): +def _should_retry(status_code: int) -> bool: return 500 <= status_code < 600 diff --git a/logtail/formatter.py b/logtail/formatter.py index 50f99d3..c76e105 100644 --- a/logtail/formatter.py +++ b/logtail/formatter.py @@ -2,21 +2,22 @@ from __future__ import print_function, unicode_literals import logging import json +from typing import Any, Callable, Optional -from .helpers import DEFAULT_CONTEXT +from .helpers import DEFAULT_CONTEXT, LogtailContext from .frame import create_frame class LogtailFormatter(logging.Formatter): def __init__(self, - context=DEFAULT_CONTEXT, - json_default=None, - json_encoder=None): + context: LogtailContext = DEFAULT_CONTEXT, + json_default: Optional[Callable[[Any], Any]] = None, + json_encoder: Optional[type[json.JSONEncoder]] = None) -> None: self.context = context self.json_default = json_default self.json_encoder = json_encoder - def format(self, record): + def format(self, record: logging.LogRecord) -> str: # Because the formatter does not have an underlying format string for # which `extra` may be used to substitute arguments (see # https://docs.python.org/2/library/logging.html#logging.debug ), we diff --git a/logtail/frame.py b/logtail/frame.py index 919ed5c..eac36ce 100644 --- a/logtail/frame.py +++ b/logtail/frame.py @@ -1,24 +1,29 @@ # coding: utf-8 from __future__ import print_function, unicode_literals from datetime import datetime, timezone - +import logging from os import path +from typing import Any, Optional import __main__ -def create_frame(record, message, context, include_extra_attributes=False): +from .helpers import LogtailContext + +def create_frame(record: logging.LogRecord, message: str, context: LogtailContext, include_extra_attributes: bool = False) -> dict[str, Any]: r = record.__dict__ # Django sends a request object in the record, which is not JSON serializable if "request" in r and not isinstance(r["request"], (dict, list, bool, int, float, str)) : del r["request"] - frame = {} + frame: dict[str, Any] = {} frame['dt'] = datetime.fromtimestamp(r['created'], timezone.utc).isoformat() frame['level'] = _levelname(r['levelname']) frame['severity'] = int(r['levelno'] / 10) frame['message'] = message - frame['context'] = ctx = {} + ctx: dict[str, Any] = {} + frame['context'] = ctx # Runtime context - ctx['runtime'] = runtime = {} + runtime: dict[str, Any] = {} + ctx['runtime'] = runtime runtime['function'] = r['funcName'] runtime['file'] = _relative_to_main_module_if_possible(r['pathname']) runtime['line'] = r['lineno'] @@ -27,7 +32,8 @@ def create_frame(record, message, context, include_extra_attributes=False): runtime['logger_name'] = r['name'] # Runtime context - ctx['system'] = system = {} + system: dict[str, Any] = {} + ctx['system'] = system system['pid'] = r['process'] system['process_name'] = r['processName'] @@ -41,14 +47,14 @@ def create_frame(record, message, context, include_extra_attributes=False): return _remove_circular_dependencies(frame) -def _parse_custom_events(record, include_extra_attributes): +def _parse_custom_events(record: logging.LogRecord, include_extra_attributes: bool) -> dict[str, Any]: default_keys = { 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'pathname', 'funcName', 'levelname', 'levelno', 'lineno', 'module', 'msecs', 'message', 'msg', 'name', 'process', 'processName', 'relativeCreated', 'thread', 'threadName' } - events = {} + events: dict[str, Any] = {} for key, val in record.__dict__.items(): if key in default_keys: continue @@ -57,7 +63,7 @@ def _parse_custom_events(record, include_extra_attributes): events[key] = val return events -def _remove_circular_dependencies(obj, memo=None): +def _remove_circular_dependencies(obj: Any, memo: Optional[set[int]] = None) -> Any: if memo is None: memo = set() @@ -85,12 +91,12 @@ def _remove_circular_dependencies(obj, memo=None): else: return obj -def _levelname(level): +def _levelname(level: str) -> str: return level.lower() -def _relative_to_main_module_if_possible(pathname): +def _relative_to_main_module_if_possible(pathname: str) -> str: has_main_module = hasattr(__main__, '__file__') return _relative_to_main_module(pathname) if has_main_module else pathname -def _relative_to_main_module(pathname): +def _relative_to_main_module(pathname: str) -> str: return path.relpath(pathname, path.dirname(__main__.__file__)) diff --git a/logtail/handler.py b/logtail/handler.py index 3c83c56..f1612f4 100644 --- a/logtail/handler.py +++ b/logtail/handler.py @@ -4,9 +4,10 @@ import json import os import weakref +from typing import Any, Optional from .compat import queue -from .helpers import DEFAULT_CONTEXT +from .helpers import DEFAULT_CONTEXT, LogtailContext from .flusher import FlushWorker, TransportFrame, in_flush_worker from .uploader import Uploader from .frame import create_frame @@ -20,22 +21,22 @@ DEFAULT_INCLUDE_EXTRA_ATTRIBUTES = True DEFAULT_TIMEOUT = 30 -_handlers = weakref.WeakSet() +_handlers: 'weakref.WeakSet[LogtailHandler]' = weakref.WeakSet() class LogtailHandler(logging.Handler): def __init__(self, - source_token, - host=DEFAULT_HOST, - buffer_capacity=DEFAULT_BUFFER_CAPACITY, - flush_interval=DEFAULT_FLUSH_INTERVAL, - check_interval=DEFAULT_CHECK_INTERVAL, - raise_exceptions=DEFAULT_RAISE_EXCEPTIONS, - drop_extra_events=DEFAULT_DROP_EXTRA_EVENTS, - include_extra_attributes=DEFAULT_INCLUDE_EXTRA_ATTRIBUTES, - context=DEFAULT_CONTEXT, - timeout=DEFAULT_TIMEOUT, - level=logging.NOTSET): + source_token: str, + host: str = DEFAULT_HOST, + buffer_capacity: int = DEFAULT_BUFFER_CAPACITY, + flush_interval: float = DEFAULT_FLUSH_INTERVAL, + check_interval: float = DEFAULT_CHECK_INTERVAL, + raise_exceptions: bool = DEFAULT_RAISE_EXCEPTIONS, + drop_extra_events: bool = DEFAULT_DROP_EXTRA_EVENTS, + include_extra_attributes: bool = DEFAULT_INCLUDE_EXTRA_ATTRIBUTES, + context: LogtailContext = DEFAULT_CONTEXT, + timeout: float = DEFAULT_TIMEOUT, + level: int = logging.NOTSET) -> None: super(LogtailHandler, self).__init__(level=level) self.source_token = source_token if host.startswith('https://') or host.startswith('http://'): @@ -43,7 +44,7 @@ def __init__(self, else: self.host = "https://" + host self.context = context - self.pipe = queue.Queue(maxsize=buffer_capacity) + self.pipe: 'queue.Queue[dict[str, Any]]' = queue.Queue(maxsize=buffer_capacity) self.uploader = Uploader(self.source_token, self.host, timeout) self.drop_extra_events = drop_extra_events self.include_extra_attributes = include_extra_attributes @@ -53,10 +54,10 @@ def __init__(self, self.raise_exceptions = raise_exceptions self.dropcount = 0 # Do not initialize the flush thread yet because it causes issues on Render. - self.flush_thread = None + self.flush_thread: Optional[FlushWorker] = None _handlers.add(self) - def ensure_flush_thread_alive(self): + def ensure_flush_thread_alive(self) -> None: if self.flush_thread and self.flush_thread.is_alive(): return @@ -69,7 +70,7 @@ def ensure_flush_thread_alive(self): ) self.flush_thread.start() - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: try: self.ensure_flush_thread_alive() @@ -90,11 +91,11 @@ def emit(self, record): if self.raise_exceptions: raise e - def flush(self): + def flush(self) -> None: if self.flush_thread and self.flush_thread.is_alive(): self.flush_thread.flush() - def _reset_after_fork(self): + def _reset_after_fork(self) -> None: # A forked child inherits the parent's queue with whatever was still buffered in it, # a flush thread object whose thread does not exist in the child, and an HTTP session # whose socket it shares with the parent, so it starts over with fresh ones. @@ -103,7 +104,7 @@ def _reset_after_fork(self): self.uploader = Uploader(self.source_token, self.host, self.uploader.timeout) -def _reset_handlers_after_fork(): +def _reset_handlers_after_fork() -> None: for handler in _handlers: handler._reset_after_fork() diff --git a/logtail/helpers.py b/logtail/helpers.py index bd026eb..65a1f21 100644 --- a/logtail/helpers.py +++ b/logtail/helpers.py @@ -1,12 +1,14 @@ # coding: utf-8 from __future__ import print_function, unicode_literals +from types import TracebackType +from typing import Any, Optional class LogtailContext(object): - def __init__(self): - self.extras = [] + def __init__(self) -> None: + self.extras: list[dict[str, dict[str, Any]]] = [] - def context(self, *args, **kwargs): + def context(self, *args: Any, **kwargs: dict[str, Any]) -> 'LogtailContext': if args: raise ValueError( 'All contexts must be passed by name as keyword arguments' @@ -19,23 +21,21 @@ def context(self, *args, **kwargs): self.extras.append(kwargs) return self - def __call__(self, *args, **kwargs): + def __call__(self, *args: Any, **kwargs: dict[str, Any]) -> 'LogtailContext': return self.context(*args, **kwargs) - def __enter__(self): + def __enter__(self) -> 'LogtailContext': return self - def __exit__(self, type_, value, traceback): - if type_ is not None: - return False - self.extras.pop() - return self + def __exit__(self, type_: Optional[type[BaseException]], value: Optional[BaseException], traceback: Optional[TracebackType]) -> None: + if type_ is None: + self.extras.pop() - def exists(self): + def exists(self) -> bool: return bool(self.extras) - def collapse(self): - x = {} + def collapse(self) -> dict[str, dict[str, Any]]: + x: dict[str, dict[str, Any]] = {} for contexts in self.extras: for name, data in contexts.items(): x.setdefault(name, {}).update(data) diff --git a/py.typed b/logtail/py.typed similarity index 100% rename from py.typed rename to logtail/py.typed diff --git a/logtail/rq.py b/logtail/rq.py index 86f0faf..6f213cc 100644 --- a/logtail/rq.py +++ b/logtail/rq.py @@ -16,7 +16,7 @@ class Worker(rq.Worker): in django-rq. """ - def perform_job(self, job, queue): + def perform_job(self, job: rq.job.Job, queue: rq.Queue) -> bool: try: return super(Worker, self).perform_job(job, queue) finally: @@ -24,6 +24,6 @@ def perform_job(self, job, queue): handler.flush() -def _logtail_handlers(): +def _logtail_handlers() -> set[LogtailHandler]: loggers = [logging.getLogger()] + [logger for logger in logging.Logger.manager.loggerDict.values() if isinstance(logger, logging.Logger)] return {handler for logger in loggers for handler in logger.handlers if isinstance(handler, LogtailHandler)} diff --git a/logtail/uploader.py b/logtail/uploader.py index 9d8dce6..868a542 100644 --- a/logtail/uploader.py +++ b/logtail/uploader.py @@ -1,15 +1,16 @@ # coding: utf-8 from __future__ import print_function, unicode_literals +from typing import Any, Union import msgpack import requests class Fake500(object): - def __init__(self, exception): + def __init__(self, exception: Exception) -> None: self.status_code = 500 self.exception = exception class Uploader(object): - def __init__(self, source_token, host, timeout): + def __init__(self, source_token: str, host: str, timeout: float) -> None: self.source_token = source_token self.host = host self.timeout = timeout @@ -19,7 +20,7 @@ def __init__(self, source_token, host, timeout): 'Content-Type': 'application/msgpack', } - def __call__(self, frame): + def __call__(self, frame: list[dict[str, Any]]) -> Union[requests.Response, Fake500]: data = msgpack.packb(frame, use_bin_type=True) try: return self.session.post(self.host, data=data, headers=self.headers, timeout=self.timeout) diff --git a/mypy.ini b/mypy.ini index 515be77..d5a8b64 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,4 @@ [mypy] files = logtail +disallow_untyped_defs = True +warn_unused_ignores = True diff --git a/setup.py b/setup.py index 66c7f87..aaa8e85 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ name='logtail-python', version=VERSION, packages=['logtail'], + package_data={'logtail': ['py.typed']}, include_package_data=True, license='ISC', description='Better Stack client library', diff --git a/test-requirements.txt b/test-requirements.txt index ca7f75c..8eae27f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,6 +4,5 @@ nose-py3 mock>=1.0.1 rq mypy>=1.13.0 -types-setuptools>=75.2.0.20241019 msgpack-types>=0.5.0 types-requests>=2.32.0.20241016 diff --git a/tox.ini b/tox.ini index e816774..7a659b4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,4 +15,3 @@ deps = -rrequirements.txt commands = nosetests --with-coverage --cover-branches --cover-package=logtail - mypy logtail From 49077316ff33f3a78ab335413658bdd9350fa62b Mon Sep 17 00:00:00 2001 From: Petr Heinz Date: Fri, 25 Sep 2026 15:36:34 +0200 Subject: [PATCH 4/4] Accept what the runtime accepts: level names and requests-style timeouts logging.Handler takes a level name as well as a number, and requests accepts a (connect, read) tuple or None as the timeout, so the annotations must not reject either. Co-Authored-By: Claude Fable 5.1 --- logtail/handler.py | 6 +++--- logtail/uploader.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/logtail/handler.py b/logtail/handler.py index e00d626..8a1f00f 100644 --- a/logtail/handler.py +++ b/logtail/handler.py @@ -4,7 +4,7 @@ import json import os import weakref -from typing import Any, Optional +from typing import Any, Optional, Union from .compat import queue from .helpers import DEFAULT_CONTEXT, LogtailContext @@ -37,9 +37,9 @@ def __init__(self, drop_extra_events: bool = DEFAULT_DROP_EXTRA_EVENTS, include_extra_attributes: bool = DEFAULT_INCLUDE_EXTRA_ATTRIBUTES, context: LogtailContext = DEFAULT_CONTEXT, - timeout: float = DEFAULT_TIMEOUT, + timeout: Union[float, tuple[float, float], None] = DEFAULT_TIMEOUT, flush_timeout: Optional[float] = DEFAULT_FLUSH_TIMEOUT, - level: int = logging.NOTSET) -> None: + level: Union[int, str] = logging.NOTSET) -> None: super(LogtailHandler, self).__init__(level=level) self.source_token = source_token if host.startswith('https://') or host.startswith('http://'): diff --git a/logtail/uploader.py b/logtail/uploader.py index 6d1dda7..9fbbf82 100644 --- a/logtail/uploader.py +++ b/logtail/uploader.py @@ -12,7 +12,7 @@ def __init__(self, exception: Exception) -> None: self.exception = exception class Uploader(object): - def __init__(self, source_token: str, host: str, timeout: float) -> None: + def __init__(self, source_token: str, host: str, timeout: Union[float, tuple[float, float], None]) -> None: self.source_token = source_token self.host = host self.timeout = timeout