Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/specify_cli/bundles/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ def components(self) -> list[ComponentRef]:
# -- construction ---------------------------------------------------------

@classmethod
def from_file(cls, path: Path) -> BundleManifest:
data = load_yaml(path)
def from_file(cls, path: Path, *, max_bytes: int | None = None) -> "BundleManifest":
data = load_yaml(path, max_bytes=max_bytes)
manifest = cls.from_dict(data)
manifest.source_path = Path(path)
return manifest
Expand Down
113 changes: 92 additions & 21 deletions src/specify_cli/bundles/packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import os
import re
import tempfile
import zipfile
from dataclasses import dataclass
from pathlib import Path
Expand All @@ -24,6 +25,11 @@
# Fixed member timestamp (zip epoch) for reproducible, byte-stable artifacts.
_FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0)

# Maximum size (in bytes) for any individual file added to the archive.
# This prevents a single oversized asset from exhausting memory during
# compression or transmission.
MAX_ZIP_MEMBER_BYTES = 10 * 1024 * 1024 # 10 MiB


@dataclass
class BuildResult:
Expand All @@ -48,7 +54,10 @@ def build_bundle(
"README.md describing it."
)

manifest = BundleManifest.from_file(manifest_path)
# The member loop below bounds every packaged file, but the manifest is
# parsed (fully read into memory) before that loop ever runs, so bound the
# read here as well.
manifest = BundleManifest.from_file(manifest_path, max_bytes=MAX_ZIP_MEMBER_BYTES)
report = validate_manifest(manifest)
if not report.ok:
raise BundlerError(
Expand Down Expand Up @@ -76,28 +85,84 @@ def build_bundle(
rf"^{re.escape(manifest.bundle.id)}-"
r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\.zip$"
)
# A leftover mkstemp staging file (<id>-<version>-<random>.tmp) from a
# previous killed build may sit inside out_dir — which defaults to the
# bundle source tree — and must never be re-packaged as a member.
staging_re = re.compile(
rf"^{re.escape(manifest.bundle.id)}-"
r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?"
r"-[0-9A-Za-z_]+\.tmp$"
)
files = _collect_files(
bundle_dir, skip=artifact_path, skip_dir=skip_dir, artifact_re=artifact_re
bundle_dir,
skip=artifact_path,
skip_dir=skip_dir,
artifact_re=artifact_re,
staging_re=staging_re,
)
with zipfile.ZipFile(artifact_path, "w", zipfile.ZIP_DEFLATED) as archive:
for file_path in files:
# Confinement: every packaged file must live under bundle_dir.
ensure_within(bundle_dir, file_path)
arcname = file_path.relative_to(bundle_dir).as_posix()
# Fixed timestamp so identical inputs yield a byte-for-byte
# identical artifact (reproducible builds).
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_TIMESTAMP)
info.compress_type = zipfile.ZIP_DEFLATED
# Reproducible, normalized permissions: preserve executability so
# bundled scripts (e.g. extension hook scripts) stay runnable after
# extraction, but collapse to two canonical modes (0755 when any
# execute bit is set on the source, otherwise 0644) so identical
# inputs yield a byte-for-byte identical artifact.
with file_path.open("rb") as fh:
st = os.fstat(fh.fileno())
mode = 0o755 if st.st_mode & 0o111 else 0o644
info.external_attr = mode << 16
archive.writestr(info, fh.read())

# Build into a temporary sibling and atomically replace the final path only
# after every member passes validation. This prevents a partial/corrupt
# archive from being left at the final output path if a member exceeds the
# size limit or another error occurs mid-build.
tmp_fd, tmp_path_str = tempfile.mkstemp(
suffix=".tmp", prefix=f"{manifest.bundle.id}-{manifest.bundle.version}-", dir=str(out_dir)
)
Comment thread
Copilot marked this conversation as resolved.
Comment thread
Copilot marked this conversation as resolved.
tmp_path = Path(tmp_path_str)
try:
with os.fdopen(tmp_fd, "wb") as tmp_fh, zipfile.ZipFile(tmp_fh, "w", zipfile.ZIP_DEFLATED) as archive:
for file_path in files:
# Confinement: every packaged file must live under bundle_dir.
ensure_within(bundle_dir, file_path)
arcname = file_path.relative_to(bundle_dir).as_posix()
# Fixed timestamp so identical inputs yield a byte-for-byte
# identical artifact (reproducible builds).
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_TIMESTAMP)
info.compress_type = zipfile.ZIP_DEFLATED
# Reproducible, normalized permissions: preserve executability so
# bundled scripts (e.g. extension hook scripts) stay runnable after
# extraction, but collapse to two canonical modes (0755 when any
# execute bit is set on the source, otherwise 0644) so identical
# inputs yield a byte-for-byte identical artifact.
with file_path.open("rb") as fh:
st = os.fstat(fh.fileno())
mode = 0o755 if st.st_mode & 0o111 else 0o644
info.external_attr = mode << 16
# Fast metadata rejection: skip files whose size exceeds the
# limit before touching the read path. Then also bound the
# actual read so a TOCTOU race (file appended after fstat)
# cannot bypass the limit.
if st.st_size > MAX_ZIP_MEMBER_BYTES:
Comment thread
Quratulain-bilal marked this conversation as resolved.
raise BundlerError(
f"Bundle file {arcname} exceeds {MAX_ZIP_MEMBER_BYTES}-byte limit"
)
content = fh.read(MAX_ZIP_MEMBER_BYTES + 1)
if len(content) > MAX_ZIP_MEMBER_BYTES:
Comment on lines +139 to +140
raise BundlerError(
f"Bundle file {arcname} exceeds {MAX_ZIP_MEMBER_BYTES}-byte limit"
)
archive.writestr(info, content)

# All members written successfully — atomically replace the final path.
# mkstemp() creates the staging file 0600 (owner-only) and os.replace()
# preserves that mode, which would silently publish every rebuild as an
# unreadable-to-others archive (a direct ZipFile(path, "w") write used
# to produce 0666 & ~umask). Set an intentional mode first: a rebuild
# keeps the existing artifact's mode so publishing pipelines that
# chmod'd it are not overridden; a fresh build honors the caller's
# umask (0666 & ~umask) so restrictive umasks still produce private
# files while normal umasks yield the expected 0644.
if artifact_path.exists():
os.chmod(tmp_path, artifact_path.stat().st_mode & 0o777)
else:
umask = os.umask(0)
os.umask(umask) # restore immediately
Comment on lines +158 to +159
os.chmod(tmp_path, 0o666 & ~umask)
os.replace(tmp_path, artifact_path)
except BaseException:
# Clean up the temporary file on any failure (exception, interrupt, etc.)
tmp_path.unlink(missing_ok=True)
raise

return BuildResult(artifact_path=artifact_path, file_count=len(files))

Expand All @@ -115,6 +180,7 @@ def _collect_files(
skip: Path,
skip_dir: Path | None = None,
artifact_re: re.Pattern[str] | None = None,
staging_re: re.Pattern[str] | None = None,
) -> list[Path]:
collected: list[Path] = []
# followlinks=False so a symlinked directory is never descended into,
Expand All @@ -140,6 +206,11 @@ def _collect_files(
if artifact_re is not None and artifact_re.match(name):
# A prior build artifact for this bundle — never re-package it.
continue
if staging_re is not None and root_path == bundle_dir and staging_re.match(name):
# A leftover staging file in the default output directory (the
# bundle root). Never re-package it. Restricted to the root so
# legitimate nested assets that merely match the pattern are kept.
continue
if path.is_symlink():
# Skip symlinked files to avoid escaping the bundle directory.
continue
Expand Down
25 changes: 20 additions & 5 deletions src/specify_cli/bundles/yamlio.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,26 +38,41 @@ def ensure_within(root: Path, candidate: Path) -> Path:
return candidate_resolved


def load_yaml(path: Path) -> Any:
def load_yaml(path: Path, *, max_bytes: int | None = None) -> Any:
"""Parse a YAML file, returning ``{}`` only for an *empty* document.

A non-empty document is returned exactly as parsed — including a
A non-empty document is returned exactly as parsed � including a
non-mapping such as ``[]``, ``false``, ``0``, ``''``, or an explicit null
(``null``/``~``) — so callers can validate the top-level shape (e.g. reject
(``null``/``~``) � so callers can validate the top-level shape (e.g. reject
a non-mapping config) instead of having it silently coerced to an empty
mapping.

``yaml.safe_load`` returns ``None`` for *both* an empty document and an
explicit null scalar, so ``yaml.compose`` (which yields no node only for a
truly empty document) is used to tell them apart: an empty document becomes
``{}`` while an explicit ``null``/``~`` is returned as ``None`` for the
``{}`` while an explicit ``null`` is returned as ``None`` for the
caller to reject.

When ``max_bytes`` is given the file is read through a bounded read, so a
file that grows after the size check (or is simply huge) never reaches the
YAML parser.
"""
path = Path(path)
if not path.exists():
raise BundlerError(f"File not found: {path}")
try:
text = path.read_text(encoding="utf-8")
if max_bytes is None:
text = path.read_text(encoding="utf-8")
else:
with open(path, "rb") as fh:
raw = fh.read(max_bytes + 1)
if len(raw) > max_bytes:
raise BundlerError(
f"File {path} exceeds {max_bytes}-byte limit"
)
text = raw.decode("utf-8")
except BundlerError:
raise
except (OSError, UnicodeError) as exc:
# A non-UTF-8 file raises UnicodeDecodeError, which is a ValueError --
# NOT an OSError -- so it escaped this module's "IO failures degrade
Expand Down
Loading
Loading