diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index d6a1e3d..acad8ad 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -5,4 +5,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: astral-sh/ruff-action@v3 \ No newline at end of file + - uses: astral-sh/ruff-action@v3 + with: + version: "0.16.1" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f175d37 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.16.1 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format \ No newline at end of file diff --git a/README.md b/README.md index 9d6cbff..c41e05a 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,14 @@ It is also wise to include the TOD information used for creating the map, so that we can later track individual TODs' contributions to various maps. Note that there does not need to be a one-to-one relationship between maps and TODs. + +`ctime`, `start_time`, and `stop_time` on depth-1 maps and coadds are +timezone-aware `datetime` objects (UTC), not unix timestamps. `map_id` +(and `coadd_id` on coadds) is a UUID7, auto-generated when the row is +created -- you don't need to set it yourself. ```python3 +from datetime import datetime, timezone + from mapcat.helper import settings from mapcat.database import DepthOneMapTable, TODDepthOneTable @@ -84,17 +91,19 @@ with settings.session() as session: map_name="15722/depth1_157221244_lati1_f090", map_path="15722/depth1_157221244_lati1_f090_map.fits", ivar_path="15722/depth1_157221244_lati1_f090_ivar.fits", - time_path="15722/depth1_157221244_lati1_f090_time.fits", + mean_time_path="15722/depth1_157221244_lati1_f090_time.fits", tube_slot="i1", frequency="090", - ctime=157221244.0, - start_time=157220244.0, - stop_time=157222423.0, + ctime=datetime.fromtimestamp(157221244.0, tz=timezone.utc), + start_time=datetime.fromtimestamp(157220244.0, tz=timezone.utc), + stop_time=datetime.fromtimestamp(157222423.0, tz=timezone.utc), tods=tods, ) session.add(map) session.commit() + + print(map.map_id) # auto-generated UUID7 ``` By referencing the TOD objects, you automatically create the required link table items. diff --git a/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py index c8c07cb..4bf3ab8 100644 --- a/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py +++ b/mapcat/alembic/versions/0762b3c7694d_swapping_to_datetime.py @@ -1,7 +1,7 @@ """Swapping to datetime Revision ID: 0762b3c7694d -Revises: 46575bc0d660 +Revises: 6eeaa35444bb Create Date: 2026-07-27 11:19:49.803412 """ @@ -16,7 +16,7 @@ # revision identifiers, used by Alembic. revision: str = "0762b3c7694d" -down_revision: str | None = "46575bc0d660" +down_revision: str | None = "6eeaa35444bb" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/mapcat/alembic/versions/6eeaa35444bb_uuid_map_and_coadd_ids.py b/mapcat/alembic/versions/6eeaa35444bb_uuid_map_and_coadd_ids.py new file mode 100644 index 0000000..d0dff5e --- /dev/null +++ b/mapcat/alembic/versions/6eeaa35444bb_uuid_map_and_coadd_ids.py @@ -0,0 +1,375 @@ +"""Convert depth_one_maps.map_id / depth_one_coadds.coadd_id to UUID +primary keys, and generalize time_domain_processing to track coadds too + +Revision ID: 6eeaa35444bb +Revises: 46575bc0d660 +Create Date: 2026-07-16 13:15:00.000000 + +This is a one-way migration: the original integer IDs are not recoverable +once dropped, so downgrade() is intentionally not implemented (see +downgrade() below for details). + +Step C (swapping in the new UUID columns and their constraints) is done +with raw SQL table recreation rather than op.batch_alter_table()'s +create_primary_key()/create_foreign_key(). In this SQLAlchemy/Alembic +version, batch mode's constraint reflection silently produced wrong +results here -- e.g. depth_one_maps ended up with *no* primary key at all, +and depth_one_sky_coverage's composite (map_id, x, y) primary key silently +shrunk to just (x, y) -- because the table's columns individually declare +primary_key=True (redundant with the table-level constraint), which +conflicts with reflection during batch recreation. Raw SQL sidesteps that +ambiguity entirely: every target table's DDL below is written out in full +and verified against a real SQLite database before this migration was +finalized. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +import uuid7 +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "6eeaa35444bb" +down_revision: str | None = "46575bc0d660" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _backfill(bind, table: str, id_column: str, mapping: dict) -> None: + """UPDATE `table` SET `id_column`_new = WHERE `id_column` = , + for every (old id -> new uuid) pair in `mapping`. + Writes the 32-char hex form (no dashes), matching the string SQLAlchemy's + sa.Uuid() bind processor produces for SQLite -- writing str(new_uuid) + (36-char, dashed) here would silently desync from any UUID value bound + via the ORM later (e.g. an explicit map_id=/coadd_id= filter), since + SQLite compares TEXT columns byte-for-byte and never normalizes the two + forms as equal. + """ + new_column = f"{id_column}_new" + for old_id, new_uuid in mapping.items(): + bind.execute( + sa.text( + f"UPDATE {table} SET {new_column} = :new_uuid " + f"WHERE {id_column} = :old_id" + ), + {"new_uuid": new_uuid.hex, "old_id": old_id}, + ) + + +def _recreate_table(bind, table: str, create_sql: str, insert_sql: str) -> None: + """Standard SQLite table-recreation dance: build the new table under a + temporary name, copy data across, drop the old table, then rename the + new one into place. `create_sql`/`insert_sql` must reference the + temporary name `{table}_new`.""" + bind.execute(sa.text(create_sql)) + bind.execute(sa.text(insert_sql)) + bind.execute(sa.text(f"DROP TABLE {table}")) + bind.execute(sa.text(f"ALTER TABLE {table}_new RENAME TO {table}")) + + +def upgrade() -> None: + bind = op.get_bind() + + # -- Step A: add nullable UUID shadow columns everywhere a map_id or + # coadd_id appears, so we can backfill without ever having a NOT NULL + # column with no value. + with op.batch_alter_table("depth_one_maps") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("depth_one_coadds") as batch_op: + batch_op.add_column(sa.Column("coadd_id_new", sa.Uuid(), nullable=True)) + + with op.batch_alter_table("time_domain_processing") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + batch_op.add_column(sa.Column("coadd_id", sa.Uuid(), nullable=True)) + batch_op.add_column(sa.Column("processing_status_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("depth_one_pointing_residuals") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("depth_one_sky_coverage") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("link_depth_one_map_to_coadd") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + batch_op.add_column(sa.Column("coadd_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("link_tod_to_depth_one_map") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + with op.batch_alter_table("pipeline_information") as batch_op: + batch_op.add_column(sa.Column("map_id_new", sa.Uuid(), nullable=True)) + + # -- Step B: generate new UUIDs for every existing map/coadd, and + # propagate the old-id -> new-uuid mapping to every dependent table. + # Raw SQL only -- never import the (still-evolving) ORM model classes + # inside a migration. + map_rows = bind.execute(sa.text("SELECT map_id FROM depth_one_maps")).fetchall() + map_id_mapping = {row.map_id: uuid7.create() for row in map_rows} + _backfill(bind, "depth_one_maps", "map_id", map_id_mapping) + + proc_rows = bind.execute( + sa.text("SELECT processing_status_id FROM time_domain_processing") + ).fetchall() + proc_id_mapping = {row.processing_status_id: uuid7.create() for row in proc_rows} + _backfill(bind, "time_domain_processing", "processing_status_id", proc_id_mapping) + + coadd_rows = bind.execute( + sa.text("SELECT coadd_id FROM depth_one_coadds") + ).fetchall() + coadd_id_mapping = {row.coadd_id: uuid7.create() for row in coadd_rows} + _backfill(bind, "depth_one_coadds", "coadd_id", coadd_id_mapping) + + _backfill(bind, "time_domain_processing", "map_id", map_id_mapping) + _backfill(bind, "depth_one_pointing_residuals", "map_id", map_id_mapping) + _backfill(bind, "depth_one_sky_coverage", "map_id", map_id_mapping) + _backfill(bind, "link_depth_one_map_to_coadd", "map_id", map_id_mapping) + _backfill(bind, "link_depth_one_map_to_coadd", "coadd_id", coadd_id_mapping) + _backfill(bind, "link_tod_to_depth_one_map", "map_id", map_id_mapping) + _backfill(bind, "pipeline_information", "map_id", map_id_mapping) + + # -- Step C: swap in the new UUID columns and their constraints via raw + # SQL table recreation (see module docstring for why). depth_one_maps/ + # depth_one_coadds go first, so dependent tables' new FK constraints + # have something to point at. + _recreate_table( + bind, + "depth_one_maps", + """ + CREATE TABLE depth_one_maps_new ( + map_name VARCHAR NOT NULL, + map_path VARCHAR, + ivar_path VARCHAR, + mean_time_path VARCHAR, + tube_slot VARCHAR NOT NULL, + frequency VARCHAR NOT NULL, + ctime FLOAT NOT NULL, + start_time FLOAT NOT NULL, + stop_time FLOAT NOT NULL, + notes JSON, + rho_path VARCHAR, + kappa_path VARCHAR, + start_time_path VARCHAR, + end_time_path VARCHAR, + flux_path VARCHAR, + snr_path VARCHAR, + map_id CHAR(32) NOT NULL, + PRIMARY KEY (map_id), + UNIQUE (map_path), + UNIQUE (map_name) + ) + """, + """ + INSERT INTO depth_one_maps_new + SELECT map_name, map_path, ivar_path, mean_time_path, tube_slot, + frequency, ctime, start_time, stop_time, notes, rho_path, + kappa_path, start_time_path, end_time_path, flux_path, + snr_path, map_id_new + FROM depth_one_maps + """, + ) + for index_sql in [ + "CREATE INDEX ix_depth_one_maps_map_name ON depth_one_maps (map_name)", + "CREATE INDEX ix_depth_one_maps_start_time ON depth_one_maps (start_time)", + "CREATE INDEX ix_depth_one_maps_ctime ON depth_one_maps (ctime)", + "CREATE INDEX ix_depth_one_maps_stop_time ON depth_one_maps (stop_time)", + "CREATE INDEX ix_depth_one_maps_frequency ON depth_one_maps (frequency)", + "CREATE INDEX ix_depth_one_maps_tube_slot ON depth_one_maps (tube_slot)", + ]: + bind.execute(sa.text(index_sql)) + + _recreate_table( + bind, + "depth_one_coadds", + """ + CREATE TABLE depth_one_coadds_new ( + coadd_name VARCHAR NOT NULL, + coadd_type VARCHAR NOT NULL, + map_path VARCHAR NOT NULL, + frequency VARCHAR NOT NULL, + ctime FLOAT NOT NULL, + start_time FLOAT NOT NULL, + stop_time FLOAT NOT NULL, + ivar_path VARCHAR, + rho_path VARCHAR, + kappa_path VARCHAR, + start_time_path VARCHAR, + mean_time_path VARCHAR, + end_time_path VARCHAR, + coadd_id CHAR(32) NOT NULL, + PRIMARY KEY (coadd_id) + ) + """, + """ + INSERT INTO depth_one_coadds_new + SELECT coadd_name, coadd_type, map_path, frequency, ctime, + start_time, stop_time, ivar_path, rho_path, kappa_path, + start_time_path, mean_time_path, end_time_path, coadd_id_new + FROM depth_one_coadds + """, + ) + + _recreate_table( + bind, + "time_domain_processing", + """ + CREATE TABLE time_domain_processing_new ( + processing_status_id CHAR(32) NOT NULL, + processing_start FLOAT, + processing_end FLOAT, + processing_status VARCHAR NOT NULL, + map_id CHAR(32), + coadd_id CHAR(32), + PRIMARY KEY (processing_status_id), + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE, + FOREIGN KEY(coadd_id) REFERENCES depth_one_coadds (coadd_id) ON DELETE CASCADE, + CONSTRAINT ck_time_domain_processing_exactly_one_target + CHECK ((map_id IS NOT NULL) != (coadd_id IS NOT NULL)) + ) + """, + """ + INSERT INTO time_domain_processing_new + SELECT processing_status_id_new, processing_start, processing_end, + processing_status, map_id_new, coadd_id + FROM time_domain_processing + """, + ) + bind.execute( + sa.text( + "CREATE INDEX ix_time_domain_processing_map_id " + "ON time_domain_processing (map_id)" + ) + ) + bind.execute( + sa.text( + "CREATE INDEX ix_time_domain_processing_coadd_id " + "ON time_domain_processing (coadd_id)" + ) + ) + bind.execute( + sa.text( + "CREATE INDEX ix_time_domain_processing_processing_status " + "ON time_domain_processing (processing_status)" + ) + ) + + _recreate_table( + bind, + "depth_one_pointing_residuals", + """ + CREATE TABLE depth_one_pointing_residuals_new ( + pointing_residual_id INTEGER NOT NULL, + residual_model JSON NOT NULL, + residual_stats JSON, + map_id CHAR(32) NOT NULL, + PRIMARY KEY (pointing_residual_id), + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE + ) + """, + """ + INSERT INTO depth_one_pointing_residuals_new + SELECT pointing_residual_id, residual_model, residual_stats, map_id_new + FROM depth_one_pointing_residuals + """, + ) + bind.execute( + sa.text( + "CREATE INDEX ix_depth_one_pointing_residuals " + "ON depth_one_pointing_residuals (map_id)" + ) + ) + + _recreate_table( + bind, + "depth_one_sky_coverage", + """ + CREATE TABLE depth_one_sky_coverage_new ( + x CHAR NOT NULL, + y CHAR NOT NULL, + map_id CHAR(32) NOT NULL, + PRIMARY KEY (map_id, x, y), + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE + ) + """, + """ + INSERT INTO depth_one_sky_coverage_new + SELECT x, y, map_id_new + FROM depth_one_sky_coverage + """, + ) + bind.execute( + sa.text( + "CREATE INDEX ix_depth_one_sky_coverage_x ON depth_one_sky_coverage (x)" + ) + ) + bind.execute( + sa.text( + "CREATE INDEX ix_depth_one_sky_coverage_y ON depth_one_sky_coverage (y)" + ) + ) + + _recreate_table( + bind, + "link_depth_one_map_to_coadd", + """ + CREATE TABLE link_depth_one_map_to_coadd_new ( + map_id CHAR(32) NOT NULL, + coadd_id CHAR(32) NOT NULL, + PRIMARY KEY (map_id, coadd_id), + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE, + FOREIGN KEY(coadd_id) REFERENCES depth_one_coadds (coadd_id) ON DELETE CASCADE + ) + """, + """ + INSERT INTO link_depth_one_map_to_coadd_new + SELECT map_id_new, coadd_id_new + FROM link_depth_one_map_to_coadd + """, + ) + + _recreate_table( + bind, + "link_tod_to_depth_one_map", + """ + CREATE TABLE link_tod_to_depth_one_map_new ( + tod_id INTEGER NOT NULL, + map_id CHAR(32) NOT NULL, + PRIMARY KEY (tod_id, map_id), + FOREIGN KEY(tod_id) REFERENCES tod_depth_one (tod_id) ON DELETE CASCADE, + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE + ) + """, + """ + INSERT INTO link_tod_to_depth_one_map_new + SELECT tod_id, map_id_new + FROM link_tod_to_depth_one_map + """, + ) + + _recreate_table( + bind, + "pipeline_information", + """ + CREATE TABLE pipeline_information_new ( + pipeline_information_id INTEGER NOT NULL, + sotodlib_version VARCHAR NOT NULL, + map_maker VARCHAR NOT NULL, + preprocess_info JSON, + map_id CHAR(32) NOT NULL, + PRIMARY KEY (pipeline_information_id), + FOREIGN KEY(map_id) REFERENCES depth_one_maps (map_id) ON DELETE CASCADE + ) + """, + """ + INSERT INTO pipeline_information_new + SELECT pipeline_information_id, sotodlib_version, map_maker, + preprocess_info, map_id_new + FROM pipeline_information + """, + ) + + +def downgrade() -> None: + raise NotImplementedError( + "This migration is one-way: the original integer map_id/coadd_id " + "values are dropped and cannot be recovered. A 'downgrade' that " + "renumbered rows from scratch would silently desync any external " + "system that cached the old integer IDs, so it is not implemented. " + "Restore from a backup taken before this migration if you need to " + "roll back." + ) diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index 94c742a..6839745 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -15,6 +15,7 @@ __all__ = [ "AtomicMapCoaddTable", "AtomicMapTable", + "DepthOneCoaddTable", "DepthOneMapTable", "PipelineInformationTable", "PointingResidualTable", diff --git a/mapcat/database/depth_one_coadd.py b/mapcat/database/depth_one_coadd.py index a2e0702..169b3ea 100644 --- a/mapcat/database/depth_one_coadd.py +++ b/mapcat/database/depth_one_coadd.py @@ -3,16 +3,23 @@ """ from datetime import datetime +from typing import TYPE_CHECKING from astropy.time import Time +from sqlalchemy import Uuid from sqlmodel import Field, Relationship, SQLModel +from uuid7 import UUID as UUID7 +from uuid7 import create as uuid7_create from .depth_one_map import DepthOneMapTable from .links import DepthOneToCoaddTable +if TYPE_CHECKING: # pragma: no cover + from .time_domain_processing import TimeDomainProcessingTable + class DepthOneCoadd(SQLModel): - coadd_id: int + coadd_id: UUID7 coadd_name: str coadd_type: str @@ -39,7 +46,7 @@ class DepthOneCoaddTable(SQLModel, table=True): __tablename__ = "depth_one_coadds" - coadd_id: int = Field(primary_key=True) + coadd_id: UUID7 = Field(default_factory=uuid7_create, primary_key=True, sa_type=Uuid) coadd_name: str = Field(nullable=False) coadd_type: str = Field(nullable=False) @@ -61,6 +68,10 @@ class DepthOneCoaddTable(SQLModel, table=True): back_populates="coadds", link_model=DepthOneToCoaddTable, ) + processing_status: list["TimeDomainProcessingTable"] = Relationship( + back_populates="coadd", + cascade_delete=True, + ) def to_model(self) -> DepthOneCoadd: """ diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index cc338f0..90af906 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -7,7 +7,10 @@ from astropy.time import Time from astropydantic import AstroPydanticTime +from sqlalchemy import Uuid from sqlmodel import JSON, Field, Relationship, SQLModel +from uuid7 import UUID as UUID7 +from uuid7 import create as uuid7_create if TYPE_CHECKING: # pragma: no cover from .depth_one_coadd import DepthOneCoaddTable @@ -21,7 +24,7 @@ class DepthOneMap(SQLModel): - map_id: int + map_id: UUID7 map_name: str map_path: str | None @@ -48,8 +51,8 @@ class DepthOneMapTable(SQLModel, table=True): Attributes ---------- - id : int - Unique map identifiers. Internal to SO + id : UUID7 + Unique map identifier. Internal to SO map_name : str Name of depth 1 map map_path : str | None @@ -102,7 +105,7 @@ class DepthOneMapTable(SQLModel, table=True): __tablename__ = "depth_one_maps" - map_id: int = Field(primary_key=True) + map_id: UUID7 = Field(default_factory=uuid7_create, primary_key=True, sa_type=Uuid) map_name: str = Field(index=True, unique=True, nullable=False) map_path: str | None = None diff --git a/mapcat/database/links.py b/mapcat/database/links.py index 1db33b2..c72ffe6 100644 --- a/mapcat/database/links.py +++ b/mapcat/database/links.py @@ -2,6 +2,9 @@ Link tables. """ +from uuid import UUID + +from sqlalchemy import Uuid from sqlmodel import Field, SQLModel @@ -12,19 +15,21 @@ class DepthOneToCoaddTable(SQLModel, table=True): __tablename__ = "link_depth_one_map_to_coadd" - map_id: int = Field( + map_id: UUID = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", + sa_type=Uuid, ) - coadd_id: int = Field( + coadd_id: UUID = Field( foreign_key="depth_one_coadds.coadd_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", + sa_type=Uuid, ) @@ -42,12 +47,13 @@ class TODToMapTable(SQLModel, table=True): index=True, ondelete="CASCADE", ) - map_id: int = Field( + map_id: UUID = Field( foreign_key="depth_one_maps.map_id", primary_key=True, nullable=False, index=True, ondelete="CASCADE", + sa_type=Uuid, ) diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 8ca6ac2..39d7ff2 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -3,7 +3,9 @@ """ from typing import Any +from uuid import UUID +from sqlalchemy import Uuid from sqlmodel import JSON, Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable @@ -30,7 +32,9 @@ class PipelineInformationTable(SQLModel, table=True): __tablename__ = "pipeline_information" pipeline_information_id: int = Field(primary_key=True) - map_id: int = Field(foreign_key="depth_one_maps.map_id", nullable=False) + map_id: UUID = Field( + foreign_key="depth_one_maps.map_id", nullable=False, sa_type=Uuid + ) map: DepthOneMapTable = Relationship(back_populates="pipeline_information") sotodlib_version: str diff --git a/mapcat/database/pointing_residual.py b/mapcat/database/pointing_residual.py index 7f9a14e..7045ef1 100644 --- a/mapcat/database/pointing_residual.py +++ b/mapcat/database/pointing_residual.py @@ -2,6 +2,9 @@ Table containing pointing residuals. """ +from uuid import UUID + +from sqlalchemy import Uuid from sqlmodel import Field, Relationship, SQLModel from mapcat.pointing.base import PointingModelStats @@ -33,11 +36,12 @@ class PointingResidualTable(SQLModel, table=True): __tablename__ = "depth_one_pointing_residuals" pointing_residual_id: int = Field(primary_key=True) - map_id: int = Field( + map_id: UUID = Field( index=True, nullable=False, foreign_key="depth_one_maps.map_id", ondelete="CASCADE", + sa_type=Uuid, ) residual_model: PointingModel = Field(sa_type=JSONEncodedPydantic(PointingModel)) residual_stats: PointingModelStats | None = Field( diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index fa5021f..3906f6f 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,7 +2,9 @@ Sky coverage table. """ -from sqlalchemy import PrimaryKeyConstraint +from uuid import UUID + +from sqlalchemy import PrimaryKeyConstraint, Uuid from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable @@ -32,11 +34,12 @@ class SkyCoverageTable(SQLModel, table=True): x: int = Field(index=True, primary_key=True) y: int = Field(index=True, primary_key=True) - map_id: int = Field( + map_id: UUID = Field( foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE", primary_key=True, + sa_type=Uuid, ) map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") diff --git a/mapcat/database/time_domain_processing.py b/mapcat/database/time_domain_processing.py index e21e62a..bd227f1 100644 --- a/mapcat/database/time_domain_processing.py +++ b/mapcat/database/time_domain_processing.py @@ -1,20 +1,26 @@ """ -Table containing information about processing status of the Depth-1 maps. +Table containing information about processing status of the Depth-1 maps +and depth-1 map coadds. """ from datetime import datetime +from typing import TYPE_CHECKING from astropy.time import Time from astropydantic import AstroPydanticTime +from sqlalchemy import CheckConstraint, Uuid from sqlmodel import Field, Relationship, SQLModel +from uuid7 import UUID as UUID7 -from .depth_one_map import DepthOneMapTable +if TYPE_CHECKING: # pragma: no cover + from .depth_one_coadd import DepthOneCoaddTable + from .depth_one_map import DepthOneMapTable class TimeDomainProcessing(SQLModel): - processing_status_id: int + processing_status_id: UUID7 - map_id: int + map_id: UUID7 processing_start: AstroPydanticTime | None processing_end: AstroPydanticTime | None @@ -23,17 +29,26 @@ class TimeDomainProcessing(SQLModel): class TimeDomainProcessingTable(SQLModel, table=True): """ - Table for tracking processing status of depth-1 maps - providing SQLModel functionality. You can export a base model, for example - for responding to a query with using the `to_model` method. Note some attributes - are inherited from TimeDomainProcessingTable. + Table for tracking processing status of depth-1 maps and depth-1 map + coadds, providing SQLModel functionality. You can export a base model, + for example for responding to a query with using the `to_model` method. + Note some attributes are inherited from TimeDomainProcessingTable. + + Exactly one of `map_id`/`coadd_id` is set per row (enforced by a DB + CHECK constraint) -- a row tracks either a depth-1 map or a coadd, + never both and never neither. Attributes ---------- - processing_status_id : int + processing_status_id : UUID7 + Internal ID of the processing status. Yet another uuid, + independent of the map/coadd being tracked. + map_id : UUID7 | None + Depth-1 map being tracked, if this row is for a map. + coadd_id : UUID7 | None + Depth-1 map coadd being tracked, if this row is for a coadd. + processing_start : float | None Internal ID of the processing status - map_name : int - Name of depth 1 map being tracked. Foreign into DepthOneMap processing_start : datetime | None Time processing started. None if not started. processing_end : datetime | None @@ -44,20 +59,37 @@ class TimeDomainProcessingTable(SQLModel, table=True): __tablename__ = "time_domain_processing" - processing_status_id: int = Field(primary_key=True) + processing_status_id: UUID7 = Field(primary_key=True, sa_type=Uuid) - map_id: int = Field( + map_id: UUID7 | None = Field( + default=None, index=True, - nullable=False, + nullable=True, foreign_key="depth_one_maps.map_id", ondelete="CASCADE", + sa_type=Uuid, ) - map: DepthOneMapTable = Relationship(back_populates="processing_status") + coadd_id: UUID7 | None = Field( + default=None, + index=True, + nullable=True, + foreign_key="depth_one_coadds.coadd_id", + ondelete="CASCADE", + sa_type=Uuid, + ) + map: "DepthOneMapTable" = Relationship(back_populates="processing_status") + coadd: "DepthOneCoaddTable" = Relationship(back_populates="processing_status") processing_start: datetime = Field(nullable=True) processing_end: datetime = Field(nullable=True) processing_status: str = Field(index=True, nullable=False) + __table_args__ = ( + CheckConstraint( + "(map_id IS NOT NULL) != (coadd_id IS NOT NULL)", + name="ck_time_domain_processing_exactly_one_target", + ), + ) def to_model(self) -> TimeDomainProcessing: """ Return a TimeDomainProcessing model from this table entry diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 0244753..3ab48c4 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -78,7 +78,6 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT ) for obs_id in file_info["observations"] ] - return DepthOneMapTable( map_name=filenames["map"].replace("_map.fits", ""), map_path=filenames["map"], diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index d7c61cc..7a18cfa 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -7,6 +7,7 @@ from sqlalchemy import select from sqlalchemy.orm import sessionmaker +from uuid7 import UUID as UUID7 from mapcat.database import ( DepthOneMapTable, @@ -23,8 +24,10 @@ (which tells the pipeline not to retry the map due to a pathological failure), or removed entirely by not specifying a target status. -Entries to reset can be filtered by map ID, time range (using the map's ctime), -and/or current processing status. +Entries to reset can be filtered by map ID, coadd ID, time range (using the +map's ctime), and/or current processing status. --start-time/--end-time only +filter map-linked entries (DepthOneCoaddTable also has a ctime, but time +filtering for coadd entries isn't supported yet). """ USAGE = """Examples: @@ -35,7 +38,7 @@ Remove processing status entries for specific map IDs: - mapcatreset --map-id 10 11 12 + mapcatreset --map-id 3fa85f64-5717-4562-b3fc-2c963f66afa6 Reset entries with status 'running' to 'failed' in a time range: @@ -44,7 +47,11 @@ Mark a specific map as 'permafail' (will not be retried by the pipeline): - mapcatreset --status permafail --map-id 42 + mapcatreset --status permafail --map-id 3fa85f64-5717-4562-b3fc-2c963f66afa6 + + Mark a specific coadd as 'permafail': + + mapcatreset --status permafail --coadd-id 9c858901-8a57-4791-81fe-4c455b0c9c78 """ @@ -67,6 +74,9 @@ def core(session: sessionmaker, args: ap.Namespace): if args.map_id: stmt = stmt.where(TimeDomainProcessingTable.map_id.in_(args.map_id)) + if args.coadd_id: + stmt = stmt.where(TimeDomainProcessingTable.coadd_id.in_(args.coadd_id)) + if args.start_time is not None or args.end_time is not None: stmt = stmt.join( DepthOneMapTable, @@ -154,12 +164,21 @@ def main(): parser.add_argument( "-m", "--map-id", - type=int, + type=UUID7, nargs="+", default=None, help="Only reset entries for these map IDs.", ) + parser.add_argument( + "-c", + "--coadd-id", + type=UUID7, + nargs="+", + default=None, + help="Only reset entries for these coadd IDs.", + ) + parser.add_argument( "--start-time", type=float, diff --git a/pyproject.toml b/pyproject.toml index 747c88f..8e404bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ mapcat = ["alembic/*", "alembic.ini", "alembic/versions/*"] [project] name = "mapcat" -version = "0.3.1" +version = "0.4.0" requires-python = ">=3.10" dependencies = [ "astropydantic", @@ -17,7 +17,8 @@ dependencies = [ "alembic", "pydantic_settings", "pixell", - "h5py" + "h5py", + "uuid7-standard" ] [project.scripts] @@ -29,7 +30,7 @@ mapcatreset = "mapcat.toolkit.reset:main" [project.optional-dependencies] dev = [ "pytest", - "ruff", + "ruff==0.16.1", "testcontainers", "coverage", "pytest-cov", diff --git a/tests/test_act.py b/tests/test_act.py index 574833e..60a2741 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -274,7 +274,6 @@ def test_sky_coverage_2(database_sessionmaker, downloaded_data_file): relative_to=downloaded_data_file, telescope="act", ) - act.core(session=database_sessionmaker, args=args) update_sky_coverage.core(session=database_sessionmaker, convention="ACT") diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index bca7e44..030bc02 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -3,6 +3,7 @@ """ from datetime import datetime, timezone +from uuid import uuid4 import pytest from astropy import units as u @@ -99,6 +100,7 @@ def test_create_depth_one(database_sessionmaker): # Make child tables with database_sessionmaker() as session: processing_status = TimeDomainProcessingTable( + processing_status_id=map_id, processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", @@ -244,7 +246,7 @@ def test_create_depth_one(database_sessionmaker): # Check bad map ID raises ValueError with pytest.raises(ValueError), database_sessionmaker() as session: - result = session.get(DepthOneMapTable, 999999) + result = session.get(DepthOneMapTable, uuid4()) if result is None: raise ValueError("Map ID does not exist") @@ -253,7 +255,6 @@ def test_add_remove_child_tables(database_sessionmaker): # Create a depth one map with database_sessionmaker() as session: dmap = DepthOneMapTable( - map_id=42, map_name="myDepthOne2", map_path="/PATH/TO/DEPTH/ONE2", tube_slot="OTi1", @@ -264,6 +265,7 @@ def test_add_remove_child_tables(database_sessionmaker): ) processing_status = TimeDomainProcessingTable( + processing_status_id=dmap.map_id, processing_start=datetime.fromtimestamp(1756787524.0, tz=timezone.utc), processing_end=datetime.fromtimestamp(1756797524.0, tz=timezone.utc), processing_status="done", diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py index 5e9df7b..a1e104b 100644 --- a/tests/test_mapmaking.py +++ b/tests/test_mapmaking.py @@ -164,7 +164,9 @@ def test_build_obslists(database_sessionmaker): obs_list = build_obslists(obs_ids=obs_ids, session=session) assert obs_list[0][obs_ids[0]][0].map_id == map_id1 - assert obs_list[0][obs_ids[1]][0].map_id == map_id1 - assert obs_list[0][obs_ids[1]][1].map_id == map_id2 + # obs_ids[1] is linked to both maps -- the relationship list's order + # isn't guaranteed by the DB (no explicit order_by on the relationship), + # so compare as a set rather than asserting a specific position. + assert {m.map_id for m in obs_list[0][obs_ids[1]]} == {map_id1, map_id2} assert obs_list[0][obs_ids[2]][0].map_id == map_id2 assert obs_ids[3] in obs_list[1] diff --git a/tests/test_reset.py b/tests/test_reset.py index 87826f3..69b0d2c 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -9,7 +9,11 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from mapcat.database import DepthOneMapTable, TimeDomainProcessingTable +from mapcat.database import ( + DepthOneCoaddTable, + DepthOneMapTable, + TimeDomainProcessingTable, +) from mapcat.toolkit.reset import VALID_STATUSES, core @@ -68,6 +72,7 @@ def _make_proc(session, map_id, status): """Helper to insert a TimeDomainProcessingTable row and return its id.""" with session() as s: proc = TimeDomainProcessingTable( + processing_status_id=map_id, map_id=map_id, processing_start=datetime.fromtimestamp(1756000000.0, tz=timezone.utc), processing_end=datetime.fromtimestamp(1756001000.0, tz=timezone.utc), @@ -85,6 +90,41 @@ def _get_proc(session, proc_id): return s.get(TimeDomainProcessingTable, proc_id) +def _make_coadd(session, name, ctime): + """Helper to insert a DepthOneCoaddTable row and return its coadd_id.""" + with session() as s: + coadd = DepthOneCoaddTable( + coadd_name=name, + coadd_type="depth1_streaming_coadd", + map_path=f"/path/{name}_flux.fits", + ivar_path=None, + frequency="f090", + ctime=datetime.fromtimestamp(ctime, tz=timezone.utc), + start_time=datetime.fromtimestamp(ctime - 500, tz=timezone.utc), + stop_time=datetime.fromtimestamp(ctime + 500, tz=timezone.utc), + ) + s.add(coadd) + s.commit() + s.refresh(coadd) + return coadd.coadd_id + + +def _make_coadd_proc(session, coadd_id, status): + """Helper to insert a coadd-linked TimeDomainProcessingTable row.""" + with session() as s: + proc = TimeDomainProcessingTable( + processing_status_id=coadd_id, + coadd_id=coadd_id, + processing_start=datetime.fromtimestamp(1756000000.0, tz=timezone.utc), + processing_end=datetime.fromtimestamp(1756001000.0, tz=timezone.utc), + processing_status=status, + ) + s.add(proc) + s.commit() + s.refresh(proc) + return proc.processing_status_id + + def test_valid_statuses(): assert "failed" in VALID_STATUSES assert "completed" in VALID_STATUSES @@ -98,6 +138,7 @@ def test_reset_all_to_failed(database_sessionmaker): args = argparse.Namespace( status="failed", + coadd_id=None, map_id=None, start_time=None, end_time=None, @@ -120,6 +161,7 @@ def test_reset_by_map_id(database_sessionmaker): args = argparse.Namespace( status="completed", + coadd_id=None, map_id=[map_id_a], start_time=None, end_time=None, @@ -145,6 +187,7 @@ def test_reset_by_time_range(database_sessionmaker): args = argparse.Namespace( status="failed", + coadd_id=None, map_id=None, start_time=1753000000.0, end_time=1755000000.0, @@ -170,6 +213,7 @@ def test_reset_by_from_status(database_sessionmaker): args = argparse.Namespace( status="completed", + coadd_id=None, map_id=None, start_time=None, end_time=None, @@ -192,6 +236,7 @@ def test_remove_entries_no_status(database_sessionmaker): args = argparse.Namespace( status=None, + coadd_id=None, map_id=[map_id], start_time=None, end_time=None, @@ -210,6 +255,7 @@ def test_permafail_status(database_sessionmaker): args = argparse.Namespace( status="permafail", + coadd_id=None, map_id=[map_id], start_time=None, end_time=None, @@ -222,6 +268,33 @@ def test_permafail_status(database_sessionmaker): assert proc.processing_status == "permafail" +def test_reset_by_coadd_id(database_sessionmaker): + """Reset only the entry for a specific coadd ID, leaving map-linked + entries untouched.""" + map_id = _make_map(database_sessionmaker, "reset_coadd_map", 1755900000.0) + coadd_id = _make_coadd(database_sessionmaker, "reset_coadd_a", 1755900500.0) + + map_proc_id = _make_proc(database_sessionmaker, map_id, "running") + coadd_proc_id = _make_coadd_proc(database_sessionmaker, coadd_id, "running") + + args = argparse.Namespace( + status="completed", + map_id=None, + coadd_id=[coadd_id], + start_time=None, + end_time=None, + from_status=None, + ) + core(session=database_sessionmaker, args=args) + + map_proc = _get_proc(database_sessionmaker, map_proc_id) + coadd_proc = _get_proc(database_sessionmaker, coadd_proc_id) + + assert coadd_proc.processing_status == "completed" + # the map-linked entry should be untouched + assert map_proc.processing_status == "running" + + def test_combined_filters(database_sessionmaker): """Combine map_id and from_status filters.""" map_id_a = _make_map(database_sessionmaker, "reset_combo_a", 1755700000.0) @@ -233,6 +306,7 @@ def test_combined_filters(database_sessionmaker): # Only reset map_id_a if its status is "running" args = argparse.Namespace( status="failed", + coadd_id=None, map_id=[map_id_a], start_time=None, end_time=None,