Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# datannurpy

## 0.33.4 (2026-10-07)

- fix: Excel/ODS, Iceberg and partitioned-Parquet scans no longer fail with `ParserException: syntax error at end of input` on fresh installs — ibis 12.0.0 emits a nameless `DROP VIEW IF EXISTS` with sqlglot ≥ 30.18 (ibis-project/ibis#12124, fixed upstream but unreleased), so sqlglot is capped below 30.18 until an ibis release ships the fix

## 0.33.3 (2026-08-15)

- fix: a configured `log_file` creates its missing parent directories instead of crashing with `FileNotFoundError` on a fresh environment
Expand Down
7 changes: 6 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "datannurpy"
version = "0.33.3"
version = "0.33.4"
description = "Scan files and databases into ready-to-use datannur catalogs"
authors = [{ name = "datannur" }]
readme = "README.md"
Expand All @@ -23,6 +23,11 @@ classifiers = [
]
dependencies = [
"ibis-framework[duckdb]>=11.0",
# ibis 12.0.0 + sqlglot >= 30.18 emit a nameless `DROP VIEW IF EXISTS`, so every
# in-memory create_table fails (Excel/ODS, Iceberg, partitioned Parquet scans).
# Fixed upstream (ibis-project/ibis#12104) but unreleased — drop this cap once an
# ibis release ships the fix. See ibis-project/ibis#12124.
"sqlglot<30.18",
"polars>=1.0",
"jsonjsdb>=0.9.4",
"openpyxl>=3.0",
Expand Down
36 changes: 18 additions & 18 deletions tests/database/test_db_introspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def sqlite_catalog(sqlite_with_constraints: Path) -> Catalog:
"""Catalog populated from SQLite DB with constraints."""
catalog = Catalog()
catalog.add_database(
f"sqlite:////{sqlite_with_constraints}",
f"sqlite:///{sqlite_with_constraints.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
)
return catalog
Expand All @@ -157,12 +157,12 @@ def _emp_vars(catalog: Catalog) -> dict[str, Variable]:

class TestSQLiteIntrospection:
def test_simple_pk(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert introspect_table(con, "sqlite", None, "departments").pk_map == {"id": 1}
con.disconnect()

def test_composite_pk(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert introspect_table(con, "sqlite", None, "order_items").pk_map == {
"order_id": 1,
"item_id": 2,
Expand All @@ -178,7 +178,7 @@ def test_no_pk(self) -> None:
con.disconnect()

def test_fk(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
fks = introspect_table(con, "sqlite", None, "employees").fks
assert len(fks) == 1
assert (fks[0].local_col, fks[0].ref_table, fks[0].ref_col) == (
Expand All @@ -189,38 +189,38 @@ def test_fk(self, sqlite_with_constraints: Path) -> None:
con.disconnect()

def test_no_fk(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert introspect_table(con, "sqlite", None, "departments").fks == []
con.disconnect()

def test_comments_empty(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
meta = introspect_table(con, "sqlite", None, "employees")
assert meta.table_comment is None and meta.col_comments == {}
con.disconnect()

def test_not_null(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert {"name", "department_id"} <= introspect_table(
con, "sqlite", None, "employees"
).not_null
con.disconnect()

def test_unique(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert "email" in introspect_table(con, "sqlite", None, "employees").unique
assert "name" in introspect_table(con, "sqlite", None, "departments").unique
con.disconnect()

def test_indexed(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert {"name", "department_id"} <= introspect_table(
con, "sqlite", None, "employees"
).indexed
con.disconnect()

def test_auto_increment(self, sqlite_with_constraints: Path) -> None:
con, _ = connect(f"sqlite:////{sqlite_with_constraints}")
con, _ = connect(f"sqlite:///{sqlite_with_constraints.as_posix()}")
assert "id" in introspect_table(con, "sqlite", None, "employees").auto_inc
con.disconnect()

Expand Down Expand Up @@ -798,7 +798,7 @@ def test_duckdb_comments(self, duckdb_with_constraints) -> None:
def test_dataset_depth_skips(self, sqlite_with_constraints: Path) -> None:
catalog = Catalog(depth="dataset")
catalog.add_database(
f"sqlite:////{sqlite_with_constraints}",
f"sqlite:///{sqlite_with_constraints.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
)
assert catalog.tag.count == 0
Expand All @@ -807,7 +807,7 @@ def test_dataset_depth_skips(self, sqlite_with_constraints: Path) -> None:
def test_schema_depth_introspects(self, sqlite_with_constraints: Path) -> None:
catalog = Catalog(depth="variable")
catalog.add_database(
f"sqlite:////{sqlite_with_constraints}",
f"sqlite:///{sqlite_with_constraints.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
)
assert catalog.tag.count > 0
Expand All @@ -822,7 +822,7 @@ def test_cached_metadata_refreshed(
sqlite_with_constraints: Path,
tmp_path: Path,
) -> None:
conn_str = f"sqlite:////{sqlite_with_constraints}"
conn_str = f"sqlite:///{sqlite_with_constraints.as_posix()}"
metadata = EntityMetadata(id="db", name="Test DB")

cat1 = Catalog(app_path=tmp_path, quiet=True)
Expand All @@ -842,7 +842,7 @@ def test_schema_depth_cached_metadata(
tmp_path: Path,
) -> None:
"""Schema-depth cache hit still refreshes introspection metadata."""
conn_str = f"sqlite:////{sqlite_with_constraints}"
conn_str = f"sqlite:///{sqlite_with_constraints.as_posix()}"
metadata = EntityMetadata(id="db", name="Test DB")

cat1 = Catalog(app_path=tmp_path, depth="variable", quiet=True)
Expand All @@ -862,7 +862,7 @@ def test_tags_survive_finalize(
) -> None:
cat = Catalog(app_path=tmp_path, quiet=True)
cat.add_database(
f"sqlite:////{sqlite_with_constraints}",
f"sqlite:///{sqlite_with_constraints.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
)
cat.finalize()
Expand All @@ -880,7 +880,7 @@ def test_introspect_returns_empty(
):
catalog = Catalog()
catalog.add_database(
f"sqlite:////{sqlite_with_constraints}",
f"sqlite:///{sqlite_with_constraints.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
)
v = _emp_vars(catalog)
Expand Down Expand Up @@ -909,7 +909,7 @@ def test_pk_removal_on_rescan(
tmp_path: Path,
) -> None:
"""If a column is no longer PK, key should be cleared on rescan."""
conn_str = f"sqlite:////{sqlite_with_constraints}"
conn_str = f"sqlite:///{sqlite_with_constraints.as_posix()}"
metadata = EntityMetadata(id="db", name="Test DB")

cat1 = Catalog(app_path=tmp_path, quiet=True)
Expand Down Expand Up @@ -944,7 +944,7 @@ def test_unresolvable_fk_ignored(self, tmp_path: Path) -> None:

catalog = Catalog()
catalog.add_database(
f"sqlite:////{db_path}",
f"sqlite:///{db_path.as_posix()}",
metadata=EntityMetadata(id="db", name="Test DB"),
include=["child"],
)
Expand Down
6 changes: 3 additions & 3 deletions tests/database/test_geopackage.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def test_dataset_is_enriched(self, tmp_path: Path) -> None:
_make_geopackage(gpkg, geometry_type_name="MULTIPOLYGON")
catalog = Catalog(app_path=tmp_path / "app", quiet=True)
catalog.add_database(
f"sqlite:////{gpkg}",
f"sqlite:///{gpkg.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
)
parcels = catalog.dataset.get_by("name", "parcels")
Expand All @@ -190,7 +190,7 @@ def test_bbox_exports_as_json_array(self, tmp_path: Path) -> None:
app_dir = tmp_path / "app"
catalog = Catalog(app_path=app_dir, quiet=True)
catalog.add_database(
f"sqlite:////{gpkg}",
f"sqlite:///{gpkg.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
)
catalog.export_db()
Expand All @@ -210,7 +210,7 @@ def test_plain_sqlite_leaves_geo_fields_null(self, tmp_path: Path) -> None:
conn.close()
catalog = Catalog(app_path=tmp_path / "app", quiet=True)
catalog.add_database(
f"sqlite:////{db}",
f"sqlite:///{db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
)
things = catalog.dataset.get_by("name", "things")
Expand Down
32 changes: 16 additions & 16 deletions tests/database/test_sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@ class TestSQLite(BaseDatabaseTests):
def db(
self, sample_sqlite_db: Path
) -> Generator[tuple[ibis.BaseBackend, str, str], None, None]:
con, _ = connect(f"sqlite:////{sample_sqlite_db}")
con, _ = connect(f"sqlite:///{sample_sqlite_db.as_posix()}")
yield con, "sqlite", "sqlite"
con.disconnect()

@pytest.fixture
def db_with_employees(
self, sample_sqlite_db: Path
) -> Generator[tuple[ibis.BaseBackend, str, str], None, None]:
con, _ = connect(f"sqlite:////{sample_sqlite_db}")
con, _ = connect(f"sqlite:///{sample_sqlite_db.as_posix()}")
yield con, "sqlite", "sqlite"
con.disconnect()

Expand Down Expand Up @@ -67,7 +67,7 @@ def test_rescan_marks_existing_prefix_folders_as_seen(
) -> None:
"""Rescanning database should mark existing prefix folders as _seen=True."""
app_dir = tmp_path
conn_str = f"sqlite:////{sample_sqlite_db}"
conn_str = f"sqlite:///{sample_sqlite_db.as_posix()}"

# First scan with prefix grouping
catalog1 = Catalog(app_path=app_dir, quiet=True)
Expand Down Expand Up @@ -124,7 +124,7 @@ def test_time_series_groups_tables(self, ts_sqlite_db: Path) -> None:
"""Tables with temporal pattern are grouped into a single dataset."""
catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{ts_sqlite_db}",
f"sqlite:///{ts_sqlite_db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
group_by_prefix=False,
)
Expand All @@ -143,7 +143,7 @@ def test_time_series_variable_periods(self, ts_sqlite_db: Path) -> None:
"""Variable start_date/end_date reflect schema evolution."""
catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{ts_sqlite_db}",
f"sqlite:///{ts_sqlite_db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
group_by_prefix=False,
)
Expand All @@ -164,7 +164,7 @@ def test_time_series_disabled(self, ts_sqlite_db: Path) -> None:
"""time_series=False keeps tables separate."""
catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{ts_sqlite_db}",
f"sqlite:///{ts_sqlite_db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
time_series=False,
group_by_prefix=False,
Expand All @@ -176,7 +176,7 @@ def test_time_series_structure_mode(self, ts_sqlite_db: Path) -> None:
"""Structure mode creates series dataset without scanning."""
catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{ts_sqlite_db}",
f"sqlite:///{ts_sqlite_db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
depth="dataset",
group_by_prefix=False,
Expand All @@ -191,7 +191,7 @@ def test_time_series_schema_mode(self, ts_sqlite_db: Path) -> None:
"""Schema mode scans columns but not stats."""
catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{ts_sqlite_db}",
f"sqlite:///{ts_sqlite_db.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
depth="variable",
group_by_prefix=False,
Expand Down Expand Up @@ -227,7 +227,7 @@ def test_time_series_with_prefix(self, tmp_path: Path) -> None:

catalog = Catalog(quiet=True)
catalog.add_database(
f"sqlite:////{db_path}",
f"sqlite:///{db_path.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
group_by_prefix=True,
prefix_min_tables=2,
Expand All @@ -251,7 +251,7 @@ def test_time_series_with_prefix(self, tmp_path: Path) -> None:

def test_time_series_rescan(self, ts_sqlite_db: Path) -> None:
"""Rescanning removes old series dataset and creates new one."""
conn_str = f"sqlite:////{ts_sqlite_db}"
conn_str = f"sqlite:///{ts_sqlite_db.as_posix()}"
catalog = Catalog(quiet=True)
catalog.add_database(
conn_str,
Expand Down Expand Up @@ -295,7 +295,7 @@ def fail_on_full(con, name, **kw):

with patch("datannurpy.add_database.scan_table", side_effect=fail_on_full):
catalog.add_database(
f"sqlite:////{db_path}",
f"sqlite:///{db_path.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
group_by_prefix=False,
)
Expand All @@ -322,7 +322,7 @@ def fail_all(*a, **kw):

with patch("datannurpy.add_database.scan_table", side_effect=fail_all):
catalog.add_database(
f"sqlite:////{db_path}",
f"sqlite:///{db_path.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
depth="variable",
group_by_prefix=False,
Expand Down Expand Up @@ -358,7 +358,7 @@ def fail_first(con, name, **kw):

with patch("datannurpy.add_database.scan_table", side_effect=fail_first):
catalog.add_database(
f"sqlite:////{db_path}",
f"sqlite:///{db_path.as_posix()}",
metadata=EntityMetadata(id="db", name="DB"),
group_by_prefix=False,
)
Expand All @@ -378,7 +378,7 @@ def test_geopackage_connect(self, gpkg_path: Path) -> None:
"""Test connecting to a GeoPackage file."""
if not gpkg_path.exists():
pytest.skip("GeoPackage test file not available")
con, backend = connect(f"sqlite:////{gpkg_path}")
con, backend = connect(f"sqlite:///{gpkg_path.as_posix()}")
try:
assert backend == "sqlite"
assert con is not None
Expand All @@ -389,7 +389,7 @@ def test_geopackage_list_tables(self, gpkg_path: Path) -> None:
"""Test listing tables in a GeoPackage (excludes system tables)."""
if not gpkg_path.exists():
pytest.skip("GeoPackage test file not available")
con, _ = connect(f"sqlite:////{gpkg_path}")
con, _ = connect(f"sqlite:///{gpkg_path.as_posix()}")
try:
tables = list_tables(con)
# Should have data tables
Expand All @@ -407,7 +407,7 @@ def test_geopackage_scan_table_with_geometry(self, gpkg_path: Path) -> None:
"""Test scanning a table with geometry columns (POINT → geometry type)."""
if not gpkg_path.exists():
pytest.skip("GeoPackage test file not available")
con, _ = connect(f"sqlite:////{gpkg_path}")
con, _ = connect(f"sqlite:///{gpkg_path.as_posix()}")
try:
# Project has: id, geom (POINT → geometry), ProjectName, etc.
variables, row_count, _, freq_table = scan_table(
Expand Down
Loading