diff --git a/.dockerignore b/.dockerignore index 5e818d1..708970f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,7 +12,7 @@ wheels/ .envrc CLAUDE.md AGENTS.md -kolkhoz.log +funes.log *.log *.db *.db-journal diff --git a/.env.example b/.env.example index e1da065..df651ea 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ -# Pravda is an in-process async library. Kolkhoz owns and runs the +# Pravda is an in-process async library. Funes owns and runs the # infrastructure it connects to: a remote headed Chrome (Playwright -# run-server), a Postgres database shared by Pravda and Kolkhoz, and an fsspec +# run-server), a Postgres database shared by Pravda and Funes, and an fsspec # artifact store. Both applications use this asyncpg URL through SQLAlchemy. # Bring them up with: docker compose up -d PRAVDA_DATABASE_URL=postgresql+asyncpg://pravda:pravda@localhost:5432/pravda diff --git a/AGENTS.md b/AGENTS.md index ebe2dcf..74b68e0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ -# Kolkhoz +# Funes -Kolkhoz is an orchestrator that turns raw web pages into structured data about political position holders. It is built on top of Pravda, the evidence layer that captures and stores durable snapshots of web pages. +Funes is an orchestrator that turns raw web pages into structured data about political position holders. It is built on top of Pravda, the evidence layer that captures and stores durable snapshots of web pages. ## Project philosophy @@ -9,18 +9,18 @@ Kolkhoz is an orchestrator that turns raw web pages into structured data about p ## Stack - **Python** 3.13+ managed by **uv**. -- **Pravda** ([github.com/opensanctions/pravda](https://github.com/opensanctions/pravda)), published on PyPI as `opensanctions-pravda` (imported as `pravda`), for web page capture and storage, embedded as an in-process async library. Kolkhoz owns the infrastructure Pravda connects to — a headed Chrome browser (remote Playwright server), an async Postgres database, and an fsspec artifact store — run via `docker compose`. Connection settings are `PRAVDA_DATABASE_URL`, `PRAVDA_BROWSER_WS_URL`, and `PRAVDA_STORAGE_BASE_PATH` (see `.env`). Kolkhoz constructs Pravda's `PravdaConfig` at the CLI boundary, reads artifacts from the shared storage backend over fsspec, and applies Pravda's packaged migrations (`pravda.migrate`) idempotently before the `run` command opens a `Pravda` instance. -- Kolkhoz does not persist extraction results. Each `run` writes only that run's records as JSONL to `OUTPUT_BASE_PATH`; PostgreSQL is used only by Pravda. +- **Pravda** ([github.com/opensanctions/pravda](https://github.com/opensanctions/pravda)), published on PyPI as `opensanctions-pravda` (imported as `pravda`), for web page capture and storage, embedded as an in-process async library. Funes owns the infrastructure Pravda connects to — a headed Chrome browser (remote Playwright server), an async Postgres database, and an fsspec artifact store — run via `docker compose`. Connection settings are `PRAVDA_DATABASE_URL`, `PRAVDA_BROWSER_WS_URL`, and `PRAVDA_STORAGE_BASE_PATH` (see `.env`). Funes constructs Pravda's `PravdaConfig` at the CLI boundary, reads artifacts from the shared storage backend over fsspec, and applies Pravda's packaged migrations (`pravda.migrate`) idempotently before the `run` command opens a `Pravda` instance. +- Funes does not persist extraction results. Each `run` writes only that run's records as JSONL to `OUTPUT_BASE_PATH`; PostgreSQL is used only by Pravda. ## Project structure ``` -kolkhoz/ # the package: cli.py (run), capture.py +funes/ # the package: cli.py (run), capture.py # (Pravda client + capture/artifact helpers), extract.py, # sources.py, export.py, config.py evaluate.py # score the extraction pipeline against synthetic fixtures fixtures/ # one directory per fixture: page.html, expected.json, optional screenshot.png -docker-compose.yml # Kolkhoz-owned browser (Playwright server) + Postgres +docker-compose.yml # Funes-owned browser (Playwright server) + Postgres ``` `input/` (gitignored) holds the input CSVs (one dataset per file). `output/` diff --git a/Dockerfile b/Dockerfile index 9934938..3667d93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.13-slim -LABEL org.opencontainers.image.title="Kolkhoz" -LABEL org.opencontainers.image.source="https://github.com/opensanctions/kolkhoz" +LABEL org.opencontainers.image.title="Funes" +LABEL org.opencontainers.image.source="https://github.com/opensanctions/funes" ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -24,7 +24,7 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN uv sync --frozen --no-install-project --no-dev -COPY kolkhoz ./kolkhoz +COPY funes ./funes COPY evaluate.py README.md ./ RUN uv sync --frozen --no-dev @@ -39,5 +39,5 @@ USER app # CronJob spec). ENV INPUT_BASE_PATH=/app/datasets -ENTRYPOINT ["kolkhoz"] +ENTRYPOINT ["funes"] CMD ["run"] diff --git a/README.md b/README.md index 861d7d8..334542a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Kolkhoz +# Funes -Kolkhoz turns the internet into lists of politicians. It orchestrates web capture (via the in-process [Pravda](https://github.com/opensanctions/pravda) async library), LLM extraction, and JSONL output to pull political position holders out of web pages. +Funes turns the internet into lists of politicians. It orchestrates web capture (via the in-process [Pravda](https://github.com/opensanctions/pravda) async library), LLM extraction, and JSONL output to pull political position holders out of web pages. ## Status @@ -8,15 +8,15 @@ Early R&D. Currently exploring what a viable automated extraction pipeline looks ## What it does -1. Captures snapshots (plaintext + rendered HTML + screenshot) via the in-process Pravda library against a remote browser, Postgres, and an artifact store that Kolkhoz owns and runs +1. Captures snapshots (plaintext + rendered HTML + screenshot) via the in-process Pravda library against a remote browser, Postgres, and an artifact store that Funes owns and runs 2. Feeds snapshots to an LLM to extract structured "human / position" pairs 3. Writes that run's extracted holders as JSONL (one record per person/position observation), shaped for ingest by zavod -Kolkhoz does not persist extraction results in PostgreSQL. Pravda still stores the captured evidence and snapshot metadata there. +Funes does not persist extraction results in PostgreSQL. Pravda still stores the captured evidence and snapshot metadata there. ## Setup -Requires uv. Kolkhoz embeds Pravda as an async library and +Requires uv. Funes embeds Pravda as an async library and owns the infrastructure Pravda connects to: a headed Chrome browser, a Postgres database, and an artifact store. Pravda ships on PyPI as `opensanctions-pravda` (imported as `pravda`); `uv sync` installs it. @@ -34,7 +34,7 @@ The `run` command applies Pravda's packaged Alembic migrations idempotently befo ## Usage -All commands run through the `kolkhoz` console script (installed by `uv sync`). Input and output +All commands run through the `funes` console script (installed by `uv sync`). Input and output locations are fsspec URLs set via `INPUT_BASE_PATH` and `OUTPUT_BASE_PATH` in `.env` (a local dir or a `gs://`/`s3://` bucket prefix): @@ -43,10 +43,10 @@ in `.env` (a local dir or a `gs://`/`s3://` bucket prefix): # through Pravda, then extract position holders from each snapshot and write # this run's JSONL to //.jsonl. Each CSV is # its own dataset, named after the file's stem. -uv run kolkhoz run -uv run kolkhoz run -d hio_leadership # one dataset only -uv run kolkhoz run -n 20 # random sample of 20 page inputs -uv run kolkhoz run -c 10 # up to 10 concurrent captures +uv run funes run +uv run funes run -d hio_leadership # one dataset only +uv run funes run -n 20 # random sample of 20 page inputs +uv run funes run -c 10 # up to 10 concurrent captures ``` ## Evaluation diff --git a/docker-compose.yml b/docker-compose.yml index 5d4c5ee..3a809e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ -# Kolkhoz-owned infrastructure for Pravda's in-process async client. +# Funes-owned infrastructure for Pravda's in-process async client. # -# Pravda connects from Kolkhoz's process to three things Kolkhoz owns and +# Pravda connects from Funes's process to three things Funes owns and # runs here: a remote headed Chrome (Playwright run-server), Pravda's Postgres # database, and the artifact store (a local path on disk by default, # configured via PRAVDA_STORAGE_BASE_PATH). diff --git a/evaluate.py b/evaluate.py index b33bd1a..8dd9e6e 100644 --- a/evaluate.py +++ b/evaluate.py @@ -33,8 +33,8 @@ from bs4 import BeautifulSoup from openai import OpenAI -from kolkhoz.config import load_config -from kolkhoz.extract import extract, flatten_persons, metadata_from_html +from funes.config import load_config +from funes.extract import extract, flatten_persons, metadata_from_html log = logging.getLogger("evaluate") diff --git a/kolkhoz/__init__.py b/funes/__init__.py similarity index 50% rename from kolkhoz/__init__.py rename to funes/__init__.py index cf69126..ec294e8 100644 --- a/kolkhoz/__init__.py +++ b/funes/__init__.py @@ -1,4 +1,4 @@ -"""Kolkhoz: turn web pages into structured data about political position holders. +"""Funes: turn web pages into structured data about political position holders. Orchestrates web capture (via Pravda), LLM extraction, and PostgreSQL storage. """ diff --git a/kolkhoz/capture.py b/funes/capture.py similarity index 95% rename from kolkhoz/capture.py rename to funes/capture.py index c63d872..13a6a74 100644 --- a/kolkhoz/capture.py +++ b/funes/capture.py @@ -3,11 +3,11 @@ - ``capture_urls`` captures each URL once, concurrently, through one long-lived Pravda instance (bounded by a semaphore). - ``read_artifact`` reads a snapshot artifact blob from the shared fsspec - storage backend Kolkhoz and Pravda both use. + storage backend Funes and Pravda both use. - ``is_blank`` / ``split_image`` are image primitives over a screenshot blob (shared by capture and the extraction tiling path). -Pravda is an in-process async library. Kolkhoz constructs one ``Pravda`` +Pravda is an in-process async library. Funes constructs one ``Pravda`` instance from its environment-backed settings and reuses it across captures; it never speaks HTTP to Pravda. """ @@ -24,16 +24,16 @@ from PIL import Image from pravda import Pravda, PravdaConfig, Snapshot -from kolkhoz.config import PravdaSettings +from funes.config import PravdaSettings -log = logging.getLogger("kolkhoz") +log = logging.getLogger("funes") def pravda_client(settings: PravdaSettings) -> Pravda: - """Construct a Pravda instance from Kolkhoz's environment-backed settings. + """Construct a Pravda instance from Funes's environment-backed settings. The ``PravdaConfig`` is built here, at the application boundary, from the - explicit settings Kolkhoz owns; Kolkhoz holds no Pravda URL of its own. + explicit settings Funes owns; Funes holds no Pravda URL of its own. """ config = PravdaConfig( database_url=settings.database_url, diff --git a/kolkhoz/cli.py b/funes/cli.py similarity index 94% rename from kolkhoz/cli.py rename to funes/cli.py index 93a3450..bc5bcf5 100644 --- a/kolkhoz/cli.py +++ b/funes/cli.py @@ -8,7 +8,7 @@ from openai import OpenAI from pravda import Snapshot, migrate -from kolkhoz.capture import ( +from funes.capture import ( OperationalError, capture_urls, is_blank, @@ -17,24 +17,24 @@ storage_filesystem, summarise_errors, ) -from kolkhoz.config import Config, load_config -from kolkhoz.export import holder_to_record, write_outputs -from kolkhoz.extract import ( +from funes.config import Config, load_config +from funes.export import holder_to_record, write_outputs +from funes.extract import ( extract, flatten_persons, metadata_from_html, screenshot_reason, ) -from kolkhoz.sources import InputRow, load_inputs +from funes.sources import InputRow, load_inputs -log = logging.getLogger("kolkhoz") +log = logging.getLogger("funes") @click.group() def cli() -> None: root = logging.getLogger() root.setLevel(logging.INFO) - root.addHandler(logging.FileHandler("kolkhoz.log")) + root.addHandler(logging.FileHandler("funes.log")) root.addHandler(logging.StreamHandler()) @@ -99,7 +99,7 @@ async def _run_pipeline( A URL repeated within a dataset collapses to one page association. The same URL in several datasets remains several associations, backed by one Pravda capture. Sampling happens over associations before capture. - Kolkhoz keeps no database state of its own; Pravda alone persists the + Funes keeps no database state of its own; Pravda alone persists the captured evidence. """ associations: list[tuple[str, str, str]] = [] diff --git a/kolkhoz/config.py b/funes/config.py similarity index 96% rename from kolkhoz/config.py rename to funes/config.py index 4f73a8a..8974ddd 100644 --- a/kolkhoz/config.py +++ b/funes/config.py @@ -19,7 +19,7 @@ class PravdaSettings: """Environment-backed settings for Pravda's in-process async client. These three values are handed to Pravda's own ``PravdaConfig`` at the - application boundary. The database URL is also used by Kolkhoz. + application boundary. The database URL is also used by Funes. """ database_url: str diff --git a/kolkhoz/export.py b/funes/export.py similarity index 96% rename from kolkhoz/export.py rename to funes/export.py index b990950..dac674c 100644 --- a/kolkhoz/export.py +++ b/funes/export.py @@ -9,9 +9,9 @@ from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper from pravda import Snapshot -from kolkhoz.config import PathsConfig +from funes.config import PathsConfig -log = logging.getLogger("kolkhoz") +log = logging.getLogger("funes") # The flat JSONL schema handed to zavod. One record per (person, position) # observation extracted from one snapshot. Fixed key order, UTF-8, source diff --git a/kolkhoz/extract.py b/funes/extract.py similarity index 99% rename from kolkhoz/extract.py rename to funes/extract.py index 00d8115..8d65d34 100644 --- a/kolkhoz/extract.py +++ b/funes/extract.py @@ -5,7 +5,7 @@ text/screenshot decision (``screenshot_reason``), and the ``extract()`` call. ``screenshot_parts`` tiles a screenshot blob into model input parts using ``split_image`` from -:mod:`kolkhoz.capture`. +:mod:`funes.capture`. The OpenAI client is an explicit argument: it is constructed at the CLI boundary, not held as module-global state. @@ -18,10 +18,10 @@ from openai import OpenAI from pydantic import BaseModel, Field -from kolkhoz.capture import split_image -from kolkhoz.config import ImageConfig, ModelConfig +from funes.capture import split_image +from funes.config import ImageConfig, ModelConfig -log = logging.getLogger("kolkhoz") +log = logging.getLogger("funes") REASONING_EFFORT = "low" diff --git a/kolkhoz/sources.py b/funes/sources.py similarity index 100% rename from kolkhoz/sources.py rename to funes/sources.py diff --git a/pyproject.toml b/pyproject.toml index ed86465..0de3696 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "kolkhoz" +name = "funes" version = "0.1.0" description = "Orchestrate web capture and LLM extraction of political position holders into structured data." readme = "README.md" @@ -21,7 +21,7 @@ dev = [ ] [project.scripts] -kolkhoz = "kolkhoz.cli:cli" +funes = "funes.cli:cli" [build-system] requires = ["hatchling"] diff --git a/uv.lock b/uv.lock index 3a2fad8..8e10e4f 100644 --- a/uv.lock +++ b/uv.lock @@ -531,6 +531,43 @@ gcs = [ { name = "gcsfs" }, ] +[[package]] +name = "funes" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "click" }, + { name = "fsspec", extra = ["gcs"] }, + { name = "openai" }, + { name = "opensanctions-pravda" }, + { name = "pillow" }, + { name = "python-dotenv" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pre-commit" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "beautifulsoup4", specifier = ">=4.15.0" }, + { name = "click", specifier = ">=8.4.1" }, + { name = "fsspec", extras = ["gcs"], specifier = ">=2026.6.0" }, + { name = "openai", specifier = ">=2.41.1" }, + { name = "opensanctions-pravda", specifier = ">=0.1.3" }, + { name = "pillow", specifier = ">=12.2.0" }, + { name = "python-dotenv", specifier = ">=1.2.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pre-commit", specifier = ">=4.6.0" }, + { name = "ruff", specifier = ">=0.15.16" }, +] + [[package]] name = "gcsfs" version = "2026.7.0" @@ -917,43 +954,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/a4/cf8d779feb133a27a2e3bc833bccb9e13aa332cdf820497ebf72c10ce8c3/jiter-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0", size = 191244, upload-time = "2026-05-19T10:09:12.74Z" }, ] -[[package]] -name = "kolkhoz" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "click" }, - { name = "fsspec", extra = ["gcs"] }, - { name = "openai" }, - { name = "opensanctions-pravda" }, - { name = "pillow" }, - { name = "python-dotenv" }, -] - -[package.dev-dependencies] -dev = [ - { name = "pre-commit" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "beautifulsoup4", specifier = ">=4.15.0" }, - { name = "click", specifier = ">=8.4.1" }, - { name = "fsspec", extras = ["gcs"], specifier = ">=2026.6.0" }, - { name = "openai", specifier = ">=2.41.1" }, - { name = "opensanctions-pravda", specifier = ">=0.1.3" }, - { name = "pillow", specifier = ">=12.2.0" }, - { name = "python-dotenv", specifier = ">=1.2.2" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pre-commit", specifier = ">=4.6.0" }, - { name = "ruff", specifier = ">=0.15.16" }, -] - [[package]] name = "mako" version = "1.3.12"