diff --git a/CHANGES.rst b/CHANGES.rst index 5fcb788067..32b29c1327 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,12 @@ noirlab - Restore access to the `NSF NOIRLab `_ `Astro Data Archive `_ [#3359]. +lco +^^^ + +- New module to access the `Las Cumbres Observatory (LCO) `_ + `Science Archive `_. [#3659] + esa.emds ^^^^^^^^ diff --git a/astroquery/lco/__init__.py b/astroquery/lco/__init__.py new file mode 100644 index 0000000000..1b19f2ab52 --- /dev/null +++ b/astroquery/lco/__init__.py @@ -0,0 +1,39 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# Generative AI was used in the making of this module + +""" +Las Cumbres Observatory (LCO) +""" + +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.lco`. + """ + archive_url = _config.ConfigItem( + ['https://archive-api.lco.global', ], + 'The LCO Archive mirror to use.') + + token_url = _config.ConfigItem( + 'https://observe.lco.global/api/api-token-auth/', + 'URL used to exchange a username and password for an API token.') + + timeout = _config.ConfigItem( + 30, + 'Time limit for connecting to LCO Archive server.') + + row_limit = _config.ConfigItem( + 100, + 'Maximum number of frames returned by a query. Set to -1 to retrieve ' + 'every matching frame.') + + +conf = Conf() + +from astroquery.lco.core import LcoArchive, LcoArchiveClass + +__all__ = ['LcoArchive', 'LcoArchiveClass', + 'Conf', 'conf', + ] diff --git a/astroquery/lco/core.py b/astroquery/lco/core.py new file mode 100644 index 0000000000..f14c191190 --- /dev/null +++ b/astroquery/lco/core.py @@ -0,0 +1,809 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Provide astroquery API access to Las Cumbres Observatory (LCO)'s Archive. + +This accesses the LCO's archive web service with or without an API Token. +""" + +import contextlib +import os +import warnings +from urllib.parse import parse_qs, urlencode, urlparse, urlunparse + +import keyring +import numpy as np +import requests + +import astropy.units as u +from astropy.coordinates import Angle +from astropy.table import MaskedColumn, Table +from astropy.time import Time +from astropy.utils.console import ProgressBarOrSpinner, Spinner + +from astroquery import cache_conf, log +from astroquery.exceptions import (LargeQueryWarning, LoginError, MaxResultsWarning, + NoResultsWarning, RemoteServiceError) +from astroquery.lco import conf +from astroquery.query import QueryWithLogin +from astroquery.utils import commons + + +__all__ = ['LcoArchive', 'LcoArchiveClass'] + + +# Filters accepted by the ``/frames/`` endpoint. Anything not in here is +# rejected client-side so that typos do not silently return the whole archive. +FRAME_FILTERS = ( + 'basename', + 'basename_exact', + 'configuration_type', + 'covers', + 'empty_target_name', + 'end', + 'exclude_calibrations', + 'exclude_configuration_type', + 'exposure_time', + 'id__gt', + 'include_configuration_type', + 'instrument_id', + 'intersects', + 'observation_id', + 'primary_optical_element', + 'processed', + 'proposal_id', + 'public', + 'reduction_level', + 'request_id', + 'site_id', + 'start', + 'submitter', + 'target_name', + 'target_name_exact', + 'target_name_iexact', + 'telescope_id', +) + +# Scalar columns kept in the result table, in display order. The fields ``related_frames``, +# ``version_set`` and ``area`` do not fit a flat table, so ``ra`` and ``dec`` give the +# center of ``area`` instead. +RESULT_COLUMNS = ( + 'id', + 'basename', + 'filename', + 'observation_date', + 'observation_day', + 'proposal_id', + 'target_name', + 'ra', + 'dec', + 'site_id', + 'telescope_id', + 'instrument_id', + 'configuration_type', + 'primary_optical_element', + 'exposure_time', + 'reduction_level', + 'observation_id', + 'request_id', + 'public_date', + 'url', +) + +# The archive caps page size at 100 for anonymous users and 1000 for +# authenticated ones. +ANONYMOUS_PAGE_SIZE = 100 +AUTHENTICATED_PAGE_SIZE = 1000 + +# Thumbnail sizes the archive publishes. +THUMBNAIL_SIZES = ('small', 'large') + +# LCO Archive presigned URLs are only valid for this long, so only +# cache responses for this long. +CACHE_TIMEOUT = 48 * 60 * 60 + +# How many frames an unbounded query may collect before it warns that it is +# still going. +LARGE_RESULT_WARNING = 10000 + + +class LcoArchiveClass(QueryWithLogin): + """ + Search functionality for the LCO Archive. + + The archive is open to anonymous users, who see every frame whose + proprietary period has expired. Logging in with an API token additionally + exposes the proprietary frames belonging to your proposals. + """ + + ARCHIVE_URL = conf.archive_url + TOKEN_URL = conf.token_url + TIMEOUT = conf.timeout + ROW_LIMIT = conf.row_limit + + @property + def _frames_url(self): + return f'{self.ARCHIVE_URL}/frames/' + + @property + def _page_size(self): + return (AUTHENTICATED_PAGE_SIZE if self.authenticated() + else ANONYMOUS_PAGE_SIZE) + + def _request(self, *args, **kwargs): + # Frame responses carry presigned download links, so no response from + # the archive may be reused for longer than those stay valid. + with cache_conf.set_temp('cache_timeout', CACHE_TIMEOUT): + return super()._request(*args, **kwargs) + + def _login(self, *, token=None, username=None, password=None, + store_password=False, reenter_password=False): + """ + Authenticate against the LCO archive. + + Parameters + ---------- + token : str, optional + An LCO API token, as shown on your profile page at + https://observe.lco.global/accounts/profile/. If given, the + username and password are not used or needed. + username : str, optional + LCO observing portal username. Used to request a token when + ``token`` is not supplied; the password is taken from the system + keyring, or prompted for. + password : str, optional + Password for ``username``. Prompted for when omitted. + store_password : bool, optional + Store the password in the system keyring for later sessions. + Default is `False`. + reenter_password : bool, optional + Prompt for the password even when one is already in the keyring, + which is how you overwrite a stored password. Default is `False`. + + Returns + ------- + success : bool + Whether the session is now authenticated. + """ + if token is None: + if username is None: + raise LoginError("One of 'token' or 'username' is required " + "to log in to the LCO archive.") + if password is None: + password, password_from_keyring = self._get_password( + "astroquery:observe.lco.global", username, reenter=reenter_password) + else: + password_from_keyring = None + + # The token exchange happens against the observing portal, + response = self._request('POST', self.TOKEN_URL, + json={'username': username, + 'password': password}, + timeout=self.TIMEOUT, cache=False) + if response.status_code != 200: + log.error("Unable to log in to the LCO observing portal, " + "please check your credentials.") + return False + + token = response.json().get('token') + if not token: + log.error("The LCO observing portal did not return a token.") + return False + + if store_password and password_from_keyring is None and password and username: + keyring.set_password("astroquery:observe.lco.global", username, password) + + self._session.headers['Authorization'] = f'Token {token}' + + # Confirm the token is actually good. An anonymous or rejected + # session comes back with an empty username. + profile = self._request('GET', f'{self.ARCHIVE_URL}/profile/', + timeout=self.TIMEOUT, cache=False) + if profile.status_code != 200 or not profile.json().get('username'): + log.error("The LCO archive rejected the supplied API token.") + del self._session.headers['Authorization'] + return False + + log.info(f"Authenticated to the LCO archive as " + f"{profile.json()['username']}.") + return True + + def logout(self): + """Drop the API token and return the session to anonymous access.""" + self._session.headers.pop('Authorization', None) + self._authenticated = False + + def query_region(self, coordinates, *, radius=None, width=None, + height=None, show_progress=True, + get_query_payload=False, cache=None, **criteria): + """ + Query for frames covering, or overlapping, a position on the sky. + + With no ``radius``, ``width`` or ``height`` this returns frames whose + footprint contains ``coordinates``. With any of them it returns frames + whose footprint overlaps the requested region. + + Parameters + ---------- + coordinates : str or `~astropy.coordinates.SkyCoord` + The single position to search around. + radius : str or `~astropy.units.Quantity`, optional + Radius of a cone search. The circle is sent to the archive as a + 32-vertex polygon inscribed in it, so the region searched is + accurate to 0.5% of the radius specified. + width : str or `~astropy.units.Quantity`, optional + Width of a box search. Must be given with ``height``. + height : str or `~astropy.units.Quantity`, optional + Height of a box search. Must be given with ``width``. + show_progress : bool, optional + Display progress while retrieving a query that needs more than one + request. Default is `True`. + get_query_payload : bool, optional + Return the dict of HTTP request parameters without querying. + Default is `False`. + cache : bool, optional + Cache the response. Defaults to `True` for anonymous queries and + `False` for authenticated ones. + **criteria + Any other frame filter, as accepted by + :meth:`~astroquery.lco.LcoArchiveClass.query_criteria`. + + Returns + ------- + frames : `~astropy.table.Table` + The matching frames, one row per frame. + """ + coordinates = commons.parse_coordinates(coordinates).icrs + + if width is not None or height is not None: + if width is None or height is None: + raise ValueError("Both 'width' and 'height' are required for " + "a box search.") + if radius is not None: + raise ValueError("Give either 'radius' or 'width' and " + "'height', not both.") + criteria['intersects'] = _box_to_wkt(coordinates, Angle(width), + Angle(height)) + elif radius is not None: + criteria['intersects'] = _circle_to_wkt(coordinates, Angle(radius)) + else: + criteria['covers'] = f'POINT({coordinates.ra.deg} {coordinates.dec.deg})' + + return self.query_criteria(get_query_payload=get_query_payload, + cache=cache, + show_progress=show_progress, + **criteria) + + def query_object(self, object_name, *, exact=True, + show_progress=True, get_query_payload=False, + cache=None, **criteria): + """ + Query for frames whose target name matches ``object_name``. + + This matches on the name the observer submitted with the observation + request, and does not resolve the name or search on position. Use + :meth:`~astroquery.lco.LcoArchiveClass.query_region` to search by + position instead. + + Parameters + ---------- + object_name : str + Target name to search for. + exact : bool, optional + Require the target name to match exactly. `False` performs case- + insensitive searches that contain ``object_name``. Default is `True`. + show_progress : bool, optional + Display progress while retrieving a query that needs more than one + request. Default is `True`. + get_query_payload : bool, optional + Return the dict of HTTP request parameters without querying. + Default is `False`. + cache : bool, optional + Cache the response. Defaults to `True` for anonymous queries and + `False` for authenticated ones. + **criteria + Any other frame filter, as accepted by + :meth:`~astroquery.lco.LcoArchiveClass.query_criteria`. + + Returns + ------- + frames : `~astropy.table.Table` + The matching frames, one row per frame. + """ + name_filter = 'target_name_exact' if exact else 'target_name' + return self.query_criteria(**{name_filter: object_name}, + get_query_payload=get_query_payload, + cache=cache, + show_progress=show_progress, + **criteria) + + def query_criteria(self, *, get_query_payload=False, cache=None, + row_limit=None, include_thumbnails=False, + thumbnail_size='small', show_progress=True, + **criteria): + """ + Query the archive on any combination of frame filters. + + Parameters + ---------- + **criteria + Frame filters to apply, as ``name=value`` pairs. Most are matched + against the archive as given, e.g. ``site_id='lsc'`` or + ``target_name_exact='M101'``; + :meth:`~astroquery.lco.LcoArchiveClass.list_criteria` returns every + accepted name, and the ``list_*`` methods give the accepted values + for the code-valued ones. The criteria below are the ones that + take a value you cannot guess from the name: + + ``start``, ``end`` + Bounds on the observation date. Accepts a string, an + `~astropy.time.Time`, or a `~datetime.datetime`. + ``covers``, ``intersects`` + A `Well-Known Text (WKT) + `_ region the frame + footprint must contain, or overlap. + :meth:`~astroquery.lco.LcoArchiveClass.query_region` builds + these for you from a position and an optional extent. + ``exposure_time`` + Seconds, either as a number or an + `~astropy.units.Quantity` that can be converted to seconds. + ``processed`` + `True` matches all processed reduced data frames, `False` matches + only raw data frames. When left out, both frame types are returned. + ``public`` + `True` allows selection to include public frames, `False` only + returns frames from proposals which your account is a member of. + ``reduction_level`` + Specific numeric reduction level to select on. + ``0`` for raw and ``91`` for reduced data. + ``empty_target_name`` + `True` selects only frames whose target name is blank. This + is a flag rather than a choice: `False` does not select the + frames that *have* a name, it simply leaves the filter off. + ``include_configuration_type``, ``exclude_configuration_type`` + Keep, or drop, several exposure types at once. These are the + only criteria that accept a sequence, e.g. + ``exclude_configuration_type=['BIAS', 'DARK']``; anywhere else + a sequence is rejected, because the archive would silently use + only its last value. + row_limit : int, optional + Maximum number of frames to return, at least 1. Defaults to + ``LcoArchiveClass.ROW_LIMIT``; ``-1`` retrieves every match. + include_thumbnails : bool, optional + Ask the archive for thumbnail links alongside the frame metadata, + which adds ``thumbnail_url`` and ``thumbnail_filename`` columns to + the result. Default is `False`. + thumbnail_size : str, optional + Which thumbnail to keep, ``'small'`` or ``'large'``. Only used + when ``include_thumbnails`` is `True`. Default is ``'small'``. + show_progress : bool, optional + Display progress while retrieving a query that needs more than one + request. Default is `True`. + get_query_payload : bool, optional + Return the dict of HTTP request parameters without querying. + Default is `False`. + cache : bool, optional + Cache the response. Defaults to `True` for anonymous queries and + `False` for authenticated ones. + + Returns + ------- + frames : `~astropy.table.Table` + The matching frames, one row per frame. + """ + if include_thumbnails and thumbnail_size not in THUMBNAIL_SIZES: + raise ValueError(f"'thumbnail_size' must be one of " + f"{', '.join(THUMBNAIL_SIZES)}.") + row_limit = self.ROW_LIMIT if row_limit is None else row_limit + if row_limit != -1 and row_limit < 1: + # A page of no rows tells a cursor query nothing, so there is no + # sensible request to make for it. + raise ValueError("'row_limit' must be 1 or more, or -1 to " + f"retrieve every matching frame; got {row_limit}.") + + payload = self._args_to_payload(**criteria) + if include_thumbnails: + payload['include_thumbnails'] = 'true' + if get_query_payload: + return payload + + rows = self._fetch_pages(payload, row_limit=row_limit, cache=cache, + show_progress=show_progress) + return self._parse_result(rows, thumbnail_size=thumbnail_size) + + def _args_to_payload(self, **criteria): + """Validate and normalise user criteria into archive query params.""" + payload = {} + + for key, value in criteria.items(): + if value is None: + continue + + if key not in FRAME_FILTERS: + raise ValueError( + f"'{key}' is not a supported LCO archive filter. " + f"Supported filters are: {', '.join(FRAME_FILTERS)}.") + + if key in ('exclude_configuration_type', 'include_configuration_type'): + # Accepts either a single value, or a list of values. + payload[key] = value if isinstance(value, str) else list(value) + continue + + if key == 'empty_target_name': + # The archive searches for targets with no defined name for any + # truthy value, so only include the parameter if it is truthy + if value: + payload[key] = 'true' + continue + + if key in ('start', 'end') and not isinstance(value, str): + # Time accepts a Time, a datetime, or anything else it can + # parse; strings are passed through as the user wrote them. + value = Time(value).isot + elif key in ('public', 'exclude_calibrations', 'processed'): + value = 'true' if value else 'false' + elif key == 'exposure_time' and isinstance(value, u.Quantity): + value = value.to(u.s).value + + payload[key] = value + + return payload + + def _progress(self, row_limit, show_progress): + """ + A progress display for a query that needs more than one request. + + A single-page query is over before any display would be useful, so + those get nothing. ``row_limit`` is used to determine if we should + show a progress bar, counting spinner, or nothing. + """ + if not show_progress or 0 < row_limit <= self._page_size: + return contextlib.nullcontext() + return _ProgressBarOrCountingSpinner( + row_limit if row_limit > 0 else None, + 'Retrieving frames from the LCO archive') + + def _fetch_pages(self, payload, *, row_limit, cache=None, + show_progress=True): + """ + Retrieve as many pages as ``row_limit`` requires. + + Cursor pagination is used since it is more efficient on the server + """ + if cache is None: + # Never cache an authenticated request: the token would be + # pickled along with the response, and the cache key does not + # include session headers, so anonymous and authenticated results + # for the same query would collide. + cache = not self.authenticated() + + params = dict(payload, pagination_style='cursor') + + rows = [] + remaining = row_limit + url = self._frames_url + warned = False + truncated = False + + with self._progress(row_limit, show_progress) as progress: + while url is not None: + # Ask for exactly what is still wanted, so the last page does + # not fetch rows that would only be thrown away. + page_size = (self._page_size if row_limit < 0 + else min(remaining, self._page_size)) + if params is None: + # The 'next' link includes the previous limit, so we need + # to substitute the value in the url if it has changed. + url = _replace_limit(url, page_size) + else: + params['limit'] = page_size + + response = self._request('GET', url, params=params, + timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + + # Keep the results from the response but drop the response to + # save memory on large queries. + page = response.json() + del response + rows.extend(page['results']) + if progress is not None: + # Never report more than was asked for: the bar's total is + # the row limit, so going past it would read as over 100%. + progress.update(len(rows) if row_limit < 0 + else min(len(rows), row_limit)) + + remaining -= len(page['results']) + if row_limit > 0 and remaining <= 0: + # A 'next' link on the last page we keep means the archive + # holds more frames than the caller asked for. + truncated = bool(page.get('next')) + break + + # An unbounded query can page until it exhausts memory, so if + # we exceed the LARGE_RESULT_WARNING number of results, report + # the potential issue so users can stop it early if they want. + if (row_limit < 0 and not warned + and len(rows) >= LARGE_RESULT_WARNING): + warnings.warn( + f"This query has returned {len(rows):,} frames so far " + "and is still paging. Narrow it with 'start' and " + "'end', or set 'row_limit', to stop it sooner.", + LargeQueryWarning) + warned = True + + # The 'next' link carries the cursor, so nothing else is needed. + url, params = page.get('next'), None + + # At the end of the query, report if we truncated the results set. + if truncated: + warnings.warn( + f"Results truncated to {row_limit} frames. Pass row_limit=-1 " + "to retrieve every matching frame.", MaxResultsWarning) + + return rows + + def _parse_result(self, rows, *, thumbnail_size): + """Turn the frame records from a query into a `~astropy.table.Table`.""" + if not rows: + warnings.warn("Query returned no results.", NoResultsWarning) + return Table(names=RESULT_COLUMNS, + dtype=[object] * len(RESULT_COLUMNS)) + + table = Table([[row.get(name) for row in rows] + for name in RESULT_COLUMNS], + names=RESULT_COLUMNS) + table['exposure_time'].unit = u.s + + # The footprint center is the mean of its vertices taken as unit + # vectors, so footprints that straddle RA 0 or a pole come out right. + # Frames without a polygon footprint are masked. + ra = np.ma.masked_all(len(rows)) + dec = np.ma.masked_all(len(rows)) + for i, row in enumerate(rows): + area = row.get('area') + if not area or area.get('type') != 'Polygon': + continue + # GeoJSON repeats the first vertex to close the ring, which would + # otherwise be counted twice. + lon, lat = np.radians(area['coordinates'][0][:-1]).T + x = np.mean(np.cos(lat) * np.cos(lon)) + y = np.mean(np.cos(lat) * np.sin(lon)) + z = np.mean(np.sin(lat)) + # The archive's longitudes run from -180 to 180. + ra[i] = np.degrees(np.arctan2(y, x)) % 360 + dec[i] = np.degrees(np.arctan2(z, np.hypot(x, y))) + table['ra'] = MaskedColumn(ra, unit=u.deg) + table['dec'] = MaskedColumn(dec, unit=u.deg) + + # Thumbnails only come back when the query asked for them. The archive + # returns every size, so keep the one the query asked for and flatten + # it into columns download_thumbnails can use. + if any('thumbnails' in row for row in rows): + thumbnails = [next((t for t in row.get('thumbnails') or [] + if t.get('size') == thumbnail_size), None) + for row in rows] + table['thumbnail_url'] = [t['url'] if t else '' for t in thumbnails] + table['thumbnail_filename'] = [ + f"{t['basename']}{t['extension']}" if t else '' + for t in thumbnails] + + return table + + def get_metadata(self, frame_id, *, cache=None): + """ + Retrieve the metadata for a single frame, including a fresh download URL. + + Parameters + ---------- + frame_id : int + The frame's ``id``. + cache : bool, optional + Cache the response. Defaults to `True` for anonymous queries and + `False` for authenticated ones. + + Returns + ------- + frame : dict + The frame record as returned by the archive, including the nested + ``version_set``, ``area`` and ``related_frames`` fields that the + query methods drop. + """ + if cache is None: + cache = not self.authenticated() + response = self._request('GET', f'{self._frames_url}{frame_id}/', + timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + return response.json() + + def download_files(self, frames, *, download_dir='.', overwrite=False): + """ + Download the data files for a set of frames. + + Parameters + ---------- + frames : `~astropy.table.Table`, int, or list of int + A table returned by one of the query methods, or one or more frame + ids. Frame ids are looked up one by one. + download_dir : str, optional + Directory to write into. Defaults to the working directory. + overwrite : bool, optional + Re-download files that already exist locally. Default is `False`. + + Returns + ------- + paths : list of str + The local paths of the downloaded files. + """ + if isinstance(frames, Table): + records = [{'filename': row['filename'], 'url': row['url']} + for row in frames] + else: + if np.isscalar(frames): + frames = [frames] + records = [self.get_metadata(frame_id) for frame_id in frames] + + return self._download_records(records, download_dir=download_dir, + overwrite=overwrite) + + def download_thumbnails(self, frames, *, download_dir='.', + overwrite=False): + """ + Download the JPEG thumbnails for a set of frames. + + The table must have been produced by a query with + ``include_thumbnails=True``, which is what puts the ``thumbnail_url`` + column in it. Frames with no thumbnail of the requested size are + skipped. + + Parameters + ---------- + frames : `~astropy.table.Table` + A table returned by one of the query methods. + download_dir : str, optional + Directory to write into. Defaults to the working directory. + overwrite : bool, optional + Re-download thumbnails that already exist locally. Default is + `False`. + + Returns + ------- + paths : list of str + The local paths of the downloaded thumbnails. + """ + if 'thumbnail_url' not in frames.colnames: + raise ValueError( + "This table has no 'thumbnail_url' column. Re-run the query " + "with include_thumbnails=True to request thumbnail links.") + + records = [{'filename': row['thumbnail_filename'], + 'url': row['thumbnail_url']} + for row in frames if row['thumbnail_url']] + return self._download_records(records, download_dir=download_dir, + overwrite=overwrite) + + def _download_records(self, records, *, download_dir, overwrite): + """Download ``{'filename', 'url'}`` records, skipping existing files.""" + paths = [] + for record in records: + local_filepath = os.path.join(download_dir, record['filename']) + if os.path.exists(local_filepath) and not overwrite: + log.info(f"Found existing {local_filepath}, skipping download. " + f"Pass overwrite=True to download it again.") + else: + try: + # The URL carries its own credentials in the query string, + # and S3 rejects a request that also has an Authorization + # header with "Only one auth mechanism allowed". A None + # value drops the header for this request alone, leaving + # the logged-in session untouched. + self._download_file(record['url'], local_filepath, + timeout=self.TIMEOUT, cache=False, + continuation=False, + headers={'Authorization': None}) + # The URLs are presigned and valid for 48 hours, + # and return a 403 after their validity expires + except requests.exceptions.HTTPError as exc: + if getattr(exc.response, 'status_code', None) != 403: + raise + raise RemoteServiceError( + f"The download link for {record['filename']} has " + "expired. LCO signs them for 48 hours; run the query " + "again to get fresh links.") from exc + paths.append(local_filepath) + return paths + + def list_criteria(self): + """ + List the frame filters the query methods accept. + + Returns + ------- + criteria : list of str + """ + return list(FRAME_FILTERS) + + def _aggregate(self, field, *, cache=True): + response = self._request('GET', f'{self._frames_url}aggregate/', + timeout=self.TIMEOUT, cache=cache) + response.raise_for_status() + return sorted(response.json()[field]) + + def list_sites(self): + """List the site codes present in the archive.""" + return self._aggregate('sites') + + def list_telescopes(self): + """List the telescope codes present in the archive.""" + return self._aggregate('telescopes') + + def list_instruments(self): + """List the instrument codes present in the archive.""" + return self._aggregate('instruments') + + def list_filters(self): + """List the ``primary_optical_element`` values present in the archive.""" + return self._aggregate('filters') + + def list_configuration_types(self): + """List the ``configuration_type`` values present in the archive.""" + return self._aggregate('obstypes') + + def list_proposals(self): + """List the proposal ids present in the archive.""" + return self._aggregate('proposals') + + +class _ProgressBarOrCountingSpinner(ProgressBarOrSpinner): + """ + A `~astropy.utils.console.ProgressBarOrSpinner` that keeps a running count + in its spinner form. + + This is used on an unbounded query (row_count = -1) so that the current + number of rows retrieved is shown instead of just a spinner. + """ + + def __init__(self, total, msg, **kwargs): + self._msg, self._kwargs = msg, kwargs + super().__init__(total, msg, **kwargs) + + def update(self, value): + if self._is_spinner: + # Must instantiate a new spinner to give it an updated message. + # Each subsequent message overwrites the previous one. + self._obj = Spinner(f'{self._msg} ({value:,} so far)', + **self._kwargs) + self._obj.update(value) + + +def _replace_limit(url, limit): + """Return ``url`` with its ``limit`` query parameter replaced.""" + parts = urlparse(url) + query = parse_qs(parts.query) + query['limit'] = [str(limit)] + return urlunparse(parts._replace(query=urlencode(query, doseq=True))) + + +def _ring_to_wkt(ra, dec): + """Build a closed WKT polygon from arrays of RA and Dec in degrees.""" + vertices = ', '.join(f'{a} {d}' for a, d in zip(ra, dec)) + return f'POLYGON(({vertices}, {ra[0]} {dec[0]}))' + + +def _circle_to_wkt(coordinates, radius, *, nvertices=32): + """Approximate a cone on the sky as a polygon.""" + position_angles = np.linspace(0, 360, nvertices, endpoint=False) * u.deg + ring = coordinates.directional_offset_by(position_angles, radius) + return _ring_to_wkt(ring.ra.deg, ring.dec.deg) + + +def _box_to_wkt(coordinates, width, height): + """Build a polygon for a box centered on ``coordinates``.""" + half_height = height.to(u.deg) / 2 + half_width = width.to(u.deg) / 2 / np.cos(coordinates.dec.radian) + + ra = coordinates.ra.deg + np.array([-1, 1, 1, -1]) * half_width.value + dec = coordinates.dec.deg + np.array([-1, -1, 1, 1]) * half_height.value + return _ring_to_wkt(ra, np.clip(dec, -90, 90)) + + +LcoArchive = LcoArchiveClass() diff --git a/astroquery/lco/tests/__init__.py b/astroquery/lco/tests/__init__.py new file mode 100644 index 0000000000..9dce85d06f --- /dev/null +++ b/astroquery/lco/tests/__init__.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/astroquery/lco/tests/data/aggregate.json b/astroquery/lco/tests/data/aggregate.json new file mode 100644 index 0000000000..9ad245c851 --- /dev/null +++ b/astroquery/lco/tests/data/aggregate.json @@ -0,0 +1,45 @@ +{ + "sites": [ + "bpl", + "coj", + "cpt", + "elp", + "lsc" + ], + "telescopes": [ + "0m4a", + "0m4b", + "0m4c", + "0m8a", + "1m0a" + ], + "filters": [ + "gp", + "rp", + "up", + "ip", + "zp", + "R", + "B" + ], + "instruments": [ + "fa16", + "kb80", + "ep05", + "fl08", + "ak01" + ], + "obstypes": [ + "ARC", + "BIAS", + "BPM", + "CATALOG", + "DARK" + ], + "proposals": [ + "TEST-PROPOSAL-001", + "TEST-PROPOSAL-002", + "TEST-PROPOSAL-003" + ], + "generated_at": "2026-09-01T19:45:03.216Z" +} diff --git a/astroquery/lco/tests/data/frames.json b/astroquery/lco/tests/data/frames.json new file mode 100644 index 0000000000..66ef94b7dc --- /dev/null +++ b/astroquery/lco/tests/data/frames.json @@ -0,0 +1,202 @@ +{ + "next": null, + "previous": null, + "results": [ + { + "id": 97456069, + "basename": "lsc1m005-fa15-20240101-0001-e91", + "observation_date": "2024-01-01T00:00:00.000000Z", + "observation_day": "2024-01-01", + "proposal_id": "TEST-PROPOSAL-001", + "instrument_id": "fa15", + "target_name": "TEST-TARGET", + "reduction_level": 91, + "site_id": "lsc", + "telescope_id": "1m0a", + "exposure_time": 600.0, + "primary_optical_element": "SII", + "public_date": "2024-01-01T00:00:00.000000Z", + "configuration_type": "EXPOSE", + "observation_id": 830398287, + "request_id": 4365225, + "version_set": [ + { + "id": 99780209, + "key": "TESTKEY99780209", + "md5": "00000000000000000000000000000000", + "extension": ".fits.fz", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/processed/lsc1m005-fa15-20240101-0001-e91.fits.fz?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000", + "created": "2024-01-01T00:05:00.000000Z" + } + ], + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/processed/lsc1m005-fa15-20240101-0001-e91.fits.fz?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000", + "filename": "lsc1m005-fa15-20240101-0001-e91.fits.fz", + "thumbnails": [ + { + "id": 16591406, + "frame": 97456069, + "size": "large", + "basename": "lsc1m005-fa15-20240101-0001-e91-large_thumbnail", + "extension": ".jpg", + "key": "TESTKEY16591406", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/thumbnails/lsc1m005-fa15-20240101-0001-e91-large_thumbnail.jpg?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000" + }, + { + "id": 16591407, + "frame": 97456069, + "size": "small", + "basename": "lsc1m005-fa15-20240101-0001-e91-small_thumbnail", + "extension": ".jpg", + "key": "TESTKEY16591407", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/thumbnails/lsc1m005-fa15-20240101-0001-e91-small_thumbnail.jpg?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "DATE_OBS": "2024-01-01T00:00:00.000000Z", + "DAY_OBS": "2024-01-01", + "PROPID": "TEST-PROPOSAL-001", + "INSTRUME": "fa15", + "OBJECT": "TEST-TARGET", + "RLEVEL": 91, + "SITEID": "lsc", + "TELID": "1m0a", + "EXPTIME": 600.0, + "FILTER": "SII", + "L1PUBDAT": "2024-01-01T00:00:00.000000Z", + "OBSTYPE": "EXPOSE", + "BLKUID": 830398287, + "REQNUM": 4365225, + "area": { + "type": "Polygon", + "coordinates": [ + [ + [ + -148.79887538711287, + 54.08620189033855 + ], + [ + -148.78857713225287, + 54.58254204685201 + ], + [ + -149.64511461767603, + 54.58556850167396 + ], + [ + -149.64513499437402, + 54.089192029060975 + ], + [ + -148.79887538711287, + 54.08620189033855 + ] + ] + ] + }, + "related_frames": [ + 65362998, + 97116214, + 97396742, + 97425596, + 97425830 + ] + }, + { + "id": 97456152, + "basename": "lsc1m005-fa15-20240101-0002-e91", + "observation_date": "2024-01-01T01:00:00.000000Z", + "observation_day": "2024-01-01", + "proposal_id": "TEST-PROPOSAL-001", + "instrument_id": "fa15", + "target_name": "TEST-TARGET", + "reduction_level": 91, + "site_id": "lsc", + "telescope_id": "1m0a", + "exposure_time": 600.0, + "primary_optical_element": "OIII", + "public_date": "2024-01-01T01:00:00.000000Z", + "configuration_type": "EXPOSE", + "observation_id": 830398287, + "request_id": 4365225, + "version_set": [ + { + "id": 99780320, + "key": "TESTKEY99780320", + "md5": "00000000000000000000000000000000", + "extension": ".fits.fz", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/processed/lsc1m005-fa15-20240101-0002-e91.fits.fz?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000", + "created": "2024-01-01T01:05:00.000000Z" + } + ], + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/processed/lsc1m005-fa15-20240101-0002-e91.fits.fz?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000", + "filename": "lsc1m005-fa15-20240101-0002-e91.fits.fz", + "thumbnails": [ + { + "id": 16591463, + "frame": 97456152, + "size": "large", + "basename": "lsc1m005-fa15-20240101-0002-e91-large_thumbnail", + "extension": ".jpg", + "key": "TESTKEY16591463", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/thumbnails/lsc1m005-fa15-20240101-0002-e91-large_thumbnail.jpg?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000" + }, + { + "id": 16591465, + "frame": 97456152, + "size": "small", + "basename": "lsc1m005-fa15-20240101-0002-e91-small_thumbnail", + "extension": ".jpg", + "key": "TESTKEY16591465", + "url": "https://archive-lco-global.s3.us-west-2.amazonaws.com/lsc/fa15/20240101/thumbnails/lsc1m005-fa15-20240101-0002-e91-small_thumbnail.jpg?versionId=TESTVERSIONID&X-Amz-Signature=0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "DATE_OBS": "2024-01-01T01:00:00.000000Z", + "DAY_OBS": "2024-01-01", + "PROPID": "TEST-PROPOSAL-001", + "INSTRUME": "fa15", + "OBJECT": "TEST-TARGET", + "RLEVEL": 91, + "SITEID": "lsc", + "TELID": "1m0a", + "EXPTIME": 600.0, + "FILTER": "OIII", + "L1PUBDAT": "2024-01-01T01:00:00.000000Z", + "OBSTYPE": "EXPOSE", + "BLKUID": 830398287, + "REQNUM": 4365225, + "area": { + "type": "Polygon", + "coordinates": [ + [ + [ + -148.80001611886118, + 54.08635364673273 + ], + [ + -148.7896832551423, + 54.582697650479396 + ], + [ + -149.6462310710969, + 54.58574431806907 + ], + [ + -149.6462857943921, + 54.08936375516897 + ], + [ + -148.80001611886118, + 54.08635364673273 + ] + ] + ] + }, + "related_frames": [ + 65362998, + 97256703, + 97396733, + 97425596, + 97425830 + ] + } + ] +} diff --git a/astroquery/lco/tests/test_lco.py b/astroquery/lco/tests/test_lco.py new file mode 100644 index 0000000000..2281c8a86a --- /dev/null +++ b/astroquery/lco/tests/test_lco.py @@ -0,0 +1,554 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import json +import os +import warnings + +import pytest +import requests + +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.table import Table +from astropy.utils import console + +from astroquery.exceptions import (LargeQueryWarning, MaxResultsWarning, + NoResultsWarning, RemoteServiceError) +from astroquery.lco import core, LcoArchiveClass +from astroquery.utils.mocks import MockResponse + + +DATA_FILES = {'frames': 'frames.json', + 'aggregate': 'aggregate.json'} + + +def data_path(filename): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + return os.path.join(data_dir, filename) + + +def read_data(name): + with open(data_path(DATA_FILES[name])) as f: + return json.load(f) + + +def nonremote_request(self, method, url, **kwargs): + name = 'aggregate' if url.endswith('aggregate/') else 'frames' + with open(data_path(DATA_FILES[name]), 'rb') as f: + return MockResponse(content=f.read(), url=url) + + +@pytest.fixture +def patch_request(request): + mp = request.getfixturevalue("monkeypatch") + mp.setattr(LcoArchiveClass, '_request', nonremote_request) + return mp + + +@pytest.fixture +def lco(): + return LcoArchiveClass() + + +# --- payload construction: no network needed at all ------------------------- + +def test_query_criteria_payload(lco): + payload = lco.query_criteria(proposal_id='LCOEPO2014B-010', + site_id='lsc', + reduction_level=91, + get_query_payload=True) + assert payload == {'proposal_id': 'LCOEPO2014B-010', + 'site_id': 'lsc', + 'reduction_level': 91} + + +def test_query_criteria_drops_none(lco): + payload = lco.query_criteria(site_id='lsc', target_name=None, + get_query_payload=True) + assert payload == {'site_id': 'lsc'} + + +def test_public_is_lowercased_bool(lco): + assert lco.query_criteria(public=True, + get_query_payload=True)['public'] == 'true' + assert lco.query_criteria(public=False, + get_query_payload=True)['public'] == 'false' + + +def test_processed_is_lowercased_bool(lco): + assert lco.query_criteria(processed=True, + get_query_payload=True) == {'processed': 'true'} + # Unlike empty_target_name, False is a choice rather than an absent flag: + # it selects the raw frames, so it has to be sent. + assert lco.query_criteria(processed=False, + get_query_payload=True) == {'processed': 'false'} + + +def test_exposure_time_quantity_becomes_seconds(lco): + payload = lco.query_criteria(exposure_time=2*u.min, get_query_payload=True) + assert payload['exposure_time'] == 120.0 + + +def test_start_end_accept_time_objects(lco): + from astropy.time import Time + payload = lco.query_criteria(start=Time('2024-01-01'), + end='2024-02-01', get_query_payload=True) + assert payload['start'].startswith('2024-01-01') + assert payload['end'] == '2024-02-01' + + +# 'PROPID' is one of the upper case names the archive no longer accepts; it +# is rejected the same way any other unknown filter is. +@pytest.mark.parametrize('unknown', ['instrument', 'PROPID']) +def test_unknown_filter_is_rejected(lco, unknown): + with pytest.raises(ValueError, match="not a supported LCO archive filter"): + lco.query_criteria(**{unknown: 'x'}, get_query_payload=True) + + +def test_multi_value_filter_accepts_a_sequence(lco): + payload = lco.query_criteria( + exclude_configuration_type=['BIAS', 'DARK'], get_query_payload=True) + # requests turns a list value into a repeated query parameter, which is + # what the archive's MultipleChoiceFilter expects. + assert payload == {'exclude_configuration_type': ['BIAS', 'DARK']} + + +def test_multi_value_filter_accepts_a_single_value(lco): + payload = lco.query_criteria(include_configuration_type='BIAS', + get_query_payload=True) + assert payload == {'include_configuration_type': 'BIAS'} + + +def test_empty_target_name_is_a_presence_flag(lco): + # The archive activates this filter for any non-empty value, so a falsy + # request has to leave the parameter out rather than send 'false'. + assert lco.query_criteria(empty_target_name=True, + get_query_payload=True) == { + 'empty_target_name': 'true'} + assert lco.query_criteria(empty_target_name=False, site_id='lsc', + get_query_payload=True) == {'site_id': 'lsc'} + + +def test_query_object_matches_exactly_by_default(lco): + payload = lco.query_object('M101', get_query_payload=True) + assert payload == {'target_name_exact': 'M101'} + + +def test_query_object_substring_match(lco): + payload = lco.query_object('M101', exact=False, get_query_payload=True) + assert payload == {'target_name': 'M101'} + + +def test_query_region_without_radius_uses_covers(lco): + coord = SkyCoord(210.80242917, 54.34875, unit='deg') + payload = lco.query_region(coord, get_query_payload=True) + assert payload['covers'] == 'POINT(210.80242917 54.34875)' + assert 'intersects' not in payload + + +def test_query_region_with_radius_uses_polygon(lco): + coord = SkyCoord(210.80242917, 54.34875, unit='deg') + payload = lco.query_region(coord, radius=0.1*u.deg, get_query_payload=True) + wkt = payload['intersects'] + assert wkt.startswith('POLYGON((') and wkt.endswith('))') + # 32 vertices plus the repeated closing vertex. + assert len(wkt[len('POLYGON(('):-2].split(',')) == 33 + + +def test_query_region_box(lco): + coord = SkyCoord(0, 0, unit='deg') + payload = lco.query_region(coord, width=1*u.deg, height=2*u.deg, + get_query_payload=True) + assert payload['intersects'].startswith('POLYGON((') + + +def test_query_region_rejects_radius_and_box(lco): + coord = SkyCoord(0, 0, unit='deg') + with pytest.raises(ValueError, match="not both"): + lco.query_region(coord, radius=1*u.deg, width=1*u.deg, height=1*u.deg, + get_query_payload=True) + + +def test_query_region_requires_both_box_sides(lco): + coord = SkyCoord(0, 0, unit='deg') + with pytest.raises(ValueError, match="Both 'width' and 'height'"): + lco.query_region(coord, width=1*u.deg, get_query_payload=True) + + +@pytest.mark.parametrize('row_limit', [0, -2]) +def test_meaningless_row_limit_is_rejected(lco, row_limit): + with pytest.raises(ValueError, match="'row_limit' must be 1 or more"): + lco.query_criteria(site_id='lsc', row_limit=row_limit, + get_query_payload=True) + + +def test_bad_thumbnail_size(lco): + with pytest.raises(ValueError, match="thumbnail_size"): + lco.query_criteria(site_id='lsc', include_thumbnails=True, + thumbnail_size='enormous', get_query_payload=True) + + # The size is unused without thumbnails, so it is not checked. + payload = lco.query_criteria(site_id='lsc', thumbnail_size='enormous', + get_query_payload=True) + assert payload == {'site_id': 'lsc'} + + +# --- parsing, against a saved archive response ------------------------------ + +def test_query_returns_table(patch_request, lco): + result = lco.query_object('TEST-TARGET', row_limit=2) + assert isinstance(result, Table) + assert len(result) == 2 + assert result['target_name'][0] == 'TEST-TARGET' + assert result['exposure_time'].unit == u.s + + +def test_deprecated_columns_are_dropped(patch_request, lco): + result = lco.query_object('TEST-TARGET', row_limit=2) + assert 'PROPID' not in result.colnames + assert 'proposal_id' in result.colnames + + +def test_nested_fields_are_dropped(patch_request, lco): + result = lco.query_object('TEST-TARGET', row_limit=2) + for column in ('area', 'version_set', 'related_frames'): + assert column not in result.colnames + + +def test_footprint_center_columns(patch_request, lco): + result = lco.query_object('TEST-TARGET', row_limit=2) + assert result['ra'].unit == u.deg + assert result['dec'].unit == u.deg + # The saved footprint spans RA 210.36 to 211.20, stored as longitudes in + # [-180, 180], and Dec 54.09 to 54.59. + assert result['ra'][0] == pytest.approx(210.7806, abs=1e-4) + assert result['dec'][0] == pytest.approx(54.3366, abs=1e-4) + + +@pytest.mark.parametrize('ring, center', [ + # Straddles RA 0, where the archive's longitudes change sign. + ([(-0.5, 9.5), (-0.5, 10.5), (0.5, 10.5), (0.5, 9.5)], (0, 10)), + # Straddles RA 180, where they jump from 180 to -180. + ([(179.5, -20.5), (179.5, -19.5), (-179.5, -19.5), (-179.5, -20.5)], (180, -20)), + # Surrounds the north pole. + ([(0, 89.5), (90, 89.5), (180, 89.5), (-90, 89.5)], (0, 90)), +]) +def test_footprint_center_handles_wraps_and_poles(lco, monkeypatch, ring, center): + row = dict(read_data('frames')['results'][0], + area={'type': 'Polygon', 'coordinates': [ring + ring[:1]]}) + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': [row], 'next': None}])) + + result = lco.query_object('TEST-TARGET', row_limit=1) + # A wrap or pole mistake would be off by degrees; the vertex mean of a + # square in RA/Dec is only off its center by arcseconds. + found = SkyCoord(result['ra'][0], result['dec'][0], unit='deg') + assert found.separation(SkyCoord(*center, unit='deg')) < 0.01*u.deg + + +def test_frames_without_a_footprint_have_a_masked_center(lco, monkeypatch): + rows = read_data('frames')['results'] + rows[1]['area'] = None + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': rows, 'next': None}])) + + result = lco.query_object('TEST-TARGET', row_limit=2) + assert not result['ra'].mask[0] + assert result['ra'].mask[1] + assert result['dec'].mask[1] + + +def test_thumbnail_columns(patch_request, lco): + result = lco.query_object('TEST-TARGET', row_limit=2, + include_thumbnails=True, thumbnail_size='small') + assert 'thumbnail_url' in result.colnames + assert result['thumbnail_filename'][0].endswith('-small_thumbnail.jpg') + + result = lco.query_object('TEST-TARGET', row_limit=2, + include_thumbnails=True, thumbnail_size='large') + assert result['thumbnail_filename'][0].endswith('-large_thumbnail.jpg') + + +def test_empty_result_warns(patch_request, lco, monkeypatch): + def empty_request(self, method, url, **kwargs): + return MockResponse(content=json.dumps({'results': [], + 'next': None}).encode(), + url=url) + + monkeypatch.setattr(LcoArchiveClass, '_request', empty_request) + with pytest.warns(NoResultsWarning): + result = lco.query_object('nothing-matches-this') + assert len(result) == 0 + assert 'basename' in result.colnames + assert 'ra' in result.colnames + + +@pytest.mark.parametrize('method, key', + [('list_sites', 'sites'), + ('list_telescopes', 'telescopes'), + ('list_instruments', 'instruments'), + ('list_filters', 'filters'), + ('list_configuration_types', 'obstypes'), + ('list_proposals', 'proposals')]) +def test_list_helpers(patch_request, lco, method, key): + assert getattr(lco, method)() == sorted(read_data('aggregate')[key]) + + +def test_download_drops_the_auth_header(lco, monkeypatch, tmp_path): + # S3 rejects a presigned URL that also carries an Authorization header + # with "Only one auth mechanism allowed", so an authenticated session + # must not send its token to the download host. + seen = {} + + def record_headers(self, url, local_filepath, **kwargs): + seen.update(kwargs.get('headers') or {}) + return local_filepath + + monkeypatch.setattr(LcoArchiveClass, '_download_file', record_headers) + lco._session.headers['Authorization'] = 'Token sekrit' + + frames = Table({'filename': ['a.fits.fz'], 'url': ['https://s3/a?X-Amz-Signature=x']}) + lco.download_files(frames, download_dir=str(tmp_path)) + + assert seen['Authorization'] is None + # The session itself keeps the token for further queries. + assert lco._session.headers['Authorization'] == 'Token sekrit' + + +def test_expired_download_link_explains_itself(lco, monkeypatch, tmp_path): + def expired(self, url, local_filepath, **kwargs): + response = requests.Response() + response.status_code = 403 + raise requests.exceptions.HTTPError(response=response) + + monkeypatch.setattr(LcoArchiveClass, '_download_file', expired) + frames = Table({'filename': ['a.fits.fz'], 'url': ['https://expired']}) + with pytest.raises(RemoteServiceError, match='has expired'): + lco.download_files(frames, download_dir=str(tmp_path)) + + +def test_other_download_errors_are_not_swallowed(lco, monkeypatch, tmp_path): + def not_found(self, url, local_filepath, **kwargs): + response = requests.Response() + response.status_code = 404 + raise requests.exceptions.HTTPError(response=response) + + monkeypatch.setattr(LcoArchiveClass, '_download_file', not_found) + frames = Table({'filename': ['a.fits.fz'], 'url': ['https://missing']}) + with pytest.raises(requests.exceptions.HTTPError): + lco.download_files(frames, download_dir=str(tmp_path)) + + +def test_download_thumbnails_requires_the_column(lco): + # A table from a query that did not ask for thumbnails has no links to + # download, and should say so rather than failing later. + with pytest.raises(ValueError, match="include_thumbnails=True"): + lco.download_thumbnails(Table({'id': [1], 'basename': ['a']})) + + +# --- authentication --------------------------------------------------------- + +def test_login_with_token_sets_header(lco, monkeypatch): + def fake_request(self, method, url, **kwargs): + assert url.endswith('/profile/') + return MockResponse(content=json.dumps({'username': 'jnation'}).encode(), + url=url) + + monkeypatch.setattr(LcoArchiveClass, '_request', fake_request) + lco.login(token='sekrit') + assert lco._session.headers['Authorization'] == 'Token sekrit' + assert lco.authenticated() + + lco.logout() + assert 'Authorization' not in lco._session.headers + assert not lco.authenticated() + + +def test_bad_token_is_rejected_and_header_removed(lco, monkeypatch): + def anonymous_profile(self, method, url, **kwargs): + return MockResponse(content=json.dumps({'username': ''}).encode(), + url=url) + + monkeypatch.setattr(LcoArchiveClass, '_request', anonymous_profile) + lco.login(token='wrong') + assert not lco.authenticated() + assert 'Authorization' not in lco._session.headers + + +def test_login_needs_token_or_username(lco): + from astroquery.exceptions import LoginError + with pytest.raises(LoginError): + lco.login() + + +def test_page_size_depends_on_auth(lco): + assert lco._page_size == 100 + lco._authenticated = True + assert lco._page_size == 1000 + + +def test_authenticated_queries_are_not_cached(lco, monkeypatch): + seen = {} + + def record_cache(self, method, url, **kwargs): + seen['cache'] = kwargs.get('cache') + return MockResponse(content=json.dumps({'results': [], + 'next': None}).encode(), + url=url) + + monkeypatch.setattr(LcoArchiveClass, '_request', record_cache) + + with pytest.warns(NoResultsWarning): + lco.query_object('TEST-TARGET') + assert seen['cache'] is True + + lco._authenticated = True + with pytest.warns(NoResultsWarning): + lco.query_object('TEST-TARGET') + assert seen['cache'] is False + + +def paged_request(pages): + """Serve a canned sequence of pages, one per request.""" + served = iter(pages) + + def request(self, method, url, **kwargs): + return MockResponse(content=json.dumps(next(served)).encode(), url=url) + + return request + + +def test_truncation_warns(lco, monkeypatch): + rows = read_data('frames')['results'] + # One page that fills the row limit, with more waiting behind it. + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': rows, 'next': 'https://archive-api.lco.global/frames/' + '?cursor=TESTCURSOR&limit=2'}])) + + with pytest.warns(MaxResultsWarning, match='truncated to 2 frames'): + result = lco.query_object('TEST-TARGET', row_limit=2) + assert len(result) == 2 + + +def test_no_truncation_warning_when_the_archive_is_exhausted(lco, monkeypatch): + rows = read_data('frames')['results'] + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': rows, 'next': None}])) + + warnings.simplefilter('error') + result = lco.query_object('TEST-TARGET', row_limit=2) + assert len(result) == 2 + + +def test_no_truncation_warning_when_unlimited(lco, monkeypatch): + rows = read_data('frames')['results'] + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': rows, 'next': 'https://archive-api.lco.global/frames/' + '?cursor=TESTCURSOR&limit=2'}, + {'results': rows, 'next': None}])) + + warnings.simplefilter('error') + result = lco.query_object('TEST-TARGET', row_limit=-1) + assert len(result) == 4 + + +def test_unbounded_query_warns_once_it_is_large(lco, monkeypatch): + # An unbounded query should warn if the result is large. + monkeypatch.setattr(core, 'LARGE_RESULT_WARNING', 3) + row = read_data('frames')['results'][0] + pages = [{'results': [row, row], 'next': 'https://a/frames/?cursor=C'}, + {'results': [row, row], 'next': 'https://a/frames/?cursor=D'}, + {'results': [row, row], 'next': None}] + + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request(pages)) + with pytest.warns(LargeQueryWarning, match='still paging'): + result = lco.query_object('TEST-TARGET', row_limit=-1) + assert len(result) == 6 + + +def test_bounded_query_does_not_warn_about_size(lco, monkeypatch): + # A row_limit stops on its own, so a large result set is intentional. + monkeypatch.setattr(core, 'LARGE_RESULT_WARNING', 1) + row = read_data('frames')['results'][0] + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': [row, row], 'next': None}])) + + warnings.simplefilter('error', LargeQueryWarning) + assert len(lco.query_object('TEST-TARGET', row_limit=2)) == 2 + + +@pytest.mark.parametrize('show_progress, expect_output', [(True, True), + (False, False)]) +def test_show_progress_controls_the_display(lco, monkeypatch, capsys, + show_progress, expect_output): + row = read_data('frames')['results'][0] + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': [row], 'next': 'https://a/frames/?cursor=C'}, + {'results': [row], 'next': None}])) + + # Two pages, so the display would otherwise be shown. + lco.query_object('TEST-TARGET', row_limit=-1, show_progress=show_progress) + assert bool(capsys.readouterr().out) is expect_output + + +def test_counting_spinner_reports_the_running_count(): + # A spinner cannot fill a bar, so an unbounded query reports how many + # frames it has instead. + progress = core._ProgressBarOrCountingSpinner(None, 'Retrieving') + assert progress._is_spinner + progress.update(12345) + assert progress._obj._msg == 'Retrieving (12,345 so far)' + + +def test_counting_progress_leaves_a_real_bar_alone(monkeypatch, tmp_path): + # A known total on a terminal gives a real progress bar, which reports its + # own counts; only the spinner needs replacing to change its message. + monkeypatch.setattr(console, 'isatty', lambda f: True) + with open(tmp_path / 'out', 'w') as stream: + progress = core._ProgressBarOrCountingSpinner(500, 'Retrieving', file=stream) + assert progress._is_spinner is False + bar = progress._obj + progress.update(250) + assert progress._obj is bar + + +def test_progress_never_reports_more_than_the_row_limit(lco, monkeypatch): + # The bar's total is the row limit, so a page that overshoots it must not + # be reported as more than 100%. + seen = [] + + class Recorder: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def update(self, value): + seen.append(value) + + monkeypatch.setattr(LcoArchiveClass, '_progress', + lambda self, row_limit, show: Recorder()) + row = read_data('frames')['results'][0] + monkeypatch.setattr(LcoArchiveClass, '_request', paged_request( + [{'results': [row] * 10, 'next': None}])) + + lco.query_object('TEST-TARGET', row_limit=4) + assert seen == [4] + + +def test_cursor_pagination_is_requested(lco, monkeypatch): + seen = {} + + def record_params(self, method, url, **kwargs): + seen.update(kwargs.get('params') or {}) + return MockResponse(content=json.dumps({'results': [], + 'next': None}).encode(), + url=url) + + monkeypatch.setattr(LcoArchiveClass, '_request', record_params) + with pytest.warns(NoResultsWarning): + lco.query_object('TEST-TARGET', row_limit=10) + + assert seen['pagination_style'] == 'cursor' + assert seen['limit'] == 10 diff --git a/astroquery/lco/tests/test_lco_remote.py b/astroquery/lco/tests/test_lco_remote.py new file mode 100644 index 0000000000..7b3f87b390 --- /dev/null +++ b/astroquery/lco/tests/test_lco_remote.py @@ -0,0 +1,265 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import os +import warnings + +import pytest + +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.table import Table + +from astroquery.exceptions import MaxResultsWarning +from astroquery.lco import LcoArchive, LcoArchiveClass + + +pytestmark = pytest.mark.remote_data + + +# Every query below is pinned to a narrow date range in the past. +# It keeps the load on the archive small and makes the responses reproducible. +# +# This window contains exactly nine public reduced frames whose footprints +# cover M101. These dates are well in the past so we should not expect +# the search results to change in the future +WINDOW = {'start': '2024-03-01', 'end': '2024-03-08'} +M101 = SkyCoord(210.802429, 54.348750, unit='deg') +EXPECTED_FRAME_IDS = {69021388, 69081552, 69081653, 69081666, 69081771, + 69081784, 69081980, 69085778, 69086104} + +# Thumbnails were only stored from 2025 onwards, so they need their own window. +THUMBNAIL_WINDOW = {'start': '2025-03-02', 'end': '2025-03-03'} + +# A single night at one site, used for the pagination test. +BIAS_WINDOW = {'start': '2024-03-02', 'end': '2024-03-03'} + + +def footprint_contains(frame, coordinates): + """Test whether a frame's footprint contains a position. + + The archive reports footprints as GeoJSON polygons with longitudes in + [-180, 180], so RA has to be wrapped before the point-in-polygon test. + """ + ring = frame['area']['coordinates'][0] + lon = coordinates.ra.deg + lon = lon - 360 if lon > 180 else lon + lat = coordinates.dec.deg + + inside = False + j = len(ring) - 1 + for i, (x, y) in enumerate(ring): + xj, yj = ring[j] + if (y > lat) != (yj > lat) and lon < (xj - x) * (lat - y) / (yj - y) + x: + inside = not inside + j = i + return inside + + +@pytest.fixture(autouse=True) +def _allow_truncation(): + # Tests here cap row_limit to keep the load on the archive small, which + # legitimately triggers the truncation warning. Tests that care about the + # warning set their own filter, which takes precedence over this one. + with warnings.catch_warnings(): + warnings.simplefilter('ignore', MaxResultsWarning) + yield + + +class TestLcoArchive: + + def test_query_region_covers_the_position(self): + result = LcoArchive.query_region(M101, reduction_level=91, + public=True, row_limit=-1, **WINDOW) + assert isinstance(result, Table) + assert set(result['id']) == EXPECTED_FRAME_IDS + + # The real invariant: every frame returned by a point query must have + # a footprint containing that point. + for frame_id in result['id']: + frame = LcoArchive.get_metadata(frame_id) + assert footprint_contains(frame, M101), \ + f"frame {frame_id} does not cover the queried position" + + # Each of these 1m frames contains M101, so its footprint center is + # within a frame's half-diagonal (about 0.3 deg) of it. + centers = SkyCoord(result['ra'], result['dec']) + assert all(centers.separation(M101) < 0.5*u.deg) + + def test_query_region_radius_includes_the_point_matches(self): + result = LcoArchive.query_region(M101, radius=0.2*u.deg, + reduction_level=91, public=True, + row_limit=-1, **WINDOW) + # A cone around the position must return at least everything whose + # footprint contains the position itself. + assert EXPECTED_FRAME_IDS <= set(result['id']) + + def test_query_region_box(self): + result = LcoArchive.query_region(M101, width=0.4*u.deg, + height=0.4*u.deg, reduction_level=91, + public=True, row_limit=-1, **WINDOW) + assert EXPECTED_FRAME_IDS <= set(result['id']) + + def test_query_object(self): + result = LcoArchive.query_object('SN2023ixf', reduction_level=91, + public=True, row_limit=-1, **WINDOW) + assert set(result['target_name']) == {'SN2023ixf'} + assert set(result['id']) == EXPECTED_FRAME_IDS + + def test_query_criteria_filters_are_applied(self): + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=5, **BIAS_WINDOW) + assert 0 < len(result) <= 5 + assert set(result['site_id']) == {'lsc'} + assert set(result['configuration_type']) == {'BIAS'} + assert set(result['reduction_level']) == {0} + + def test_processed_selects_reduced_or_raw_frames(self): + # This night at lsc holds both raw and reduced frames. + reduced = LcoArchive.query_criteria(site_id='lsc', processed=True, + public=True, row_limit=10, + **BIAS_WINDOW) + raw = LcoArchive.query_criteria(site_id='lsc', processed=False, + public=True, row_limit=10, + **BIAS_WINDOW) + assert len(reduced) > 0 and len(raw) > 0 + assert 0 not in set(reduced['reduction_level']) + assert set(raw['reduction_level']) == {0} + + def test_dates_are_within_the_requested_window(self): + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=10, **BIAS_WINDOW) + for date in result['observation_date']: + assert BIAS_WINDOW['start'] <= date[:10] <= BIAS_WINDOW['end'] + + def test_row_limit_paginates(self): + # Anonymous pages hold 100 frames, so this needs a second page. + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=150, **BIAS_WINDOW) + assert len(result) == 150 + assert len(set(result['id'])) == 150 + + def test_truncated_results_warn(self): + # This window holds far more than three frames, so the user needs to + # be told the table is not the whole answer. + with pytest.warns(MaxResultsWarning, match='truncated to 3 frames'): + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=3, **BIAS_WINDOW) + assert len(result) == 3 + + def test_no_warning_when_everything_fits(self): + # The nine M101 frames are the complete answer, so nothing is + # truncated and no warning should be raised. + with warnings.catch_warnings(): + warnings.simplefilter('error', MaxResultsWarning) + result = LcoArchive.query_object('SN2023ixf', reduction_level=91, + public=True, row_limit=-1, + **WINDOW) + assert len(result) == len(EXPECTED_FRAME_IDS) + + def test_row_limit_is_respected_below_a_page(self): + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=3, **BIAS_WINDOW) + assert len(result) == 3 + + def test_include_thumbnails(self): + result = LcoArchive.query_criteria(reduction_level=91, public=True, + include_thumbnails=True, + thumbnail_size='small', + row_limit=20, **THUMBNAIL_WINDOW) + assert 'thumbnail_url' in result.colnames + urls = [url for url in result['thumbnail_url'] if url] + assert len(urls) > 0 + assert all(url.startswith('https://') for url in urls) + + names = [n for n in result['thumbnail_filename'] if n] + assert all(n.endswith('-small_thumbnail.jpg') for n in names) + + def test_get_metadata_keeps_nested_fields(self): + frame = LcoArchive.get_metadata(sorted(EXPECTED_FRAME_IDS)[0]) + assert frame['id'] == sorted(EXPECTED_FRAME_IDS)[0] + for key in ('area', 'version_set', 'related_frames'): + assert key in frame + + def test_download_files(self, tmp_path): + result = LcoArchive.query_criteria(site_id='lsc', + configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=1, **BIAS_WINDOW) + paths = LcoArchive.download_files(result, download_dir=str(tmp_path)) + assert len(paths) == 1 + assert os.path.getsize(paths[0]) > 0 + assert paths[0].endswith(result['filename'][0]) + + def test_download_thumbnails(self, tmp_path): + result = LcoArchive.query_criteria(reduction_level=91, public=True, + include_thumbnails=True, + row_limit=20, **THUMBNAIL_WINDOW) + # Not every frame has a thumbnail, so keep the first one that does. + result = result[[bool(url) for url in result['thumbnail_url']]][:1] + assert len(result) == 1 + + paths = LcoArchive.download_thumbnails(result, + download_dir=str(tmp_path)) + assert len(paths) == 1 + assert os.path.getsize(paths[0]) > 0 + + def test_list_helpers(self): + assert 'lsc' in LcoArchive.list_sites() + assert 'BIAS' in LcoArchive.list_configuration_types() + assert len(LcoArchive.list_instruments()) > 0 + + def test_anonymous_users_see_no_proprietary_frames(self): + from astroquery.exceptions import NoResultsWarning + with pytest.warns(NoResultsWarning): + result = LcoArchive.query_criteria(public=False, row_limit=5, + **WINDOW) + assert len(result) == 0 + + +@pytest.mark.skipif(not os.environ.get('LCO_API_TOKEN'), + reason='requires an LCO API token in $LCO_API_TOKEN') +class TestLcoArchiveAuthenticated: + """Runs only when a token is supplied; no credentials live in the repo.""" + + @pytest.fixture + def lco(self): + archive = LcoArchiveClass() + archive.login(token=os.environ['LCO_API_TOKEN']) + assert archive.authenticated() + return archive + + def test_login(self, lco): + assert lco.authenticated() + + def test_larger_page_size_when_authenticated(self, lco): + assert lco._page_size == 1000 + + def test_query_still_works_when_authenticated(self, lco): + result = lco.query_object('SN2023ixf', reduction_level=91, + row_limit=5, **WINDOW) + assert len(result) > 0 + + def test_download_works_when_authenticated(self, lco, tmp_path): + # presigned URLs carry their own credentials, so make sure we are + # stripping out any Auth headers before downloading. + result = lco.query_criteria(site_id='lsc', configuration_type='BIAS', + reduction_level=0, public=True, + row_limit=1, **BIAS_WINDOW) + paths = lco.download_files(result, download_dir=str(tmp_path)) + assert len(paths) == 1 + assert os.path.getsize(paths[0]) > 0 + + def test_bad_token_is_rejected(self): + archive = LcoArchiveClass() + archive.login(token='definitely-not-a-real-token') + assert not archive.authenticated() diff --git a/docs/index.rst b/docs/index.rst index 66dc09723c..4d2ce92941 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -281,6 +281,7 @@ The following modules have been completed using a common API: ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst + lco/lco.rst linelists/jplspec/jplspec.rst magpis/magpis.rst mast/mast.rst @@ -398,6 +399,7 @@ generally return a table listing the available data first. heasarc/heasarc.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst + lco/lco.rst magpis/magpis.rst mast/mast.rst ipac/ned/ned.rst diff --git a/docs/lco/lco.rst b/docs/lco/lco.rst new file mode 100644 index 0000000000..2c93a08e97 --- /dev/null +++ b/docs/lco/lco.rst @@ -0,0 +1,328 @@ +.. _astroquery.lco: + +**************************************************** +LCO Archive Queries (`astroquery.lco`) +**************************************************** + +Getting Started +=============== + +This module searches the `science archive `_ of +`Las Cumbres Observatory `_, a global network of robotic telescopes. +The archive holds one record per **frame**: a single exposure from one instrument, +either raw or reduced. Full details on the LCO Archive can be found +`here `_. + +Anyone can query the archive without logging in, which returns every frame +whose proprietary period has expired. Logging in with an API token additionally +returns the proprietary frames belonging to your proposals; see +`Authentication`_ below. + +Every query returns a `~astropy.table.Table` of frame metadata, including a +download link for each frame. The `Downloading Data`_ section below describes +how to download the frames from a query. For more information on + + +Positional Queries +------------------ + +`~astroquery.lco.LcoArchiveClass.query_region` searches by position. Given only +a position, it returns the frames whose footprint contains that point: + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> from astropy import coordinates, units as u + >>> coord = coordinates.SkyCoord(210.802429, 54.348750, unit="deg") + >>> frames = LcoArchive.query_region(coord, reduction_level=91, + ... start="2024-03-01", end="2024-03-08") + >>> print(frames["basename", "target_name", "site_id", "exposure_time"][:5]) + basename target_name site_id exposure_time + s + ------------------------------- ----------- ------- ------------- + tfn1m001-fa11-20240304-0217-e91 SN2023ixf tfn 399.999 + tfn1m001-fa11-20240304-0216-e91 SN2023ixf tfn 400.0 + tfn1m014-fa20-20240304-0345-e91 SN2023ixf tfn 90.0 + tfn1m001-fa11-20240304-0163-e91 SN2023ixf tfn 90.0 + tfn1m001-fa11-20240304-0162-e91 SN2023ixf tfn 90.0 + +Adding ``radius`` searches a cone instead, returning frames whose footprint +*overlaps* the region rather than covers the point. A box search is available +by passing ``width`` and ``height`` together: + +.. doctest-remote-data:: + + >>> frames = LcoArchive.query_region(coord, radius=0.2*u.deg, + ... reduction_level=91, + ... start="2024-03-01", end="2024-03-08") + >>> frames = LcoArchive.query_region(coord, width=0.4*u.deg, height=0.4*u.deg, + ... reduction_level=91, + ... start="2024-03-01", end="2024-03-08") + +A cone is sent to the archive as a 32-vertex polygon inscribed in the circle, +so the region searched is accurate to 0.5% of the radius specified. + + +Object Name Queries +------------------- + +`~astroquery.lco.LcoArchiveClass.query_object` searches on the target name the +observer submitted with the observation request. It does not resolve the name +or search on position, so use ``query_region`` if you want everything covering +a target regardless of what it was called. The default parameter of ``exact=True`` +performs a case-sensitive exact match. Set it to ``False`` to perform a +case-insensitive contains query: + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> frames = LcoArchive.query_object("SN2023ixf", exact=True, reduction_level=91, + ... start="2024-03-01", end="2024-03-08") + >>> print(frames["basename", "instrument_id", "reduction_level"][:5]) + basename instrument_id reduction_level + ------------------------------- ------------- --------------- + tfn1m001-fa11-20240304-0217-e91 fa11 91 + tfn1m001-fa11-20240304-0216-e91 fa11 91 + tfn1m014-fa20-20240304-0345-e91 fa20 91 + tfn1m001-fa11-20240304-0163-e91 fa11 91 + tfn1m001-fa11-20240304-0162-e91 fa11 91 + + +Criteria Queries +---------------- + +`~astroquery.lco.LcoArchiveClass.query_criteria` searches on any combination of +frame criteria. A list of the criteria is available at +`~astroquery.lco.LcoArchiveClass.list_criteria`, while a more in depth +explanation of the available criteria can be found on +`LCO's Developer Docs `_. +Queries should be as constrained as possible to provide results quickly and efficiently. +Some helpful parameters to constrain a query are date ranges through ``start`` and ``end``, +or filters on a specific ``proposal_id`` or ``request_id`` + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> LcoArchive.list_criteria() # doctest: +ELLIPSIS + ['basename', 'basename_exact', 'configuration_type', 'covers', ...] + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> frames = LcoArchive.query_criteria(site_id="lsc", + ... configuration_type="BIAS", + ... reduction_level=0, instrument_id="kb98", + ... start="2024-03-02", end="2024-03-03") + >>> print(frames["basename", "site_id", "telescope_id", "configuration_type"][:3]) + basename site_id telescope_id configuration_type + ------------------------------- ------- ------------ ------------------ + lsc0m409-kb98-20240302-0017-b00 lsc 0m4a BIAS + lsc0m409-kb98-20240302-0016-b00 lsc 0m4a BIAS + lsc0m409-kb98-20240302-0015-b00 lsc 0m4a BIAS + +The same criteria can be passed to ``query_region`` and ``query_object``, which +narrow a positional or name search rather than replacing it. + +``start`` and ``end`` bound the observation date and accept a string, an +`~astropy.time.Time`, or a `~datetime.datetime`. The LCO Archive is +very large, so filtering with a date range is **always** a good idea. + +The main ``reduction_level`` values are ``0`` for raw data and ``91`` for +most reduced data. There are several other more obscure reduction levels +defined in the +`Archive Documentation `_. + +To select on processing state rather than one specific level, pass +``processed=True`` for frames at any reduced level, or ``processed=False`` for +raw frames only. Leaving it out returns both: + +.. doctest-remote-data:: + + >>> frames = LcoArchive.query_criteria(site_id="lsc", processed=True, + ... start="2024-03-02", + ... end="2024-03-03") # doctest: +IGNORE_WARNINGS + >>> set(frames["reduction_level"].tolist()) + {91} + +Two criteria accept several values at once, +``include_configuration_type`` and ``exclude_configuration_type``: + +.. doctest-remote-data:: + + >>> frames = LcoArchive.query_criteria(site_id="lsc", instrument_id="kb98", + ... exclude_configuration_type=["BIAS", "DARK"], + ... start="2024-03-02", + ... end="2024-03-03") # doctest: +IGNORE_WARNINGS + +All other criteria expect a single value only. + + +Discovering Criteria Values +------------------------------- + +The ``list_*`` methods list which sites, telescopes, +instruments, filters, configuration types or proposals are available to filter: + +.. doctest-remote-data:: + + >>> LcoArchive.list_sites() + ['bpl', 'coj', 'cpt', 'elp', 'lsc', 'mfg', 'ogg', 'sor', 'sqa', 'tfn', 'tlv', 'tst'] + >>> LcoArchive.list_configuration_types() # doctest: +IGNORE_OUTPUT + ['ARC', 'BIAS', 'CATALOG', 'DARK', 'DOMEFLAT', ...] + +An unrecognised criterion is rejected before any request is made, and the error +lists the ones that are accepted: + +.. doctest-remote-data:: + + >>> LcoArchive.query_criteria(not_a_criteria="fa16") + Traceback (most recent call last): + ... + ValueError: 'not_a_criteria' is not a supported LCO archive filter. Supported filters are: basename, ... + + +How Many Rows You Get +--------------------- + +Queries return at most ``row_limit`` frames, 100 by default. When the archive +holds more than were returned, a ``MaxResultsWarning`` is printed. + +.. doctest-remote-data:: + + >>> frames = LcoArchive.query_criteria(site_id="lsc", row_limit=3, + ... start="2024-03-02", end="2024-03-03") # doctest: +SHOW_WARNINGS + MaxResultsWarning: Results truncated to 3 frames. Pass row_limit=-1 to retrieve every matching frame. + +Pass ``row_limit=-1`` to retrieve every match, paging through the archive as +needed. If the rows retrieved from an unbounded query exceed 10,000, a warning +will be printed. + +.. doctest-skip:: + + >>> LcoArchive.ROW_LIMIT = 500 + + +Downloading Data +---------------- + +`~astroquery.lco.LcoArchiveClass.download_files` downloads the data files for the +frames in a result table: + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> frames = LcoArchive.query_criteria( + ... basename_exact="lsc0m409-kb98-20240302-0017-b00") + >>> LcoArchive.download_files(frames, download_dir=".") # doctest: +IGNORE_OUTPUT + ['./lsc0m409-kb98-20240302-0017-b00.fits.fz'] + +Files already present are skipped unless you pass ``overwrite=True``. You can +also pass frame ids instead of a table, in which case each one is looked up so +that its download link is fresh: + +.. doctest-skip:: + + >>> LcoArchive.download_files([69021388, 69081552], download_dir=".") + +The download links the archive returns are presigned and expire in 48 hours. +Query again to refresh it, or use `~astroquery.lco.LcoArchiveClass.get_metadata`, +which returns the full record for a single frame including the fields the result +table leaves out (``area``, ``version_set`` and ``related_frames``): + +.. doctest-remote-data:: + + >>> frame = LcoArchive.get_metadata(69008745) + >>> frame["basename"] + 'lsc0m409-kb98-20240302-0017-b00' + >>> sorted(frame)[:4] + ['BLKUID', 'DATE_OBS', 'DAY_OBS', 'EXPTIME'] + + +Thumbnails +---------- + +Passing ``include_thumbnails=True`` asks the archive for JPEG preview links +alongside the metadata, which adds ``thumbnail_url`` and ``thumbnail_filename`` +columns. ``thumbnail_size`` chooses between ``'small'`` and ``'large'``: + +.. doctest-remote-data:: + + >>> from astroquery.lco import LcoArchive + >>> frames = LcoArchive.query_criteria(site_id="cpt", instrument_id="sq39", + ... reduction_level=91, + ... include_thumbnails=True, + ... thumbnail_size="large", + ... start="2025-03-02T18:00", + ... end="2025-03-02T19:00") + >>> print(frames["thumbnail_filename"][0]) + cpt0m439-sq39-20250302-0135-e91-large_thumbnail.jpg + >>> LcoArchive.download_thumbnails(frames[:1], download_dir=".") # doctest: +IGNORE_OUTPUT + +Not every frame has a thumbnail — the archive only began storing them in +2025 — and frames without one are skipped. + + +Authentication +============== + +Queries work without logging in and return the public archive. To reach the +proprietary frames on your own proposals, log in with the API token from your +`LCO profile page `_: + +.. doctest-skip:: + + >>> from astroquery.lco import LcoArchive + >>> LcoArchive.login(token="your-api-token") + >>> LcoArchive.authenticated() + True + +You can log in with your observing portal username instead, which exchanges +your credentials for a token. The password is read from your system keyring if +it is stored there, and prompted for otherwise; pass ``store_password=True`` to +save it in the system keyring for later sessions: + +.. doctest-skip:: + + >>> LcoArchive.login(username="your-username") + +Every query method behaves identically once you are logged in — the difference +is only in what the archive returns. Authenticated sessions also page through +results 1000 frames at a time rather than 100. + + +Configuration +============= + +The module reads four configuration items, which can be changed at runtime through +``astroquery.lco.conf``: + +* ``archive_url``: The archive to query. Defaults to + https://archive-api.lco.global. + +* ``token_url``: Where a username and password are exchanged for an API token. + This is the LCO observing portal rather than the archive, and defaults to + https://observe.lco.global/api/api-token-auth/. + +* ``timeout``: Seconds to wait for the archive to respond. Defaults to ``30``. + +* ``row_limit``: The default maximum number of frames a query returns. Defaults + to ``100``; set it to ``-1`` to retrieve every match for a query. + +.. doctest-skip:: + + >>> from astroquery.lco import conf + >>> conf.timeout = 60 + + +.. testcleanup:: + + >>> from astroquery.utils import cleanup_saved_downloads + >>> cleanup_saved_downloads(['lsc0m409-kb98-20240302-0017-b00.fits.fz', + ... 'cpt0m439-sq39-20250302-0135-e91-large_thumbnail.jpg']) + + +Reference/API +============= + +.. automodapi:: astroquery.lco + :no-inheritance-diagram: