diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index 2051d24b..2dcdb914 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -27,7 +27,7 @@ on: - "python/**" - "core-spec/**" - "ontology/**" - - "examples/flights.yaml" + - "examples/flights*.yaml" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" pull_request: @@ -37,7 +37,7 @@ on: - "python/**" - "core-spec/**" - "ontology/**" - - "examples/flights.yaml" + - "examples/flights*.yaml" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" @@ -77,3 +77,10 @@ jobs: - name: Validate canonical example run: uv run validation/validate.py examples/tpcds_semantic_model.yaml + + - name: Validate flights examples + run: | + uv run validation/validate.py examples/flights.yaml --schema ontology/ontology.json + uv run validation/validate.py examples/flights.ontology.yaml --schema ontology/ontology.json + uv run validation/validate.py examples/flights.semantic_model.yaml + uv run validation/validate.py examples/flights.mapping.yaml --schema ontology/mapping.json diff --git a/examples/flights.mapping.yaml b/examples/flights.mapping.yaml new file mode 100644 index 00000000..ac3dbc9b --- /dev/null +++ b/examples/flights.mapping.yaml @@ -0,0 +1,314 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: flights_mapping +description: Maps the Flights ontology onto the Flights semantic model +ontology_ref: + name: Flights + iri: ./flights.ontology.yaml +semantic_model_ref: + name: Flights semantic model + iri: ./flights.semantic_model.yaml +concept_mappings: +- concept: Runway + object_mappings: + - referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + children: + - object_mapping: + concept: RunwayLength + expression: RUNWAY.length + relationship: length + - object_mapping: + concept: RunwayGeometry + expression: RUNWAY.shape + relationship: geometry +- concept: Manufacturer + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer +- concept: Model + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer +- concept: Aircraft + object_mappings: + - referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + link_mappings: + - object_mapping: + referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + children: + - object_mapping: + concept: SerialNr + expression: AIRCRAFT.serial_nr + relationship: serial_number + - object_mapping: + concept: String + expression: AIRCRAFT.name + relationship: name + - object_mapping: + concept: Integer + expression: AIRCRAFT.nr_seats + relationship: number_of_seats + - object_mapping: + concept: Model + referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + relationship: model + - object_mapping: + concept: Year + expression: AIRCRAFT.year + relationship: year_manufactured + - object_mapping: + concept: Capacity + expression: AIRCRAFT.capacity + relationship: capacity + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: AIRCRAFT.carrier_code + relationship: carrier +- concept: State + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.state_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + children: + - object_mapping: + concept: StateName + expression: AIRPORT.state_nm + relationship: name +- concept: City + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code +- concept: Market + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.market_nm +- concept: Airport + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.code + children: + - object_mapping: + concept: City + referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + relationship: city + - object_mapping: + concept: Market + referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + relationship: serves + - object_mapping: + concept: DegreesLongitude + expression: AIRPORT.longitude + relationship: longitude + - object_mapping: + concept: AirportName + expression: AIRPORT.name + relationship: name + - object_mapping: + concept: DegreesLatitude + expression: AIRPORT.latitude + relationship: latitude +- concept: Flight + object_mappings: + - referent_mappings: + - relationship: id + expression: FLIGHT.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.diverted == TRUE ) + relationship: diverted + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.cancelled == TRUE ) + relationship: canceled + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id + children: + - object_mapping: + concept: Delay + expression: FLIGHT.arr_delay + relationship: arrival_delay + - object_mapping: + concept: Delay + expression: FLIGHT.dep_delay + relationship: departure_delay + - object_mapping: + concept: FlightNr + expression: FLIGHT.nr + relationship: number + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_departure + relationship: scheduled_departure + - object_mapping: + concept: CancelationReason + referent_mappings: + - relationship: code + expression: FLIGHT.cancel_code + relationship: canceled_due_to + - object_mapping: + concept: Distance + expression: FLIGHT.distance + relationship: distance + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_arrival + relationship: scheduled_arrival + - object_mapping: + concept: Date + expression: FLIGHT.date + relationship: date + - object_mapping: + concept: DateTime + expression: FLIGHT.departure + relationship: departs_at + - object_mapping: + concept: DateTime + expression: FLIGHT.arrival + relationship: arrives_at + - object_mapping: + concept: Route + referent_mappings: + - relationship: id + expression: FLIGHT.route_id + relationship: route + - object_mapping: + concept: Aircraft + referent_mappings: + - relationship: tailnum + expression: FLIGHT.tail_nr + relationship: aircraft + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: FLIGHT.carrier_code + relationship: operated_by +- concept: Carrier + object_mappings: + - referent_mappings: + - relationship: code + expression: CARRIER.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: CARRIER.code + children: + - object_mapping: + concept: CarrierName + expression: CARRIER.name + relationship: name +- concept: Route + object_mappings: + - referent_mappings: + - relationship: id + expression: ROUTE.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: ROUTE.id + children: + - object_mapping: + concept: Distance + expression: ROUTE.distance + relationship: distance + - object_mapping: + concept: DistanceGroup + expression: ROUTE.dist_grp + relationship: lies_in + - object_mapping: + concept: String + expression: ROUTE.name + relationship: route_name + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.dest_airport_code + relationship: destination + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.orig_airport_code + relationship: departure diff --git a/examples/flights.ontology.yaml b/examples/flights.ontology.yaml new file mode 100644 index 00000000..2fd62522 --- /dev/null +++ b/examples/flights.ontology.yaml @@ -0,0 +1,539 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: Flights +description: Ontology of flights into and out of airports. +requires: + - COUNT[Airport] > 0 # there must be at least one Airport + - COUNT[Carrier] > 0 # there must be at least one Carrier +ontology: +- concept: NrFeet + description: "Unit of measure for distance in feet" + type: ValueType + extends: [ Decimal ] +- concept: NrPounds + description: "Unit of measure for weight in pounds" + type: ValueType + extends: [ Integer ] +- concept: NrMiles + description: "Unit of measure for distance in miles" + type: ValueType + extends: [ Decimal ] +- concept: NrMinutes + description: "Unit of measure for time in minutes" + type: ValueType + extends: [ Decimal ] +- concept: CancelationCode + description: "The single character code that identifies the reason a flight is canceled" + type: ValueType + extends: [ String ] + requires: [ CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' ] +- concept: Capacity + description: "The capacity of an aircraft, measured in pounds." + type: ValueType + extends: [ NrPounds ] +- concept: DegreesLatitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] +- concept: DegreesLongitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] +- concept: Polygon + description: "A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates." + type: ValueType + extends: [ String ] +- concept: CityName + type: ValueType + extends: [ String ] +- concept: Delay + type: ValueType + extends: [ NrMinutes ] +- concept: Distance + type: ValueType + extends: [ NrMiles ] +- concept: DistanceGroup + description: "A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest." + type: ValueType + extends: [ Integer ] + requires: [ 1 <= DistanceGroup, DistanceGroup <= 10 ] +- concept: RunwayGeometry + description: "A polygon that models the shape of a runway." + type: ValueType + extends: [ Polygon ] +- concept: RunwayLength + description: "The unit for measuring the lengths of runways in American airports." + type: ValueType + extends: [ NrFeet ] +- concept: RunwayDesignator + description: "Used to distinguish runways within a given airport." + type: ValueType + extends: [ String ] +- concept: SerialNr + type: ValueType + extends: [ String ] +- concept: StateCode + type: ValueType + extends: [ String ] +- concept: StateName + type: ValueType + extends: [ String ] +- concept: TailNr + type: ValueType + extends: [ String ] +- concept: Year + type: ValueType + extends: [ String ] +- concept: CancelationReason + description: "A curated set of reasons that explain why a flight is canceled" + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: CancelationCode + verbalizes: [ '{CancelationReason} is identified by {CancelationCode}'] + multiplicity: OneToOne + - name: description + roles: + - concept: String + verbalizes: [ '{CancelationReason} has description- {String}' ] # The hyphen after "description" is significant when verbalizing constraints + multiplicity: ManyToOne # Each CancelationReason has at most one description String +- concept: State + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: StateCode + verbalizes: + - '{State} is identified by {StateCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: StateName + verbalizes: + - '{State} has {StateName}' + multiplicity: ManyToOne +- concept: City + type: EntityType + identify_by: [ name, state ] + relationships: + - name: name + roles: + - concept: CityName + verbalizes: + - '{City} has {CityName}' + multiplicity: ManyToOne + - name: state + roles: + - concept: State + verbalizes: + - '{City} is located in {State}' + multiplicity: ManyToOne +- concept: Runway + type: EntityType + identify_by: [ designator, airport ] + relationships: + - name: airport + roles: + - concept: Airport + verbalizes: + - '{Runway} belongs to {Airport}' + multiplicity: ManyToOne + - name: designator + roles: + - concept: RunwayDesignator + verbalizes: + - '{Runway} uses {RunwayDesignator}' + multiplicity: ManyToOne + - name: length + roles: + - concept: RunwayLength + verbalizes: + - '{Runway} has {RunwayLength}' + multiplicity: ManyToOne + - name: geometry + roles: + - concept: RunwayGeometry + verbalizes: + - '{Runway} has {RunwayGeometry}' + multiplicity: ManyToOne +- concept: ManufacturerName + type: ValueType + extends: [ String ] +- concept: Manufacturer + type: EntityType + identify_by: [ name ] + relationships: + - name: name + roles: + - concept: ManufacturerName + verbalizes: + - '{Manufacturer} is identified by {ManufacturerName}' + multiplicity: OneToOne +- concept: ModelName + type: ValueType + extends: [ String ] +- concept: Model + type: EntityType + identify_by: [ name, manufacturer ] + relationships: + - name: name + roles: + - concept: ModelName + verbalizes: + - '{Model} has {ModelName}' + multiplicity: ManyToOne + - name: manufacturer + roles: + - concept: Manufacturer + verbalizes: + - '{Model} is manufactured by {Manufacturer}' + multiplicity: ManyToOne +- concept: Aircraft + type: EntityType + identify_by: [ tailnum ] + relationships: + - name: serial_number + roles: + - concept: SerialNr + verbalizes: + - '{Aircraft} has {SerialNr}' + multiplicity: ManyToOne + - name: name + roles: + - concept: String + verbalizes: + - '{Aircraft} has name {String}' + multiplicity: ManyToOne + - name: number_of_seats + roles: + - concept: Integer + verbalizes: + - '{Aircraft} has {Integer} seats' + multiplicity: ManyToOne + - name: tailnum + roles: + - concept: TailNr + verbalizes: + - '{Aircraft} is identified by {TailNr}' + multiplicity: OneToOne + - name: model + roles: + - concept: Model + verbalizes: + - '{Aircraft} has {Model}' + multiplicity: ManyToOne + - name: year_manufactured + roles: + - concept: Year + verbalizes: + - '{Aircraft} was manufactured in {Year}' + multiplicity: ManyToOne + - name: capacity + roles: + - concept: Capacity + verbalizes: + - '{Aircraft} has {Capacity}' + multiplicity: ManyToOne + - name: carrier + roles: + - concept: Carrier + verbalizes: + - '{Aircraft} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: AirportName + type: ValueType + extends: [ String ] +- concept: AirportCode + description: "The three-letter IATA code for the airport." + type: ValueType + extends: [ String ] +- concept: AirportId + description: "Five digit number used as an alternate identifier for airports." + type: ValueType + extends: [ String ] +- concept: Airport + type: EntityType + identify_by: [ code ] + requires: [ Airport.latitude, Airport.longitude ] + relationships: + - name: city + roles: + - concept: City + verbalizes: + - '{Airport} is located in {City}' + multiplicity: ManyToOne + - name: serves + roles: + - concept: Market + verbalizes: + - '{Airport} serves {Market}' + multiplicity: ManyToOne + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- departure {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Airport has at most one average departure Delay + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Airport == Flight.route.departure GROUP BY Airport]' ] + - name: longitude + roles: + - concept: DegreesLongitude + verbalizes: + - '{Airport} centers at {DegreesLongitude}' + multiplicity: ManyToOne + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- arrival {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Airport has at most one average arrival Delay + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport]' ] + - name: name + roles: + - concept: AirportName + verbalizes: + - '{Airport} has {AirportName}' + multiplicity: ManyToOne + - name: code + roles: + - concept: AirportCode + verbalizes: + - '{Airport} has {AirportCode}' + multiplicity: OneToOne + - name: latitude + roles: + - concept: DegreesLatitude + verbalizes: + - '{Airport} centers at {DegreesLatitude}' + multiplicity: ManyToOne +- concept: MarketName + type: ValueType + extends: [ String ] +- concept: Market + type: EntityType + identify_by: [name] + relationships: + - name: name + roles: [ { concept: MarketName }] + verbalizes: [ '{Market} is identified by {MarketName}' ] + multiplicity: OneToOne +- concept: FlightNr + description: "The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234')." + type: ValueType + extends: [ String ] +- concept: FlightId + description: "A unique identifier for an instance of a flight" + type: ValueType + extends: [ String ] +- concept: Flight + type: EntityType + identify_by: [ id ] + relationships: + - name: departure_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has departure- {Delay}' # The hyphen after "departure" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight has at most one departure Delay + - name: number + roles: + - concept: FlightNr + verbalizes: + - '{Flight} has {FlightNr}' + multiplicity: ManyToOne + - name: scheduled_departure + roles: [ { concept: DateTime } ] + verbalizes: + - '{Flight} is scheduled to depart at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.scheduled_departure < Flight.scheduled_arrival ] + - name: arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has arrival- {Delay}' # The hyphen after "arrival" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight has at most one arrival Delay + - name: canceled + verbalizes: + - '{Flight} was canceled' + - name: canceled_due_to + roles: + - concept: CancelationReason + verbalizes: + - '{Flight} was canceled due to {CancelationReason}' + multiplicity: ManyToOne + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Flight} spans actual- {Distance}' # The hyphen after "actual" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight spans at most one actual Distance + - name: id + roles: + - concept: FlightId + verbalizes: + - '{Flight} is identified by {FlightId}' + multiplicity: OneToOne + - name: scheduled_arrival + roles: + - concept: DateTime + verbalizes: + - '{Flight} is scheduled to arrive at {DateTime}' + multiplicity: ManyToOne + - name: registers_longitude_series + roles: + - concept: DateTime + - concept: DegreesLongitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLongitude}' + multiplicity: ManyToOne + - name: registers_latitude_series + roles: + - concept: DateTime + - concept: DegreesLatitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLatitude}' + multiplicity: ManyToOne + - name: date + roles: + - concept: Date + verbalizes: + - '{Flight} is scheduled to depart on {Date}' + multiplicity: ManyToOne + - name: departs_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} departs at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.departs_at < Flight.arrives_at ] + - name: diverted + verbalizes: + - '{Flight} was diverted' + - name: arrives_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} arrives at {DateTime}' + multiplicity: ManyToOne + - name: route + roles: + - concept: Route + verbalizes: + - '{Flight} traverses {Route}' + multiplicity: ManyToOne + - name: aircraft + roles: + - concept: Aircraft + verbalizes: + - '{Flight} uses {Aircraft}' + multiplicity: ManyToOne + requires: [ Flight.operated_by(Aircraft.carrier) ] + - name: operated_by + roles: + - concept: Carrier + verbalizes: + - '{Flight} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: CarrierCode + description: "The two-letter IATA code for the airline carrier." + type: ValueType + extends: [ String ] +- concept: CarrierName + type: ValueType + extends: [ String ] +- concept: Carrier + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: CarrierCode + verbalizes: + - '{Carrier} uses {CarrierCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: CarrierName + verbalizes: + - '{Carrier} has {CarrierName}' + multiplicity: ManyToOne +- concept: RouteId + type: ValueType + description: "A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA')" + extends: [ String ] +- concept: Route + type: EntityType + identify_by: [ id ] + relationships: + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- departure {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route has at most one average departure Delay + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- arrival {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route has at most one average arrival Delay + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Route} spans {Distance}' + multiplicity: ManyToOne + - name: lies_in + roles: + - concept: DistanceGroup + verbalizes: + - '{Route} has {DistanceGroup}' + multiplicity: ManyToOne + - name: id + roles: + - concept: RouteId + verbalizes: + - '{Route} is identified by {RouteId}' + multiplicity: OneToOne + - name: route_name + roles: + - concept: String + verbalizes: + - '{Route} has name- {String}' # The hyphen after "name" has significance when verbalizing constraints + multiplicity: ManyToOne + - name: destination + roles: + - concept: Airport + verbalizes: + - '{Route} connects to destination- {Airport}' # The hyphen after "destination" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route connects to at most one destination Airport + requires: [ 'NOT Route.departure(Airport)' ] + - name: departure + roles: + - concept: Airport + verbalizes: + - '{Route} connects to departure- {Airport}' # The hyphen after "departure" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route connects to at most one departure Airport + requires: [ 'NOT Route.destination(Airport)' ] diff --git a/examples/flights.semantic_model.yaml b/examples/flights.semantic_model.yaml new file mode 100644 index 00000000..eeadf0d1 --- /dev/null +++ b/examples/flights.semantic_model.yaml @@ -0,0 +1,299 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: Flights semantic model +datasets: +- name: RUNWAY + source: DATABASE.SCHEMA.RUNWAYS + fields: + - name: airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: airport_code + - name: length + expression: + dialects: + - dialect: ANSI_SQL + expression: length + - name: shape + expression: + dialects: + - dialect: ANSI_SQL + expression: shape + - name: designator + expression: + dialects: + - dialect: ANSI_SQL + expression: designator +- name: AIRCRAFT + source: DATABASE.SCHEMA.AIRCRAFT + description: "An airplane, helicopter, or other machine capable of flight" + fields: + - name: serial_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: serial_nr + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: nr_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: nr_seats + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: manufacturer + expression: + dialects: + - dialect: ANSI_SQL + expression: manufacturer + - name: model + expression: + dialects: + - dialect: ANSI_SQL + expression: model + - name: year + expression: + dialects: + - dialect: ANSI_SQL + expression: year + - name: capacity + expression: + dialects: + - dialect: ANSI_SQL + expression: capacity +- name: AIRPORT + source: DATABASE.SCHEMA.AIRPORTS + description: "An airport that is identified by an IATA code." + fields: + - name: state_code + expression: + dialects: + - dialect: ANSI_SQL + expression: state_code + - name: market_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: market_nm + - name: longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: longitude + - name: opened + expression: + dialects: + - dialect: ANSI_SQL + expression: opened + - name: state_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: state_nm + - name: city_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: city_nm + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: latitude +- name: FLIGHT + source: DATABASE.SCHEMA.FLIGHTS + description: "A commercial passenger flight." + fields: + - name: dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: dep_delay + - name: air_time + expression: + dialects: + - dialect: ANSI_SQL + expression: air_time + - name: nr + expression: + dialects: + - dialect: ANSI_SQL + expression: nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: duration + expression: + dialects: + - dialect: ANSI_SQL + expression: duration + - name: scheduled_departure + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_departure + - name: arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: arr_delay + - name: cancelled + expression: + dialects: + - dialect: ANSI_SQL + expression: cancelled + - name: cancel_code + expression: + dialects: + - dialect: ANSI_SQL + expression: cancel_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: scheduled_arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_arrival + - name: wheels_on + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_on + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: date + - name: wheels_off + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_off + - name: scheduled_duration + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_duration + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: departure + expression: + dialects: + - dialect: ANSI_SQL + expression: departure + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: route_id + - name: diverted + expression: + dialects: + - dialect: ANSI_SQL + expression: diverted + - name: arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: arrival +- name: CARRIER + source: DATABASE.SCHEMA.CARRIERS + description: "An airline, such as Delta, United, or American." + fields: + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name +- name: ROUTE + source: DATABASE.SCHEMA.ROUTES + description: "Represents the existence of one or more flights between a pair of departure + arrival airports." + fields: + - name: orig_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: orig_airport_code + - name: dest_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: dest_airport_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: dist_grp + expression: + dialects: + - dialect: ANSI_SQL + expression: dist_grp + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name diff --git a/ontology/mapping.json b/ontology/mapping.json new file mode 100644 index 00000000..62e2ac45 --- /dev/null +++ b/ontology/mapping.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/ontology/mapping.json", + "title": "Apache Ossie Mapping Specification", + "description": "JSON Schema for validating a standalone Apache Ossie mapping document: the constructs of one semantic model mapped onto one ontology, as its own document rather than embedded inside the ontology's ontology_mappings.", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.2.0.dev0", + "description": "Mapping specification version" + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for this mapping" + }, + "description": { + "type": "string", + "description": "Human-readable description of this mapping" + }, + "ontology_ref": { + "$ref": "#/$defs/DocumentReference", + "description": "Reference to the ontology document this mapping targets" + }, + "semantic_model_ref": { + "$ref": "#/$defs/DocumentReference", + "description": "Reference to the semantic model document this mapping draws from" + }, + "concept_mappings": { + "type": "array", + "items": { + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/ontology/ontology.json#/$defs/ConceptMapping" + }, + "description": "Maps logical model constructs to some concept and its relationships in the referenced ontology" + }, + "custom_extensions": { + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json#/$defs/SemanticModel/properties/custom_extensions" + } + }, + "required": ["version", "name", "ontology_ref", "semantic_model_ref", "concept_mappings"], + "additionalProperties": false, + "$defs": { + "DocumentReference": { + "type": "object", + "description": "Reference to another Ossie document by its logical name, with an iri saying where to resolve it from", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Must equal the referenced document's own 'name'; carries this reference's checkable identity regardless of location" + }, + "iri": { + "type": "string", + "minLength": 1, + "description": "Where to resolve the referenced document from: a relative reference resolved against the mapping document's own location or base IRI, or an absolute IRI" + } + }, + "required": ["name"], + "additionalProperties": false + } + } +} diff --git a/ontology/ontology.json b/ontology/ontology.json index 68e16b83..d6508d4e 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -38,7 +38,8 @@ }, "ontology_mappings": { "type": "array", - "description": "Collection of ontology maps from logical models", + "description": "Deprecated. Embedded ontology maps, accepted only so existing documents continue to validate; write mappings as standalone documents validated against ontology/mapping.json instead.", + "deprecated": true, "items": { "$ref": "#/$defs/OntologyMap" } diff --git a/ontology/ontology.md b/ontology/ontology.md index 3b2ab6a6..3f558c5a 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -89,6 +89,7 @@ hierarchically, grouping each relationship under the concept that plays its firs | `ai_context` | string/object | No | Additional context for AI tools | | `ontology` | list | Yes | Concepts and relationships they group that form this ontology | | `prefixes` | object | No | Namespace prefixes used to abbreviate [IRIs](#global-identifiers) | +| `ontology_mappings` | list | No | Deprecated; accepted only so existing documents continue to validate. Write mappings as [mapping documents](#mapping-documents) instead | Each component of an ontology declares a concept and lists the relationships where that concept plays the first role. The concept's name is the value of the `concept` field, and @@ -442,21 +443,48 @@ Ontology mappings declare how to map the values of fields at the logical level t in the ontology. Just as ontologies are partitioned by concept, ontology maps partition into concept mappings that group by some concept. -Each mapping's `semantic_model` is a complete -[core document](../core-spec/spec.md#semantic-model), with its own `version`, -`name`, and at least one dataset. For example: +### Mapping documents -```yaml -ontology_mappings: - - name: sales_mapping - semantic_model: - version: 0.2.0.dev0 - name: sales_analytics - datasets: - - name: orders - source: sales.public.orders - concept_mappings: [] -``` +A mapping is written as its own document, validated against `ontology/mapping.json`. It maps the +constructs of one semantic model onto one ontology, and references both rather than embedding +either: + +| Field | Type | Required | Description | +|---------------|---------|-----|-------| +| `version` | string | Yes | Mapping specification version | +| `name` | string | Yes | Unique identifier for this mapping | +| `description` | string | No | Human-readable description of this mapping | +| `ontology_ref` | object | Yes | Reference to the ontology document this mapping targets (see below) | +| `semantic_model_ref` | object | Yes | Reference to the semantic model document this mapping draws from (see below) | +| `concept_mappings` | list | Yes | Maps logical model constructs to concepts and relationships in the referenced ontology | +| `custom_extensions` | list | No | Vendor-specific attributes for extensibility, matching the core specification's mechanism | + +Both `ontology_ref` and `semantic_model_ref` are references with the following schema, mirroring +`DocumentReference` in `mapping.json`: + +| Field | Type | Required | Description | +|---------------|---------|-----|-------| +| `name` | string | Yes | Must equal the referenced document's own `name`; this is the reference's identity | +| `iri` | string | No | Where to resolve the referenced document from. A relative reference such as `./flights.ontology.yaml` is resolved against the mapping document's own location or base IRI; an absolute IRI identifies the location directly. May be omitted where a catalog resolves documents by `name` | + +A mapping document references exactly one ontology and exactly one semantic model. When more than +one semantic model maps to an ontology, each mapping is its own document. + +A mapping document is recognized by the combination of `concept_mappings`, `ontology_ref`, and +`semantic_model_ref`. These reference keys do not overlap with the root keys of ontology or +semantic model documents; there is no separate field declaring a document's kind. + +See `examples/flights.ontology.yaml`, `examples/flights.semantic_model.yaml`, and +`examples/flights.mapping.yaml` for a complete example. + +**Deprecated:** mappings were originally embedded in the ontology document's `ontology_mappings` +list (`OntologyMap` in `ontology.json`), each carrying a complete +[core document](../core-spec/spec.md#semantic-model), with its own `version`, `name`, and at +least one dataset, as its `semantic_model`. That list is still accepted so existing documents +continue to validate, but it is deprecated and will be removed in a future version. New mappings +should be written as mapping documents. Parser and converter support for standalone mapping +documents is not yet available and will be added separately; until then, use embedded mappings +when a tool requires them. ### Concept mappings @@ -665,6 +693,8 @@ though `Store` plays a role in three of the relationships. - Core ontology structure: Concepts, relationships, and business rules (requires and derived_by) - Schema mappings from one or more logical models into an ontology - Optional IRIs on concepts and relationships, with namespace prefixes declared at the top level + - Mapping documents (`ontology/mapping.json`) that reference one ontology and one semantic + model; embedded `ontology_mappings` is deprecated --- diff --git a/validation/tests/test_ontology.py b/validation/tests/test_ontology.py index 00aee8cc..00df3111 100644 --- a/validation/tests/test_ontology.py +++ b/validation/tests/test_ontology.py @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. -"""Validate ontology mappings against the local core document schema.""" +"""Validate ontology mappings and mapping documents against the local schemas.""" import json +import urllib.request +from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path import pytest @@ -127,3 +129,215 @@ def test_flights_embedded_models_validate_as_core_documents(ontology_validator): # Validate the embedded core documents without expanding this test to # unrelated ontology concept and relationship constraints. ontology_validator.validate(_ontology(mapping["semantic_model"])) + + +# --- Standalone mapping documents ------------------------------------------- + +_VALIDATE_PATH = Path(__file__).parents[1] / "validate.py" +_SPEC = spec_from_file_location("ossie_validate_for_ontology", _VALIDATE_PATH) +assert _SPEC is not None and _SPEC.loader is not None +_VALIDATE = module_from_spec(_SPEC) +_SPEC.loader.exec_module(_VALIDATE) + +_ONTOLOGY_RAW_URL = "https://raw.githubusercontent.com/apache/ossie/main/ontology/ontology.json" + + +@pytest.fixture +def offline(monkeypatch): + def reject_request(*args, **kwargs): + pytest.fail("Schema validation must not make HTTP requests") + + monkeypatch.setattr(urllib.request, "urlopen", reject_request) + monkeypatch.setattr("jsonschema.validators.urlopen", reject_request, raising=False) + + +@pytest.fixture +def mapping_schema() -> dict: + schema = json.loads((_REPO / "ontology/mapping.json").read_text()) + Draft202012Validator.check_schema(schema) + return schema + + +def _schema(relative_path: str) -> dict: + return json.loads((_REPO / relative_path).read_text()) + + +def _mapping_document() -> dict: + return { + "version": "0.2.0.dev0", + "name": "sales_mapping", + "ontology_ref": {"name": "sales", "iri": "./sales.ontology.yaml"}, + "semantic_model_ref": { + "name": "sales_model", + "iri": "./sales.semantic_model.yaml", + }, + "concept_mappings": [ + { + "concept": "Order", + "object_mappings": [{"expression": "orders.order_id"}], + } + ], + } + + +@pytest.mark.parametrize( + "uri", ["https://github.com/apache/ossie/ontology/ontology.json", _ONTOLOGY_RAW_URL] +) +def test_ontology_schema_references_resolve_offline(offline, uri): + concept_mapping = _mapping_document()["concept_mappings"][0] + + assert _VALIDATE.validate_schema( + concept_mapping, {"$ref": uri + "#/$defs/ConceptMapping"} + ) == [] + + +def test_plain_semantic_model_does_not_load_ontology_schema(offline, monkeypatch): + core_schema = _schema("core-spec/ossie-schema.json") + ontology_path = (_REPO / "ontology/ontology.json").resolve() + original_read_text = Path.read_text + _VALIDATE._retrieve_local_schema.cache_clear() + + def reject_ontology_read(path, *args, **kwargs): + if path.resolve() == ontology_path: + pytest.fail("Plain semantic model validation must not load ontology.json") + return original_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", reject_ontology_read) + + assert _VALIDATE.validate_schema(_semantic_model(), core_schema) == [] + + +@pytest.mark.parametrize("failure", ["missing", "malformed"]) +def test_unavailable_referenced_ontology_is_validation_error( + offline, mapping_schema, monkeypatch, failure +): + ontology_path = (_REPO / "ontology/ontology.json").resolve() + original_read_text = Path.read_text + _VALIDATE._retrieve_local_schema.cache_clear() + + def fail_ontology_read(path, *args, **kwargs): + if path.resolve() == ontology_path: + if failure == "missing": + raise FileNotFoundError(path) + return "{" + return original_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", fail_ontology_read) + + errors = _VALIDATE.validate_schema(_mapping_document(), mapping_schema) + expected_ref = f"{_ONTOLOGY_RAW_URL}#/$defs/ConceptMapping" + + assert errors == [f"[Schema] Cannot resolve schema reference: {expected_ref}"] + + +@pytest.mark.parametrize( + "example, schema_path", + [ + ("flights.ontology.yaml", "ontology/ontology.json"), + ("flights.semantic_model.yaml", "core-spec/ossie-schema.json"), + ("flights.mapping.yaml", "ontology/mapping.json"), + ], +) +def test_split_flights_examples_validate_offline(offline, example, schema_path): + document = yaml.safe_load((_REPO / "examples" / example).read_text()) + + assert _VALIDATE.validate_schema(document, _schema(schema_path)) == [] + + +def test_split_flights_examples_match_embedded_example(): + flights = yaml.safe_load((_REPO / "examples/flights.yaml").read_text()) + ontology = yaml.safe_load((_REPO / "examples/flights.ontology.yaml").read_text()) + model = yaml.safe_load((_REPO / "examples/flights.semantic_model.yaml").read_text()) + mapping = yaml.safe_load((_REPO / "examples/flights.mapping.yaml").read_text()) + (embedded,) = flights.pop("ontology_mappings") + + assert ontology == flights + assert model == embedded["semantic_model"] + assert mapping["concept_mappings"] == embedded["concept_mappings"] + assert mapping["ontology_ref"]["name"] == ontology["name"] + assert mapping["semantic_model_ref"]["name"] == model["name"] + + +def test_mapping_document_accepts_references(offline, mapping_schema): + assert _VALIDATE.validate_schema(_mapping_document(), mapping_schema) == [] + + +def test_mapping_document_reference_iri_is_optional(offline, mapping_schema): + document = _mapping_document() + del document["ontology_ref"]["iri"] + del document["semantic_model_ref"]["iri"] + + assert _VALIDATE.validate_schema(document, mapping_schema) == [] + + +@pytest.mark.parametrize( + "required_property", + ["version", "name", "ontology_ref", "semantic_model_ref", "concept_mappings"], +) +def test_mapping_document_requires_property(offline, mapping_schema, required_property): + document = _mapping_document() + del document[required_property] + + errors = _VALIDATE.validate_schema(document, mapping_schema) + + assert errors == [f"[Schema] (root): '{required_property}' is a required property"] + + +@pytest.mark.parametrize("reference", ["ontology_ref", "semantic_model_ref"]) +def test_mapping_document_reference_requires_name(offline, mapping_schema, reference): + document = _mapping_document() + del document[reference]["name"] + + assert _VALIDATE.validate_schema(document, mapping_schema) + + +@pytest.mark.parametrize( + "container, field", + [ + (None, "name"), + ("ontology_ref", "name"), + ("ontology_ref", "iri"), + ("semantic_model_ref", "name"), + ("semantic_model_ref", "iri"), + ], +) +def test_mapping_document_rejects_empty_identifiers( + offline, mapping_schema, container, field +): + document = _mapping_document() + target = document if container is None else document[container] + target[field] = "" + + assert _VALIDATE.validate_schema(document, mapping_schema) + + +@pytest.mark.parametrize( + "mutate", + [ + # Embedding a complete semantic model instead of referencing one. + lambda d: d.update(semantic_model_ref=_semantic_model()), + # More than one semantic model or ontology per mapping document. + lambda d: d.update( + semantic_model_ref=[d["semantic_model_ref"], d["semantic_model_ref"]] + ), + lambda d: d.update(ontology_ref=[d["ontology_ref"], d["ontology_ref"]]), + # Unknown fields, including a document-kind discriminator. + lambda d: d.update(kind="mapping"), + lambda d: d["ontology_ref"].update(version="1.0"), + # Wrong specification version. + lambda d: d.update(version="0.1.0"), + ], + ids=[ + "embedded-semantic-model", + "two-semantic-models", + "two-ontologies", + "kind-field", + "unknown-reference-field", + "wrong-version", + ], +) +def test_mapping_document_rejects_invalid_shapes(offline, mapping_schema, mutate): + document = _mapping_document() + mutate(document) + + assert _VALIDATE.validate_schema(document, mapping_schema) diff --git a/validation/validate.py b/validation/validate.py index b51982d9..2a5b1d8e 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -50,8 +50,9 @@ try: import yaml from jsonschema import Draft202012Validator - from referencing import Registry, Resource - from referencing.exceptions import Unresolvable + from referencing import Registry + from referencing.exceptions import NoSuchResource, Unresolvable + from referencing.retrieval import to_cached_resource from yaml.constructor import ConstructorError except ImportError: print("Missing dependencies. Install with:") @@ -148,20 +149,34 @@ def _check_unique_keys(self, node: yaml.Node, visited: set) -> None: self._check_unique_keys(child, visited) +# Ossie schemas reference one another by raw GitHub URL or canonical $id. +# Resolve those URLs onto files in this checkout only when a reference needs +# them. The decorator parses and caches each retrieved schema across calls. +_REPO_ROOT = Path(__file__).parent.parent.resolve() +_SCHEMA_BASES = ( + "https://raw.githubusercontent.com/apache/ossie/main/", + "https://github.com/apache/ossie/", +) + + +@to_cached_resource() +def _retrieve_local_schema(uri: str) -> str: + """Read a referenced Ossie schema from this checkout.""" + for base in _SCHEMA_BASES: + if uri.startswith(base): + path = (_REPO_ROOT / uri[len(base):]).resolve() + if path.is_relative_to(_REPO_ROOT): + return path.read_text(encoding="utf-8") + break + raise NoSuchResource(ref=uri) + + +_SCHEMA_REGISTRY = Registry(retrieve=_retrieve_local_schema) + + def validate_schema(data: dict, schema: dict) -> list[str]: - """Validate against JSON Schema, resolving core references locally.""" - core_path = Path(__file__).parent.parent / "core-spec" / "ossie-schema.json" - core = json.loads(core_path.read_text()) - resource = Resource.from_contents(core) - # Ontology references use the raw URL; also register the canonical schema ID. - registry = Registry().with_resources([ - (core["$id"], resource), - ( - "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json", - resource, - ), - ]) - validator = Draft202012Validator(schema, registry=registry) + """Validate against JSON Schema, resolving Ossie schema references locally.""" + validator = Draft202012Validator(schema, registry=_SCHEMA_REGISTRY) errors = [] try: for error in validator.iter_errors(data):