diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 744b506..e0d74ce 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v6 diff --git a/readme.md b/readme.md index ff95402..d7b8f4c 100644 --- a/readme.md +++ b/readme.md @@ -78,8 +78,8 @@ Usage: underground feed [OPTIONS] ROUTE_OR_URL Request an MTA feed via a route or URL. - ROUTE_OR_URL may be either a feed URL or a route (which will be used to - look up the feed url). + ROUTE_OR_URL may be either a feed URL or a route (which will be used to look + up the feed url). ROUTE_OR_URL may also be "BUS" to access the bus feed. Examples (both access the same feed): @@ -102,30 +102,29 @@ Options: ``` $ underground stops --help -Usage: underground stops [OPTIONS] [H|M|D|1|Z|A|N|GS|SI|J|G|Q|L|B|R|F|E|2|7|W| - 6|4|C|5|FS] - +Usage: underground stops [OPTIONS] ROUTE + Print out train departure times for all stops on a subway line. Options: - - -f, --format TEXT strftime format for stop times. Use `epoch` for a - unix timestamp. - -r, --retries INTEGER Retry attempts in case of API connection failure. - Default 100. - -t, --timezone TEXT Output timezone. Ignored if --epoch. Default to NYC - time. + -f, --format TEXT strftime format for stop times. Use `epoch` + for a unix timestamp. + -r, --retries INTEGER Retry attempts in case of API connection + failure. Default 100. + -t, --timezone TEXT Output timezone. Ignored if --epoch. Default + to NYC time. -s, --stalled-timeout INTEGER Number of seconds between the last movement of a train and the API update before considering a train stalled. Default is 90 as recommended by the MTA. Numbers less than 1 disable this check. - --help Show this message and exit. + --bus Set if the route is a bus route. + --help Show this message and exit. ``` Stops are printed to stdout in the format `stop_id t1 t2 ... tn` . -``` sh +```sh $ underground stops Q | tail -2 Q05S 19:01 19:09 19:16 19:25 19:34 19:44 19:51 19:58 Q04S 19:03 19:11 19:18 19:27 19:36 19:46 19:53 20:00 @@ -168,3 +167,50 @@ ID: D27S Direction: SOUTH Lat/Lon: 40.655292, -73.961495 Name: PARKSIDE ``` Some names are ambiguous (try "fulton st"), for these you'll have to dig into the [metadata](https://www.mta.info/developers#static-gtfs-data) more carefully. + +## Bus support + +`underground` was initially written for the MTA subway feeds. However, contributors to the package identified that some level of bus support could be achieved with minimal maintenance burden. Currently, `underground` supports the bus feed on a best-effort basis. + +### Python API + +All bus lines share the same feed, but lines can be ephemeral so `underground` cannot list them all in advance. So special care needs to be taken when resolving routes. Use the special `BUS` route to get the feed, and then find your route in the stops: + +```py +from underground import SubwayFeed +feed = SubwayFeed.get("BUS") +stops = feed.extract_stop_dict()['BX30'] +``` + +### CLI + +Use the `BUS` identifier in the `feed` cli to obtain the bus feed. Querying a bus route will not work! + +```sh +underground feed BUS +``` + +Similarly, use a `--bus` flag when obtaining stops to let underground know that it needs to retrieve the bus feed: + +```sh +➜ underground stops --bus BX30 | tail -2 +101735 18:42 18:57 19:12 +104619 18:43 18:58 19:13 +``` + +The bus stops lister in `findstops` is more expensive/time-consuming than the subway stops list, so use the `--bus` flag to indicate bus stops are desired: + +```sh +➜ underground findstops --bus parkside +ID: D27N Direction: NORTH Data Source: subway Lat/Lon: 40.655292,-73.961495 Name: PARKSIDE AV +ID: D27S Direction: SOUTH Data Source: subway Lat/Lon: 40.655292,-73.961495 Name: PARKSIDE AV +ID: 301309 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655129,-73.961122 Name: PARKSIDE AV/OCEAN AV +ID: 301310 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655518,-73.960028 Name: PARKSIDE AV/FLATBUSH AV +ID: 301311 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655815,-73.956542 Name: PARKSIDE AV/BEDFORD AV +ID: 303307 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655257,-73.959894 Name: FLATBUSH AV/PARKSIDE AV +ID: 303981 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655541,-73.956442 Name: BEDFORD AV/PARKSIDE AV +ID: 307645 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.654503,-73.961946 Name: OCEAN AV/PARKSIDE AV +ID: 308349 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.655139,-73.961837 Name: OCEAN AV/PARKSIDE AV +ID: 901089 Direction: (BUS) Data Source: buses_bk Lat/Lon: 40.654848,-73.961754 Name: PARKSIDE AV/OCEAN AV +``` + diff --git a/setup.py b/setup.py index f345221..a2fc9a9 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ], keywords=["nyc", "transit", "subway", "command-line", "cli"], license="MIT", diff --git a/src/underground/cli/feed.py b/src/underground/cli/feed.py index d169e6a..c51d901 100644 --- a/src/underground/cli/feed.py +++ b/src/underground/cli/feed.py @@ -4,7 +4,7 @@ import click -from underground import feed, metadata +from underground import feed @click.command() @@ -23,11 +23,11 @@ type=int, help="Retry attempts in case of API connection failure. Default 100.", ) -def main(route_or_url, output_json, retries): +def main(route_or_url: str, output_json: bool, retries: int): """Request an MTA feed via a route or URL. ROUTE_OR_URL may be either a feed URL or a route (which will be used to look up - the feed url). + the feed url). ROUTE_OR_URL may also be "BUS" to access the bus feed. Examples (both access the same feed): @@ -38,18 +38,7 @@ def main(route_or_url, output_json, retries): URL='https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs-nqrw' && underground feed $URL --json > feed_nrqw.json """ - if route_or_url not in metadata.VALID_ROUTES.union(metadata.VALID_FEED_URLS): - raise click.ClickException( - f"Invalid route or URL. Try a route like {metadata.VALID_ROUTES} " - "or a url from https://api.mta.info/#/subwayRealTimeFeeds." - ) - - data = feed.request_robust( - route_or_url=route_or_url, - retries=retries, - return_dict=output_json, - ) - + data = feed.request_robust(route_or_url=route_or_url, retries=retries, return_dict=output_json) if output_json: click.echo(json.dumps(data)) else: diff --git a/src/underground/cli/findstops.py b/src/underground/cli/findstops.py index c7f2bbd..63c652e 100644 --- a/src/underground/cli/findstops.py +++ b/src/underground/cli/findstops.py @@ -54,6 +54,7 @@ def add_source(row: dict[str, str]) -> dict[str, str]: help="Option to output the data as JSON. Otherwise will be human readable table.", ) @click.option( + "--bus", "--buses", "include_buses", is_flag=True, diff --git a/src/underground/cli/stops.py b/src/underground/cli/stops.py index 28448c0..0992db2 100644 --- a/src/underground/cli/stops.py +++ b/src/underground/cli/stops.py @@ -15,7 +15,7 @@ def datetime_to_epoch(dttm: datetime.datetime) -> int: @click.command() -@click.argument("route", nargs=1, type=click.Choice(metadata.VALID_ROUTES)) +@click.argument("route", type=str, nargs=1) @click.option( "-f", "--format", @@ -48,10 +48,12 @@ def datetime_to_epoch(dttm: datetime.datetime) -> int: " update before considering a train stalled. Default is 90 as recommended" " by the MTA. Numbers less than 1 disable this check.", ) -def main(route, fmt, retries, timezone, stalled_timeout): +@click.option("--bus", is_flag=True, help="Set if the route is a bus route.") +def main(route: str, fmt: str, retries: int, timezone: str, stalled_timeout: int, bus: bool): """Print out train departure times for all stops on a subway line.""" + route_or_url = route if not bus else metadata.BUS_URL stops = ( - SubwayFeed.get(route_or_url=route, retries=retries) + SubwayFeed.get(route_or_url=route_or_url, retries=retries) .extract_stop_dict(timezone=timezone, stalled_timeout=stalled_timeout) .get(route, dict()) ) diff --git a/src/underground/metadata.py b/src/underground/metadata.py index 2da769b..964348e 100644 --- a/src/underground/metadata.py +++ b/src/underground/metadata.py @@ -3,8 +3,10 @@ # because its for the NYC subway DEFAULT_TIMEZONE = "America/New_York" +# all buses share the same feed url +BUS_URL = "https://gtfsrt.prod.obanyc.com/tripUpdates" + # map a tuple of all routes per -# fmt: off FEED_GROUPS = { "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs": ( "1", @@ -15,8 +17,11 @@ "6", "GS", "7", - # SS route is undocumented but i have seen it here "in the wild". + # Some unusual routes that i have seen in the wild "SS", + "5X", + "6X", + "7X", ), "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs-ace": ( "A", @@ -30,6 +35,7 @@ "F", "M", "FS", + "FX", # seen this in the wild as well ), "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs-g": ("G",), "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs-jz": ("J", "Z"), @@ -41,69 +47,32 @@ "W", ), "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs-si": ("SI",), - # buses all share the same feed - "https://gtfsrt.prod.obanyc.com/tripUpdates": ( - "B1", "B100", "B101", "B103", "B106", "B11", "B111", "B12", "B13", "B14", "B15", - "B16", "B17", "B2", "B20", "B24", "B25", "B26", "B3", "B31", "B32", "B35", - "B36", "B37", "B38", "B39", "B4", "B41", "B42", "B43", "B44", "B44+", "B45", - "B46", "B46+", "B47", "B48", "B49", "B52", "B54", "B57", "B6", "B60", "B61", - "B62", "B63", "B64", "B65", "B67", "B68", "B69", "B7", "B70", "B74", "B8", - "B82", "B82+", "B83", "B84", "B9", "B90", "B93", "B94", "B96", "B98", "B99", - "BM1", "BM2", "BM3", "BM4", "BM5", "BX1", "BX10", "BX11", "BX12", "BX12+", - "BX13", "BX15", "BX16", "BX17", "BX18A", "BX18B", "BX19", "BX2", "BX20", "BX21", - "BX22", "BX23", "BX24", "BX25", "BX26", "BX27", "BX28", "BX29", "BX3", "BX30", - "BX31", "BX32", "BX33", "BX34", "BX35", "BX36", "BX38", "BX39", "BX4", "BX40", - "BX41", "BX41+", "BX42", "BX46", "BX4A", "BX5", "BX6", "BX6+", "BX7", "BX8", - "BX9", "BX90", "BX92", "BX95", "BXM1", "BXM10", "BXM11", "BXM18", "BXM2", - "BXM3", "BXM4", "BXM6", "BXM7", "BXM8", "BXM9", "D90", "D99", "F1", "J90", - "J99", "L0202", "L0719", "L1227", "L90", "L91", "L92", "M1", "M10", "M100", - "M101", "M102", "M103", "M104", "M106", "M11", "M116", "M12", "M125", "M14A+", - "M14D+", "M15", "M15+", "M191", "M2", "M20", "M21", "M22", "M23+", "M3", "M31", - "M34+", "M34A+", "M35", "M4", "M42", "M5", "M50", "M55", "M57", "M60+", "M66", - "M7", "M72", "M79+", "M8", "M86+", "M9", "M90", "M96", "M98", "Q06", "Q07", - "Q08", "Q09", "Q1", "Q10", "Q100", "Q101", "Q102", "Q103", "Q104", "Q107", - "Q108", "Q11", "Q110", "Q111", "Q112", "Q113", "Q114", "Q115", "Q12", "Q13", - "Q14", "Q15", "Q16", "Q17", "Q18", "Q19", "Q2", "Q20", "Q22", "Q23", "Q24", - "Q25", "Q26", "Q27", "Q28", "Q29", "Q3", "Q30", "Q31", "Q32", "Q33", "Q35", - "Q36", "Q37", "Q38", "Q39", "Q4", "Q40", "Q41", "Q42", "Q43", "Q44+", "Q45", - "Q46", "Q47", "Q48", "Q49", "Q5", "Q50", "Q51", "Q52+", "Q53+", "Q54", "Q55", - "Q56", "Q58", "Q59", "Q60", "Q61", "Q63", "Q64", "Q65", "Q66", "Q67", "Q69", - "Q70+", "Q72", "Q74", "Q75", "Q76", "Q77", "Q80", "Q82", "Q83", "Q84", "Q85", - "Q86", "Q87", "Q88", "Q89", "Q90", "Q92", "Q93", "Q96", "Q98", "QM1", "QM10", - "QM11", "QM12", "QM15", "QM16", "QM17", "QM18", "QM2", "QM20", "QM21", "QM24", - "QM25", "QM31", "QM32", "QM34", "QM35", "QM36", "QM4", "QM40", "QM42", "QM44", - "QM5", "QM6", "QM63", "QM64", "QM65", "QM68", "QM7", "QM8", "S40", "S42", "S44", - "S46", "S48", "S51", "S52", "S53", "S54", "S55", "S56", "S57", "S59", "S61", - "S62", "S66", "S74", "S76", "S78", "S79+", "S81", "S84", "S86", "S89", "S90", - "S91", "S92", "S93", "S94", "S96", "S96B1", "S96B2", "S96B3", "S98", "SIM1", - "SIM10", "SIM11", "SIM15", "SIM1C", "SIM2", "SIM22", "SIM23", "SIM24", "SIM25", - "SIM26", "SIM3", "SIM30", "SIM31", "SIM32", "SIM33", "SIM33C", "SIM34", "SIM35", - "SIM3C", "SIM4", "SIM4C", "SIM4X", "SIM5", "SIM6", "SIM7", "SIM8", "SIM8X", - "SIM9", "X27", "X28", "X37", "X38" ) } -# fmt: on # get a route to feed mapping ROUTE_FEED_MAP = {route: url for url, routes in FEED_GROUPS.items() for route in routes} -# There are some routes with aliases that we need to map onto the original ID. -# The empty string occurs if the trip has not yet been assigned a route. I have found -# these cases "in the wild" and would rather write code to skip over them than have -# the application break. -ROUTE_REMAP = {"5X": "5", "6X": "6", "7X": "7", "FX": "F", "": ""} -for k in ROUTE_FEED_MAP: - ROUTE_REMAP[k] = k VALID_ROUTES = set(ROUTE_FEED_MAP.keys()) VALID_FEED_URLS = set(ROUTE_FEED_MAP.values()) +class UnknownRouteOrURL(Exception): + """Thrown when an unknown route or URL is provided.""" + + def resolve_url(route_or_url: str) -> str: """Return the correct URL, given an unknown route ID or a url.""" if route_or_url in VALID_FEED_URLS: return route_or_url - if route_or_url not in ROUTE_REMAP: - raise ValueError(f"Unknown route or url: {route_or_url}") + if route_or_url == BUS_URL: + return route_or_url + + if route_or_url == "BUS": + return BUS_URL + + if route_or_url not in ROUTE_FEED_MAP: + raise UnknownRouteOrURL(f"Unknown route or url: {route_or_url}") - return ROUTE_FEED_MAP[ROUTE_REMAP[route_or_url]] + return ROUTE_FEED_MAP[route_or_url] diff --git a/src/underground/models.py b/src/underground/models.py index 62bb569..b09652b 100644 --- a/src/underground/models.py +++ b/src/underground/models.py @@ -50,24 +50,6 @@ def check_start_date(cls, start_date): return start_date - @pydantic.field_validator("route_id") - def check_route(cls, route_id): - """Check for a valid route ID value.""" - if route_id not in metadata.ROUTE_REMAP: - raise ValueError( - f"Invalid route ({route_id}). Must be one of {set(metadata.ROUTE_REMAP.keys())}." - ) - - return route_id - - @property - def route_id_mapped(self) -> str: - """Find the parent route ID. - - This is helpful for grabbing the, e.g., 5 Train when you might have a 5X. - """ - return metadata.ROUTE_REMAP[self.route_id] - @property def route_is_assigned(self) -> bool: """Return a flag indicating that there is a route.""" @@ -187,7 +169,7 @@ def get(cls, route_or_url: str, retries: int = 100) -> "SubwayFeed": route_or_url : str Route ID or feed url (per ``https://api.mta.info/#/subwayRealTimeFeeds``). If a route, the URL for that route is looked up. All routes served by that - URL will be included in the result. + URL will be included in the result. Set route_or_url to 'BUS' to obtain bus updates. retries : int Number of retry attempts, with 1 second timeout between attempts. Set to -1 for unlimited. Default 100. @@ -198,6 +180,9 @@ def get(cls, route_or_url: str, retries: int = 100) -> "SubwayFeed": An instance of the SubwayFeed class with the requested data. """ + if route_or_url == "BUS": + route_or_url = metadata.BUS_URL + return cls( **feed.request_robust(route_or_url=route_or_url, retries=retries, return_dict=True) ) @@ -246,7 +231,7 @@ def is_trip_active(update: TripUpdate) -> bool: # create (route, stop, time) tuples from each trip stops_flat = ( ( - trip.trip.route_id_mapped, + trip.trip.route_id, stop.stop_id, (stop.depart_or_arrive).time.astimezone(zoneinfo.ZoneInfo(timezone)), ) diff --git a/src/underground/version b/src/underground/version index 6d7de6e..227cea2 100644 --- a/src/underground/version +++ b/src/underground/version @@ -1 +1 @@ -1.0.2 +2.0.0 diff --git a/test/test_feed.py b/test/test_feed.py index 169e101..36a13d0 100644 --- a/test/test_feed.py +++ b/test/test_feed.py @@ -59,7 +59,7 @@ def test_emptyfeederror(monkeypatch, dict_data): def test_request_invalid_feed(): """Test that request raises value error for an invalid feed.""" - with pytest.raises(ValueError): + with pytest.raises(metadata.UnknownRouteOrURL): feed.request("NOT REAL") diff --git a/test/test_metadata.py b/test/test_metadata.py new file mode 100644 index 0000000..b22af1f --- /dev/null +++ b/test/test_metadata.py @@ -0,0 +1,27 @@ +import pytest + +from underground import metadata + + +@pytest.mark.parametrize( + "fin,expected_url", + [ + ("1", "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs"), + ( + "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs", + "https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs", + ), + ["BUS", metadata.BUS_URL], + [metadata.BUS_URL, metadata.BUS_URL] + ], +) +def test_resolve_url(fin: str, expected_url: str): + """Test that resolve_url works correctly.""" + url = metadata.resolve_url(fin) + assert url == expected_url + + +def test_resolve_url_invalid(): + """Test that resolve_url raises for invalid route or url.""" + with pytest.raises(metadata.UnknownRouteOrURL): + metadata.resolve_url("INVALID_ROUTE") diff --git a/test/test_models.py b/test/test_models.py index 2a72859..7642baa 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -34,12 +34,6 @@ def test_trip_invalid_date(): models.Trip(trip_id="1", start_date=0, route_id="5") -def test_trip_invalid_feed(): - """Test valuerror for invalid feeds.""" - with pytest.raises(ValueError): - models.Trip(trip_id="1", start_date=20190101, route_id="FAKE") - - def test_extract_stop_dict(): """Test that the correct train times are extracted.""" sample_data = { @@ -93,34 +87,6 @@ def test_get(requests_mock, filename): assert isinstance(feed.extract_stop_dict(), dict) -def test_trip_route_remap(): - """Test that the remapping works for a known route.""" - trip = models.Trip(trip_id="FAKE", start_time="01:00:00", start_date=20190101, route_id="5X") - assert trip.route_id_mapped == "5" - - -def test_extract_dict_route_remap(): - """Test that the route remap is active for dict extraction.""" - sample_data = { - "header": {"gtfs_realtime_version": "1.0", "timestamp": 0}, - "entity": [ - { - "id": "X", - "trip_update": { - "trip": { - "trip_id": "X", - "start_date": "20190726", - "route_id": "5X", - }, - "stop_time_update": [{"arrival": {"time": 0}, "stop_id": "X"}], - }, - } - ], - } - stops = SubwayFeed(**sample_data).extract_stop_dict() - assert len(stops["5"]["X"]) == 1 - - def test_extract_dict_elapsed_ignored(): """Test that elapsed stops are ignored for stop extraction.""" sample_data = { @@ -230,13 +196,13 @@ def test_extract_dict_stalled_train_omitted(): def test_empty_route_id(): - """Test the route functionality when the route id is a blacnk string. + """Test the route functionality when the route id is a blank string. This uses example data i found in the wild. """ trip = {"route_id": "", "start_date": "20191120", "trip_id": "060750_..N"} trip = models.Trip(**trip) - assert trip.route_id_mapped == "" + assert trip.route_id == "" assert not trip.route_is_assigned