diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f90ba4d6b..614481c89 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -131,6 +131,7 @@ jobs: pip3 install . pip3 install GDAL==`gdal-config --version` pip3 install "pyproj<3.8" + pip3 install "SQLAlchemy==2.0.54" - name: setup test data ⚙️ run: | pybabel compile -d locale -l es diff --git a/.github/workflows/vulnerabilities.yml b/.github/workflows/trivy.yml similarity index 89% rename from .github/workflows/vulnerabilities.yml rename to .github/workflows/trivy.yml index 3c3e0ed25..1d8e97ad0 100644 --- a/.github/workflows/vulnerabilities.yml +++ b/.github/workflows/trivy.yml @@ -12,6 +12,8 @@ on: release: types: - released + schedule: + - cron: '0 8 * * *' # runs daily at 8h UTC jobs: @@ -24,7 +26,7 @@ jobs: - name: Checkout pygeoapi uses: actions/checkout@master - name: Scan vulnerabilities with trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs exit-code: 1 @@ -39,7 +41,7 @@ jobs: run: | docker buildx build -t ${{ env.IMAGE_NAME }}:${{ github.sha }} --platform linux/amd64 --no-cache -f Dockerfile . - name: Scan locally built Docker image for vulnerabilities with trivy - uses: aquasecurity/trivy-action@v0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 env: TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2 TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1 diff --git a/README.md b/README.md index d0d8c8a50..b2ed3036e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![DOI](https://zenodo.org/badge/121585259.svg)](https://zenodo.org/badge/latestdoi/121585259) [![Build](https://github.com/geopython/pygeoapi/actions/workflows/main.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/main.yml) [![Docker](https://github.com/geopython/pygeoapi/actions/workflows/containers.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/containers.yml) -[![Vulnerabilities](https://github.com/geopython/pygeoapi/actions/workflows/vulnerabilities.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/vulnerabilities.yml) +[![Vulnerabilities](https://github.com/geopython/pygeoapi/actions/workflows/trivy.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/trivy.yml) [pygeoapi](https://pygeoapi.io) is a Python server implementation of the [OGC API](https://ogcapi.ogc.org) suite of standards. The project emerged as part of the next generation OGC API efforts in 2018 and provides the capability for organizations to deploy a RESTful OGC API endpoint using OpenAPI, GeoJSON, and HTML. pygeoapi is [open source](https://opensource.org/) and released under an [MIT license](https://github.com/geopython/pygeoapi/blob/master/LICENSE.md). diff --git a/docs/source/development.rst b/docs/source/development.rst index 8ea86cc66..bf49a0426 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -16,7 +16,7 @@ A given GitHub Pull Request is evaluated against the following GitHub Actions: - ``main``: mainline testing harness (as defined in ``tests``) - ``flake8``: code linting - ``docs``: documentation updates (for files updated in ``docs/**.rst``) -- ``vulnerabilities``: Trivy vulnerability scanning +- ``trivy``: Trivy vulnerability scanning Testing ------- diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index 619e07127..08fd6ff2e 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -64,12 +64,12 @@ def __init__(self, provider_def): open_options = {} squeeze = provider_def.get('options', {}).get('squeeze', False) - zarr_options = provider_def.get('options', {}).get('zarr', {}) + self.zarr_options = provider_def.get('options', {}).get('zarr', {}) try: if provider_def['data'].endswith('.zarr'): open_func = xarray.open_zarr - open_options = zarr_options + open_options = self.zarr_options else: if '*' in self.data: LOGGER.debug('Detected multi file dataset') @@ -158,7 +158,8 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, if not properties and not subsets and format_ != 'json': LOGGER.debug('No parameters specified, returning native data') if format_ == 'zarr': - return _get_zarr_data(self._data) + zarr_format = self.zarr_options.get('zarr_format') + return _get_zarr_data(self._data, zarr_format) else: return read_data(self.data) @@ -658,7 +659,7 @@ def _zip_dir(path, ziph, cwd): os.chdir(cwd) -def _get_zarr_data(data): +def _get_zarr_data(data, zarr_format=None): """ Returns bytes to read from Zarr directory zip :param data: Xarray dataset of coverage data @@ -671,7 +672,7 @@ def _get_zarr_data(data): zarr_data_filename = f'{tmp_dir}zarr.zarr' zarr_zip_filename = f'{tmp_dir}zarr.zarr.zip' - data.to_zarr(zarr_data_filename, mode='w') + data.to_zarr(zarr_data_filename, mode='w', zarr_format=zarr_format) with zipfile.ZipFile(zarr_zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: # noqa _zip_dir(zarr_data_filename, zipf, os.getcwd()) diff --git a/requirements-docker.txt b/requirements-docker.txt index 5a98fca38..38306f01f 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -4,8 +4,8 @@ pygeofilter[backend-sqlalchemy] pygeoif pygeometa sodapy -s3fs<=2023.6.0 +s3fs Flask>=2.2.0 Flask-Cors urllib3>2 -botocore>1.42 \ No newline at end of file +botocore>1.42 diff --git a/requirements-provider.txt b/requirements-provider.txt index 92d90045a..6e64a3b57 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -18,10 +18,10 @@ pyarrow pygeofilter[backend-sqlalchemy] pygeoif pygeometa -pymongo==4.18.3 +pymongo pymysql scipy sodapy xarray -zarr<3 -s3fs<=2023.6.0 +zarr +s3fs diff --git a/tests/provider/test_xarray_zarr_provider.py b/tests/provider/test_xarray_zarr_provider.py index 1b66f6a07..76a99c85c 100644 --- a/tests/provider/test_xarray_zarr_provider.py +++ b/tests/provider/test_xarray_zarr_provider.py @@ -52,6 +52,11 @@ def config(): 'format': { 'name': 'zarr', 'mimetype': 'application/zip' + }, + 'options': { + 'zarr': { + 'zarr_format': 2 + } } } @@ -61,12 +66,17 @@ def config_no_time(tmp_path): ds = xr.open_zarr(path) ds = ds.sel(time=ds.time[0]) ds = ds.drop_vars('time') - ds.to_zarr(tmp_path / 'no_time.zarr') + ds.to_zarr(tmp_path / 'no_time.zarr', zarr_format=2) return { 'name': 'zarr', 'type': 'coverage', 'data': str(tmp_path / 'no_time.zarr'), 'format': {'name': 'zarr', 'mimetype': 'application/zip'}, + 'options': { + 'zarr': { + 'zarr_format': 2 + } + } }