diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ec0bd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +**__pycache__** + +# Python packaging / build artifacts +*.egg-info/ +build/ +dist/ +*.egg + +# Virtual environments +.venv/ +venv/ +env/ + +# Test / linter caches +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Jupyter +.ipynb_checkpoints/ + +# Editors / OS +.DS_Store +.vscode/ +.idea/ diff --git a/data_strategy.md b/data_strategy.md index b97f08b..8399987 100644 --- a/data_strategy.md +++ b/data_strategy.md @@ -184,7 +184,8 @@ __Special note on `IdentityFile=~/.ssh/alpine`:__ this `sshfs` option tells SSH - __MacOS:__ install [macFUSE](https://github.com/macfuse/macfuse/wiki/File-Systems-%E2%80%90-SSHFS) and `sshfs`: ```shell - brew install --cask macfuse + brew tap macos-fuse-t/homebrew-cask + brew install fuse-t fuse-t-sshfs ``` Then mount with `sshfs`, for example: @@ -217,3 +218,40 @@ __Special note on `IdentityFile=~/.ssh/alpine`:__ this `sshfs` option tells SSH ``` Note: speed comparisons between `sshfs` (PetaLibrary) and CIFS (Isilon) have not yet been performed; consider this if performance becomes a concern. + +### Connection script + +Please feel free to use the following script to automatically help setup your mount point to the Way Lab specific mount point on CU Boulder Research Computing PetaLibrary: `koala`. + +```shell +curl https://raw.githubusercontent.com/WayScience/playbooks/refs/heads/main/internal/mount_koala.sh | sh +``` + +## Mounting both Isilon and PetaLibrary at once + +Please feel free to use the following script to automatically help setup your mount point to the Way Lab specific mount point for both PetaLibrary and Isilon: `koala` and `bandicoot`, respectively. + +```shell +curl https://raw.githubusercontent.com/WayScience/playbooks/refs/heads/main/internal/mount_nas.sh | sh +``` + +## Interacting with both filesystems programmatically + +This can be done using a helper function `nas_path_check` in `internal/nas_path_set.py` which will check if the path is on either filesystem and return the root directory of the filesystem and whether the code is running in a notebook environment. + +This function can be installed in your Python environment with the following command: + +```shell +pip install git+https://github.com/WayScience/playbooks.git#subdirectory=internal/nas_path_package +``` + +```python +from nas_path_package import init_notebook, nas_path_check + +root_dir, in_notebook = init_notebook() +data_dir = nas_path_check(root_dir, nas_name="bandicoot") +``` + +The data_dir can now be used to access the data on the NAS filesystem. +If the NAS is not mounted, it will fall back to the enclosing Git repository's root directory. +This keeps the code portable and allows for easy access to data on the NAS filesystem without hardcoding paths and allows for someone without access to the NAS to still run the code without errors using local storage. diff --git a/internal/mount_bandicoot.sh b/internal/mount_bandicoot.sh index d0b0da9..ac4cfc8 100644 --- a/internal/mount_bandicoot.sh +++ b/internal/mount_bandicoot.sh @@ -9,7 +9,6 @@ # verifies VPN/network access, and works under any POSIX shell # (sh, bash, zsh, dash, etc.). # –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– - set -eu # -e: exit immediately on any error # -u: treat unset variables as an error @@ -51,7 +50,7 @@ case "$OS" in echo "→ Detected macOS (Darwin). Using mount_smbfs." # # mount_smbfs is the built-in macOS SMB client. - # It will prompt you for credentials if required, + # It will prompt you for a password if required, # or use your current login keychain. # # NOTE: mount_smbfs falls back to your local Mac shortname as the @@ -68,8 +67,35 @@ case "$OS" in exit 1 fi - SHARE_WITH_USER="//${SMB_USERNAME}@${SHARE#//}" - mount_smbfs "$SHARE_WITH_USER" "$MOUNT_POINT" + # NOTE: unlike mount.cifs on Linux, mount_smbfs has no "domainauto" + # option — there is no way for it to negotiate the AD domain on its + # own, so it must be supplied explicitly as //DOMAIN;user@host/share. + # A wrong domain here fails with a generic "Authentication error" + # that looks identical to a wrong password, which is what was + # happening with the hardcoded DOMAIN="UCDENVER" value. Since the + # correct NetBIOS domain can differ depending on how an account was + # provisioned (CU Anschutz's AD forest predates its current name), + # prompt for it instead of hardcoding it, so anyone hitting this can + # self-correct without editing the script. + printf "AD domain for %s [default: UCDENVER, leave as '-' to omit]: " "$HOST" >/dev/tty + read -r SMB_DOMAIN &2 + echo " A failure here is almost always a wrong AD domain, not a" >&2 + echo " wrong password. Try again with a different domain (e.g." >&2 + echo " UCHSC, the campus's legacy AD name) or '-' to omit it." >&2 + exit 1 + fi ;; Linux) @@ -106,8 +132,11 @@ case "$OS" in exit 1 fi # Mount the share with domainauto for automatic domain selection - sudo mount -t cifs "$SHARE" "$MOUNT_POINT" \ - -o username="$CIFS_USERNAME",uid="$USER",gid="$USER",domainauto,file_mode=0777,dir_mode=0777 + if ! sudo mount -t cifs "$SHARE" "$MOUNT_POINT" \ + -o username="$CIFS_USERNAME",uid="$USER",gid="$USER",domainauto,file_mode=0777,dir_mode=0777; then + echo "✗ mount.cifs failed to mount $SHARE at $MOUNT_POINT." >&2 + exit 1 + fi ;; *) diff --git a/internal/mount_koala.sh b/internal/mount_koala.sh new file mode 100755 index 0000000..e3a5dd6 --- /dev/null +++ b/internal/mount_koala.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env sh +# shellcheck shell=sh +# –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– +# mount_koala.sh +# +# This script creates a local mount point and mounts a PetaLibrary +# directory on CU Boulder Research Computing's HPC Cluster Alpine +# (aka "koala") into ~/mnt/koala using sshfs over SSH. +# It auto-detects macOS vs. Linux, installs sshfs (and macFUSE on macOS) +# if needed, and works under any POSIX shell (sh, bash, zsh, dash, etc.). +# –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– + +set -eu +# -e: exit immediately on any error +# -u: treat unset variables as an error + +# Local directory where the PetaLibrary directory will be mounted +MOUNT_POINT="$HOME/mnt/koala" + +# Alpine SSH login host +ALPINE_HOST="login.rc.colorado.edu" + +# Path to the SSH private key used to authenticate to Alpine +IDENTITY_FILE="$HOME/.ssh/alpine" + +# ──────────────────────────────────────────────────────────────────────────── +# 1) Ensure the mount directory exists +# ──────────────────────────────────────────────────────────────────────────── +if [ ! -d "$MOUNT_POINT" ]; then + echo "→ Creating mount point directory: $MOUNT_POINT" + mkdir -p "$MOUNT_POINT" + # mkdir -p will also create any missing parent directories +fi + +# If a previous (possibly stale/broken) mount is already attached here, +# sshfs will fail with a generic "mount failed with error: -1" and we'd +# rather unmount it first than leave a confusing failure. +if mount | grep -q " on $MOUNT_POINT "; then + echo "→ $MOUNT_POINT is already mounted. Unmounting first..." + if ! umount "$MOUNT_POINT" 2>/dev/null; then + if command -v diskutil >/dev/null 2>&1; then + diskutil unmount force "$MOUNT_POINT" + elif command -v fusermount >/dev/null 2>&1; then + fusermount -u "$MOUNT_POINT" + fi + fi +fi + +# ──────────────────────────────────────────────────────────────────────────── +# 2) Prompt for Alpine username and PetaLibrary directory +# ──────────────────────────────────────────────────────────────────────────── +# NOTE: for CU Anschutz users, this will almost always take the form of an +# XSEDE-style identity: @xsede.org, rather than a plain +# CU Anschutz username. +printf "Alpine username (e.g. @xsede.org): " >/dev/tty +read -r ALPINE_USERNAME &2 + exit 1 +fi + +printf "PetaLibrary directory (relative to /pl/active/): " >/dev/tty +read -r PETALIBRARY_DIR &2 + exit 1 +fi + +if [ ! -f "$IDENTITY_FILE" ]; then + echo "✗ SSH identity file not found at $IDENTITY_FILE." >&2 + echo " Set up an SSH key for Alpine access and place it there," >&2 + echo " or edit IDENTITY_FILE in this script to point at your key." >&2 + exit 1 +fi + +# ──────────────────────────────────────────────────────────────────────────── +# 3) Detect operating system, ensure sshfs is installed, and mount +# ──────────────────────────────────────────────────────────────────────────── +OS="$(uname)" +case "$OS" in + Darwin) + # macOS branch + echo "→ Detected macOS (Darwin). Verifying FUSE-T and sshfs are installed..." + # + # sshfs on macOS requires a FUSE implementation. FUSE-T is used here + # instead of macFUSE because it doesn't need a kernel extension + # (no reboot, no Privacy & Security approval step). + # + if ! command -v sshfs >/dev/null 2>&1; then + echo "→ sshfs not found. Attempting installation via Homebrew..." + if ! command -v brew >/dev/null 2>&1; then + echo "✗ Homebrew not found. Please install FUSE-T and sshfs manually:" >&2 + echo " https://github.com/macos-fuse-t/fuse-t" >&2 + exit 1 + fi + # updating from deprecated tap macos-fuse-t/fuse-t + # to macos-fuse-t/homebrew-cask + brew tap macos-fuse-t/homebrew-cask + brew install fuse-t fuse-t-sshfs + + # FUSE-T ships libfuse-t.dylib instead of the classic libfuse + # name sshfs looks for, so symlink it in. + if [ ! -e /usr/local/lib/libfuse.2.dylib ]; then + sudo mkdir -p /usr/local/lib + sudo ln -s /usr/local/lib/libfuse-t.dylib /usr/local/lib/libfuse.2.dylib + fi + fi + ;; + + Linux) + # Linux branch + echo "→ Detected Linux. Verifying sshfs is installed..." + # + # sshfs is provided by the sshfs package (fuse-sshfs on some + # distributions). If it's missing, we detect your package manager + # and install it. + # + if ! command -v sshfs >/dev/null 2>&1; then + echo "→ sshfs not found. Attempting installation..." + if command -v apt-get >/dev/null 2>&1; then + echo " • Using apt-get to install sshfs" + sudo apt-get update + sudo apt-get install -y sshfs + elif command -v dnf >/dev/null 2>&1; then + echo " • Using dnf to install fuse-sshfs" + sudo dnf install -y fuse-sshfs + elif command -v apk >/dev/null 2>&1; then + echo " • Using apk to install sshfs" + sudo apk add sshfs + else + echo "✗ Unsupported package manager. Please install sshfs manually." >&2 + exit 1 + fi + fi + ;; + + *) + # Unsupported OS + echo "✗ Unsupported operating system: $OS" >&2 + exit 1 + ;; +esac + +echo "→ Mounting PetaLibrary directory /pl/active/$PETALIBRARY_DIR via sshfs..." +if ! sshfs -o IdentityFile="$IDENTITY_FILE" \ + "$ALPINE_USERNAME@$ALPINE_HOST:/pl/active/$PETALIBRARY_DIR" \ + "$MOUNT_POINT"; then + echo "✗ sshfs failed to mount /pl/active/$PETALIBRARY_DIR at $MOUNT_POINT." >&2 + exit 1 +fi + +# ──────────────────────────────────────────────────────────────────────────── +# 4) Success message +# ──────────────────────────────────────────────────────────────────────────── +echo "✔ Successfully mounted /pl/active/$PETALIBRARY_DIR at $MOUNT_POINT" diff --git a/internal/mounting_nas.sh b/internal/mounting_nas.sh new file mode 100755 index 0000000..9f936a0 --- /dev/null +++ b/internal/mounting_nas.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env sh +# shellcheck shell=sh +# –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– +# mounting_nas.sh +# +# Interactive entry point for mounting the lab's network storage. +# Prompts for which storage solution(s) to mount and delegates to +# mount_bandicoot.sh (Isilon, CIFS/SMB) and/or mount_koala.sh +# (PetaLibrary/Alpine, sshfs), which live alongside this script. +# –––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––– + +set -eu +# -e: exit immediately on any error +# -u: treat unset variables as an error + +echo "Which NAS would you like to mount?" +echo " 1) bandicoot (Isilon)" +echo " 2) koala (PetaLibrary / Alpine)" +echo " 3) both" +printf "Enter choice [1-3]: " >/dev/tty +read -r CHOICE &2 + exit 1 + ;; +esac diff --git a/internal/nas_path_package/README.md b/internal/nas_path_package/README.md new file mode 100644 index 0000000..06b3a17 --- /dev/null +++ b/internal/nas_path_package/README.md @@ -0,0 +1,26 @@ +# nas_path_package + +Helpers for locating Way Lab NAS mount points (`bandicoot`, `koala`) from notebooks and scripts, falling back to the enclosing Git repository's root directory when a mount isn't present. + +## Install + +If locally developing, install in editable mode: + +```shell +pip install -e internal/nas_path_package +``` + +If not developing, install from github: + +```shell +pip install git+https://github.com/WayScience/playbooks.git#subdirectory=internal/nas_path_package +``` + +## Usage + +```python +from nas_path_package import init_notebook, nas_path_check + +root_dir, in_notebook = init_notebook() +data_dir = nas_path_check(root_dir, nas_name="bandicoot") +``` diff --git a/internal/nas_path_package/pyproject.toml b/internal/nas_path_package/pyproject.toml new file mode 100644 index 0000000..5849fc2 --- /dev/null +++ b/internal/nas_path_package/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "nas_path_package" +version = "0.1.0" +description = "Helpers for locating Way Lab NAS mount points (bandicoot, koala) from notebooks and scripts." +readme = "README.md" +requires-python = ">=3.9" + +[project.optional-dependencies] +test = ["pytest"] + +[tool.hatch.build.targets.wheel] +packages = ["src/nas_path_package"] diff --git a/internal/nas_path_package/src/nas_path_package/core.py b/internal/nas_path_package/src/nas_path_package/core.py new file mode 100644 index 0000000..801a17c --- /dev/null +++ b/internal/nas_path_package/src/nas_path_package/core.py @@ -0,0 +1,92 @@ +"""Notebook initialization helpers and Bandicoot path utilities.""" + +from __future__ import annotations + +import os +import pathlib +from typing import Tuple + + +def init_notebook() -> Tuple[pathlib.Path, bool]: + """ + Initializes the notebook environment by determining the root directory of the Git repository + and checking if the code is running in a Jupyter notebook. + + Returns + ------- + Tuple[pathlib.Path, bool] + - pathlib.Path: The root directory of the Git repository. + - bool: True if running in a Jupyter notebook, False otherwise. + """ + try: + cfg = get_ipython().config + in_notebook = True + except NameError: + in_notebook = False + + # Get the current working directory + cwd = pathlib.Path.cwd() + + if (cwd / ".git").is_dir(): + root_dir = cwd + + else: + root_dir = None + for parent in cwd.parents: + if (parent / ".git").is_dir(): + root_dir = parent + break + + # Check if a Git root directory was found + if root_dir is None: + raise FileNotFoundError("No Git root directory found.") + return root_dir, in_notebook + + +def nas_path_check( + root_dir: pathlib.Path | None = None, + nas_name: str | None = None, +) -> pathlib.Path: + """ + This function determines if the external mount point for Bandicoot exists. + + Parameters + ---------- + root_dir : pathlib.Path | None, optional + The root directory of the Git repository. Defaults to the result of init_notebook(). + nas_name : str | None, optional + The name of the NAS mount point. If None, defaults to "bandicoot". + + Returns + ------- + pathlib.Path + The path to the Bandicoot mount point if it exists, otherwise the Git root directory. + + Notes + ----- + - If nas_name is None, a warning is printed and the function defaults to the Git root directory. + - If nas_name is not "bandicoot" or "koala", a warning is printed and the function defaults to the Git root directory. + - If the specified NAS mount point does not exist, a warning is printed and the function defaults to the Git root directory. + """ + if root_dir is None: + root_dir = init_notebook()[0] + + if nas_name is None: + print("Warning: nas_name is None. Defaulting to 'git root directory'.") + return root_dir + + if nas_name not in ["bandicoot", "koala"]: + print(f"Warning: nas_name must be either 'bandicoot' or 'koala'. Defaulting to 'git root directory'.") + return root_dir + + if nas_name == "bandicoot": + nas_path = pathlib.Path(os.path.expanduser("~/mnt/bandicoot")).resolve() + else: + nas_path = pathlib.Path(os.path.expanduser("~/mnt/koala")).resolve() + + if not os.path.ismount(nas_path): + # revert to the git root directory if the NAS mount point does not exist + print(f"Warning: {nas_name} mount point does not exist. Reverting to the Git root directory.") + return root_dir + + return nas_path diff --git a/internal/nas_path_package/tests/test_core.py b/internal/nas_path_package/tests/test_core.py new file mode 100644 index 0000000..09ce897 --- /dev/null +++ b/internal/nas_path_package/tests/test_core.py @@ -0,0 +1,69 @@ +import os +import pathlib + +import pytest + +from nas_path_package.core import init_notebook, nas_path_check + + +def test_init_notebook_finds_git_root_at_cwd(tmp_path, monkeypatch): + (tmp_path / ".git").mkdir() + monkeypatch.chdir(tmp_path) + + root_dir, in_notebook = init_notebook() + + assert root_dir == tmp_path + assert in_notebook is False + + +def test_init_notebook_finds_git_root_in_parent(tmp_path, monkeypatch): + (tmp_path / ".git").mkdir() + nested = tmp_path / "a" / "b" + nested.mkdir(parents=True) + monkeypatch.chdir(nested) + + root_dir, in_notebook = init_notebook() + + assert root_dir == tmp_path + + +def test_init_notebook_raises_without_git_root(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + with pytest.raises(FileNotFoundError): + init_notebook() + + +def test_nas_path_check_defaults_to_root_when_nas_name_none(tmp_path, capsys): + result = nas_path_check(root_dir=tmp_path, nas_name=None) + + assert result == tmp_path + assert "Warning" in capsys.readouterr().out + + +def test_nas_path_check_defaults_to_root_when_nas_name_invalid(tmp_path, capsys): + result = nas_path_check(root_dir=tmp_path, nas_name="not-a-real-nas") + + assert result == tmp_path + assert "Warning" in capsys.readouterr().out + + +@pytest.mark.parametrize("nas_name", ["bandicoot", "koala"]) +def test_nas_path_check_returns_mount_when_it_exists(tmp_path, monkeypatch, nas_name): + monkeypatch.setattr(os.path, "ismount", lambda path: True) + + result = nas_path_check(root_dir=tmp_path, nas_name=nas_name) + + assert result == pathlib.Path(f"~/mnt/{nas_name}").expanduser().resolve() + + +@pytest.mark.parametrize("nas_name", ["bandicoot", "koala"]) +def test_nas_path_check_falls_back_to_root_when_mount_missing( + tmp_path, monkeypatch, capsys, nas_name +): + monkeypatch.setattr(os.path, "ismount", lambda path: False) + + result = nas_path_check(root_dir=tmp_path, nas_name=nas_name) + + assert result == tmp_path + assert "Warning" in capsys.readouterr().out