Add Micropolis world generation and evaluation workflows

README.md

@@ -63,3 +63,7 @@ Both worlds resolve through the *same* core resolver and registry — that's the
 ## Offline cached-forecast interfaces
 
 The additive [fbsim-benchmark package](packages/fbsim-benchmark/README.md) provides installed-package examples and synthetic fixture tests for FreeCiv, Micropolis and Starsim without running engines or calling models. It depends on the existing core; world and human-subject workflows remain in place. See its [workflow map](packages/fbsim-benchmark/docs/WORKFLOWS.md) for retained, migrated and legacy routes.
+
+### Micropolis world
+
+[Micropolis setup and workflows](worlds/micropolis/README.md) cover the external engine, question generation, cached/model gathering and raw scoring. Synthetic offline tests do not validate a live engine or reproduce paper results.

worlds/micropolis/.gitignore

@@ -0,0 +1,7 @@
+.env
+.venv*
+data/
+__pycache__/
+*.egg-info/
+build/
+.pytest_cache/

worlds/micropolis/AGENTS.md

@@ -0,0 +1,5 @@
+Read the repository CLAUDE.md and this world README.md. Use uv run.
+Preserve Fabio Rocha's source attribution and mathematical/parser contracts.
+Tests must use synthetic cached worlds and blocked providers; no production runs.
+Paper reports, normalization, administered prompts/configurations and data remain separate.
+MICROPOLIS_CORE_PATH is an external dependency, never vendor an engine implicitly.

worlds/micropolis/PROVENANCE.json

@@ -0,0 +1,203 @@
+{
+  "source_repository": "https://github.com/fdrocha/forecastbench-sim",
+  "source_commit": "8c70972b698f079d54b8a2255c81fbefc55e7d99",
+  "author": "Fabio Rocha and contributors",
+  "license": "GPL-3.0 (source repository LICENSE; copied here)",
+  "files": [
+    {
+      "path": "pyproject.toml",
+      "source_sha256": "bdaef90ed5934654399bad186d5655103dcd59f3f90c2994ed714ae130b30f38"
+    },
+    {
+      "path": "micropolis_world/__init__.py",
+      "source_sha256": "7aadf72c57a9188addb4193d272debfb7b266fce2cd31b72eeb95ca9775639fb"
+    },
+    {
+      "path": "micropolis_world/binary_eval.py",
+      "source_sha256": "cf409f3d39645f9ae16de5a459eeb3c10be83ac59d1929d09007ead579cb00d8"
+    },
+    {
+      "path": "micropolis_world/binary_questions.py",
+      "source_sha256": "f7e693ebfd3b98bcc6716cc0e6595713f6ff142df8248f86258636b785558b8e"
+    },
+    {
+      "path": "micropolis_world/city_sim.py",
+      "source_sha256": "12d98e07eba382e123d479c6809104452bd489ddcaf0ea549d38d95e31199d75"
+    },
+    {
+      "path": "micropolis_world/config.py",
+      "source_sha256": "4055ab7664503e4a353d650de928dffaca85d2dbe5d46da061ecb82073481073"
+    },
+    {
+      "path": "micropolis_world/continuous_eval.py",
+      "source_sha256": "70b41e5227b1e8dade6565c973c821efb61901091209b32d143607f457f5daa4"
+    },
+    {
+      "path": "micropolis_world/gather.py",
+      "source_sha256": "9ada49b2d03ae59f00c2ae6e390ed1b98b62525d500b35df8791cba5b7d9a307"
+    },
+    {
+      "path": "micropolis_world/ground_truth.py",
+      "source_sha256": "b8c98a68a35e63f0c9594c2a11e22c82acbd00ff4bf230d589e117cd23db5c4e"
+    },
+    {
+      "path": "micropolis_world/llm_backend.py",
+      "source_sha256": "496235292c859be15f05d5ed834cd3d025d7e3a2bbc0510da26f145cbaf07940"
+    },
+    {
+      "path": "micropolis_world/messages.py",
+      "source_sha256": "7a9c7df336f069e42bb0cad6f06024a207e1683fd84ce6acb3c9eb0b0dd2dbe8"
+    },
+    {
+      "path": "micropolis_world/model_ids.py",
+      "source_sha256": "02eb778c7b4a720ce85ec7b93ee80e3ff36955f4bd20a005102c37cd8dc728b8"
+    },
+    {
+      "path": "micropolis_world/module_globals.py",
+      "source_sha256": "9aeb37a1a9841682bd1646d260fd841b48e0872c9f86adf95c4990b6266b6f3a"
+    },
+    {
+      "path": "micropolis_world/openrouter_completion.py",
+      "source_sha256": "73dbe773c323792aca0a26558f5a87c6ca8dd4d7fb50d703620d4102d81719d4"
+    },
+    {
+      "path": "micropolis_world/plot_sim.py",
+      "source_sha256": "c124911f9bbeaec06f7214debe90314f5492f1c006283fffbe4fd83a8aa0775c"
+    },
+    {
+      "path": "micropolis_world/prompting.py",
+      "source_sha256": "1689b6b6edf4b3487566718880bae565b47f4d4f65b15de70809b040399b9901"
+    },
+    {
+      "path": "micropolis_world/report.py",
+      "source_sha256": "b3459d262c284618cd6145e6a74380dc21c70eb85a52046b99c4426dd50b8abb"
+    },
+    {
+      "path": "micropolis_world/scenarios.py",
+      "source_sha256": "6fc75760fa0f7ba562f451164f8318163c8404feacd0b36e1b55582ba645e90a"
+    },
+    {
+      "path": "micropolis_world/templates.py",
+      "source_sha256": "1bf4edcecae573a262ec3c5f181b512514b1670b9df8a2c65ca113c8fe527089"
+    },
+    {
+      "path": "micropolis_world/usage.py",
+      "source_sha256": "6288a2ca387a4f689a95f43d6e5763b9c07f98f9827c1bf0c65056057be02390"
+    },
+    {
+      "path": "scripts/run_sim.py",
+      "source_sha256": "562d2742caa8335114a683faa2fd1953762af6ee7e59dd5d010c4c8154cfa67d"
+    },
+    {
+      "path": "scripts/gen_corpus.py",
+      "source_sha256": "31e728d8992032f4ceb99ff38ac63a330cf44dcd4e3254fcdf5eca59c5d077cf"
+    },
+    {
+      "path": "scripts/gen_report.py",
+      "source_sha256": "2100df8afc751d7b27cdeecf4241a196d48776ddb585a99df1af26a424aab098"
+    },
+    {
+      "path": "scripts/generate_prompt.py",
+      "source_sha256": "a9cb81d5056e9ac4d9f34010c5228fa63f20c51ed1674f66850a192350fba640"
+    },
+    {
+      "path": "scripts/run_eval_binary.py",
+      "source_sha256": "0b25c1d36e47973018f2d1538a6009d723cd1e80bc8b0233ab6e29db3b34f472"
+    },
+    {
+      "path": "scripts/run_eval_continuous.py",
+      "source_sha256": "c4822d84c82225f692ed851b5e1c1003a0d527dd09087e1bda0a1e4ddb32a517"
+    },
+    {
+      "path": "scripts/extract_ground_truth.py",
+      "source_sha256": "953be01b90d29b1164ec2a189e779d70bae6abba6d6e5eff59f895b948aa8b8a"
+    },
+    {
+      "path": "scripts/check_determinism.py",
+      "source_sha256": "afce560f79d7b2a9c16c28fd834cf215918b0a2d5418e4ebe8b12d1da3f6e876"
+    },
+    {
+      "path": "tests/conftest.py",
+      "source_sha256": "416d14442e5b36383b50ccdf4ee9b62bcabd883e7b3924c07c2576a9395ca317"
+    },
+    {
+      "path": "tests/test_binary_questions.py",
+      "source_sha256": "74e2913f54183b4c197f4d0c4fbec49ff5e0ef5268d1da02dbe7e0195b17a26e"
+    },
+    {
+      "path": "tests/test_binary_batching.py",
+      "source_sha256": "34166e2d825d49360324499600e787b85f4a1e2b5fe807979957947a88775102"
+    },
+    {
+      "path": "tests/test_continuous_batching.py",
+      "source_sha256": "9e10df31705ef61124e30a537a7c3f80ea8f2a3f677b886ce3334553663a2be7"
+    },
+    {
+      "path": "tests/test_ground_truth.py",
+      "source_sha256": "6e189455c6f6af4c2c7c28a8cd606c2be92ceb3ccf44a19b129f1bef43fb1c92"
+    },
+    {
+      "path": "tests/test_messages.py",
+      "source_sha256": "2c8754f1d396ec3c59b40fcce7ed1ea08e470bb93e0dcf2a16d198b4aabbfda8"
+    },
+    {
+      "path": "tests/test_metric_ranges.py",
+      "source_sha256": "0b7c4925483a574b13e656e10c65fb318810d42ee72ded3b07214951350c7a20"
+    },
+    {
+      "path": "tests/test_model_ids.py",
+      "source_sha256": "bc4b51d231d54e925e4fbe1d9df57dfa45fc790581c09c200c09eabbb05e7c8e"
+    },
+    {
+      "path": "tests/test_openrouter_build.py",
+      "source_sha256": "61de9d4b7cc48fed73cffdd4e551fbb2dffe5072d3c00ef7f5fc3cfda7f6d518"
+    },
+    {
+      "path": "tests/test_plot_repeats.py",
+      "source_sha256": "cd588d74bc72f6ff819c080a41404257301d64f33949b588d73251259852e71b"
+    },
+    {
+      "path": "tests/test_prompting.py",
+      "source_sha256": "cf5bb7c5a928f5d7d0b24e4d1ec00ef8ee933ec1270f88caced814beb6ee94f8"
+    },
+    {
+      "path": "tests/test_report.py",
+      "source_sha256": "73e30d8fce3894bc48a296553ca9129aad56243f9ae03d3f37d7e3751478a8a8"
+    },
+    {
+      "path": "tests/test_select_incomplete.py",
+      "source_sha256": "856017c160b462509335941eb76b38b56656733b4b4af60ac5376ca78e2efcbd"
+    },
+    {
+      "path": "tests/test_usage.py",
+      "source_sha256": "3e39e90152e3614b9433fff6886e2d50396609c00cd2c165c549428f6bee0e74"
+    },
+    {
+      "path": "tests/test_determinism_check.py",
+      "source_sha256": "1bb898e179c7073d924d0920c1257f198b99bd728fc2450121cbb11512f9e253"
+    }
+  ],
+  "excluded": [
+    "production configs",
+    "administered prompt files and knowledge statements",
+    "model panel and routing specs",
+    "paper reporting/gathering scripts and fitted normalization constants",
+    "engine binaries/maps and raw datasets"
+  ],
+  "adaptations": [
+    "Lazy engine configuration and explicit credential validation; no cloud lookup",
+    "Canonical fbsim-core and fbsim-benchmark imports",
+    "Standalone cache digest instead of importing knowledge dataset",
+    "Synthetic example config and prompt wrappers",
+    "Dataset/gather APIs retained; paper normalization/report analysis excluded",
+    "Inherited engine-dependent tests converted to invented trajectories; real-response fixture replaced with invented text",
+    "Installed synthetic example and safe CLI diagnostics",
+    "Raw scoring command preserves realized Brier versus raw quantile CRPS distinction; no paper normalization"
+  ],
+  "engine_interface_reference": {
+    "repository": "https://github.com/fdrocha/MicropolisCore",
+    "branch": "exploration",
+    "commit": "4fe3a7238b735f9cdd3ec13c957ccc8a3a3b0481",
+    "validation": "source interface inspected only; no historical producer pin or live run established"
+  }
+}

worlds/micropolis/README.md

@@ -0,0 +1,74 @@
+# Micropolis world
+
+This world adds simulation, question generation, cached gathering and binary/continuous forecast datasets to the existing workspace. It derives from [Fabio Rocha's Micropolis branch](https://github.com/fdrocha/forecastbench-sim/tree/8c70972b698f079d54b8a2255c81fbefc55e7d99/worlds/micropolis), source commit `8c70972b698f079d54b8a2255c81fbefc55e7d99`. See [PROVENANCE.json](PROVENANCE.json) for source hashes and adaptations; the source repository's GPL-3.0 text is retained in [LICENSE](LICENSE).
+
+`fbsim-core` owns the shared schema/resolver. `fbsim-benchmark` owns the existing Micropolis parsers, raw replay scoring and cached trajectory converter; this world imports them. The five continuous quantiles are p10/p25/p50/p75/p90. Binary resolution uses half-open windows `(snapshot, resolution]` and engine turns (`tick // 16`). Original question/resolver bodies are retained. Their original specification remains at the pinned source's `binary_forecasts.md`.
+
+Paper normalization, model panels, administered prompt wrappers, knowledge statement sets, historical configurations, frozen data and paper CSV/figure producers are not part of this world. The included prompt wrappers and example configs are new illustrative examples; they cannot reproduce historical prompt hashes or results. Existing private paper work remains separate. No simulator, maps or engine binary is vendored.
+
+## Python installation and offline checks
+
+From the repository root, Python 3.11+ and uv are required. The workspace already discovers `worlds/*`. To install just core, benchmark and this world as non-editable wheels:
+
+```sh
+uv venv --python 3.13 .venv-micropolis
+uv run --no-project --python .venv-micropolis/bin/python packages/fbsim-benchmark/tools/build_wheels.py --output dist/micropolis-wheels
+uv build --wheel --no-sources --out-dir dist/micropolis-wheels worlds/micropolis
+uv pip install --python .venv-micropolis/bin/python --find-links dist/micropolis-wheels 'fbsim-core==0.1.0' 'fbsim-benchmark[test]==0.3.0' 'micropolis-world[dev]==0.1.0'
+uv pip check --python .venv-micropolis/bin/python
+```
+
+For cached offline installation, add `--offline` to all uv commands (including the build helper). An optional `--find-links /path/to/wheelhouse` on installation supplies locally available dependencies. Offline mode fails on missing dependencies; it does not download. These packages do not install the engine. Full `uv sync` additionally installs other worlds' dependencies.
+
+```sh
+LITELLM_LOCAL_MODEL_COST_MAP=True uv run --offline --no-project --python .venv-micropolis/bin/python -m pytest -q worlds/micropolis/tests
+uv run --offline --no-project --python .venv-micropolis/bin/python -m pytest -q --import-mode=importlib packages/fbsim-core/tests packages/fbsim-benchmark/tests
+uv run --offline --no-project --python .venv-micropolis/bin/python worlds/micropolis/scripts/run_sim.py --help
+```
+
+Tests block engine launches and provider transports. Corpus tests use invented trajectory rows; the process-contract test mocks pnpm. Neither is a live-engine validation. CLI `--help`, cached parsing/scoring and imports need no engine path or credentials. A missing engine is reported when a runner is invoked, rather than during import. No `.env` file or cloud secret store is loaded implicitly.
+
+## External engine setup
+
+Use [fdrocha/MicropolisCore, exploration](https://github.com/fdrocha/MicropolisCore/tree/4fe3a7238b735f9cdd3ec13c957ccc8a3a3b0481), pinned for this integration's **source-interface inspection** to `4fe3a7238b735f9cdd3ec13c957ccc8a3a3b0481`. It contains both `apps/micropolis/cli/run_sim.js` and `run_continuations.js`. The `fbs` branch at `d50b799d5915944263c0385fc43a8990e5d4e32e` has run_sim but lacks run_continuations; it is not a substitute for ground-truth extraction.
+
+Prerequisites in that engine revision: Node >=20, pnpm 10.28.0 (declared packageManager), make, and an activated Emscripten toolchain providing `em++`. The engine makefile uses Emscripten APIs including `--emit-tsd`; no exact tested Emscripten version or historical producer-engine pin was established in this port. Its source specifies `pnpm run build:engine` → `make install`. Setup below is for a separately provisioned engine checkout and may download/build substantial dependencies; it was **not executed by the offline integration tests**:
+
+```sh
+# In a separately obtained fdrocha/MicropolisCore checkout:
+git checkout 4fe3a7238b735f9cdd3ec13c957ccc8a3a3b0481
+pnpm install --frozen-lockfile
+pnpm run build:engine
+export MICROPOLIS_CORE_PATH="$PWD"
+```
+
+Return to forecastbench-sim before the following commands. `MICROPOLIS_CORE_PATH` names the engine root, not apps/micropolis. Engine assets and their licenses/additional terms remain in that external repository; this package grants no new rights to them. Live engine/toolchain compatibility remains a runtime gate, not a claim about historical reproducibility.
+
+## Entrypoints and effects
+
+All script paths are under `worlds/micropolis/scripts/`. Use `uv run --no-project --python .venv-micropolis/bin/python <script> ...` from the repository root. Each configuration-driven command accepts an explicit JSON5 file; continuous commands default to the invented `configs/example.json5` (six turns, horizon four). Binary evaluation and continuation extraction default to `configs/example-binary.json5` (97 logged turns, snapshot 48, horizon 48, one continuation seed), also shipped inside the installed package. Binary resolution requires a snapshot and horizon of at least 48 turns to include a yearly baseline and resolution checkpoint; the short continuous example is intentionally invalid for those two commands. These defaults are illustrative, not historical configurations. Replace `example/model` with a deliberately selected provider model before any paid evaluation.
+
+| Script | What it does |
+|---|---|
+| `run_sim.py CONFIG --no-plot` | Runs the external engine and writes log/event files; optional plot |
+| `gen_corpus.py CONFIG` | Runs/reuses city trajectories, generates and resolves continuous questions |
+| `gen_report.py CONFIG` | Produces a situation report; reads cached trajectory files |
+| `generate_prompt.py CONFIG` | Builds an example continuous prompt; may run simulations on cache misses |
+| `run_eval_binary.py CONFIG`, `run_eval_continuous.py CONFIG` | Builds corpus and gathers model responses, then writes the respective dataset |
+| Same eval scripts with `--cache-only` | No provider calls; reads existing raw-response cache, but corpus construction can still simulate |
+| Same eval scripts with `--dry-run` | Builds corpus without gathering; **can still simulate** |
+| `extract_ground_truth.py CONFIG --jobs 1` | Runs reseeded continuations through external `run_continuations.js`; not an offline smoke command |
+| `check_determinism.py CONFIG` | Repeats live engine runs; opt-in only |
+| `score_cached.py --input DATASET --kind binary\|continuous` | Reads gathered JSON and prints per-forecast realized Brier or raw quantile CRPS; no engine/provider/normalization |
+
+Choose a fresh output root before running:
+
+```sh
+export FBSIM_DATA_ROOT=/path/to/new/run-data
+uv run --no-project --python .venv-micropolis/bin/python worlds/micropolis/scripts/run_sim.py worlds/micropolis/configs/example.json5 --no-plot
+uv run --no-project --python .venv-micropolis/bin/python worlds/micropolis/scripts/gen_corpus.py worlds/micropolis/configs/example.json5
+```
+
+The config's `data_dir` selects one child of `FBSIM_DATA_ROOT`; absent the environment variable, the root is `./data` relative to the working directory. Cached gathering preserves the native content-addressed prompt/response scheme. Missing cached forecasts remain absent, unparsed forecasts remain null: the raw scorer does not apply paper eligibility or imputation rules.
+
+Actual model gathering requires explicitly exporting `OPENROUTER_API_KEY`. No GCP fallback is attempted. Optionally set `MICROPOLIS_MODEL_SPECS` to your own JSON5 routing/spec file before invoking Python; the installed registry is empty. An unqualified model uses provider defaults, while a suffixed slug must have a configured spec. No production model panel is supplied, and no model call was made to validate this integration.