From aa7b749b3e55cf15c7ac639eac1bd19bd76b2da4 Mon Sep 17 00:00:00 2001 From: peter Date: Wed, 2 Sep 2026 11:14:46 -0700 Subject: [PATCH 1/2] trait tests in python use dev pyslim --- .github/workflows/tests.yml | 76 ++---- treerec/tests/README.md | 3 + treerec/tests/conftest.py | 7 +- treerec/tests/init.slim | 13 + treerec/tests/pyproject.toml | 6 +- treerec/tests/recipe_specs.py | 15 ++ treerec/tests/test_recipes/recipe_WF.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_H.slim | 23 ++ treerec/tests/test_recipes/recipe_WF_HF.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_HM.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_W.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_X.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_Y.slim | 24 ++ treerec/tests/test_recipes/recipe_WF_Z.slim | 24 ++ .../tests/test_recipes/recipe_WF_adds.slim | 32 +++ .../recipe_WF_no_substitutions.slim | 25 ++ treerec/tests/test_recipes/recipe_nonWF.slim | 34 +++ .../tests/test_recipes/recipe_nonWF_H.slim | 34 +++ .../tests/test_recipes/recipe_nonWF_X.slim | 35 +++ .../tests/test_recipes/recipe_nonWF_Y.slim | 35 +++ .../recipe_nonWF_substitutions.slim | 36 +++ treerec/tests/test_trait_consistency.py | 235 ++++++++++++++++++ 22 files changed, 718 insertions(+), 59 deletions(-) create mode 100644 treerec/tests/test_recipes/recipe_WF.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_H.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_HF.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_HM.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_W.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_X.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_Y.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_Z.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_adds.slim create mode 100644 treerec/tests/test_recipes/recipe_WF_no_substitutions.slim create mode 100644 treerec/tests/test_recipes/recipe_nonWF.slim create mode 100644 treerec/tests/test_recipes/recipe_nonWF_H.slim create mode 100644 treerec/tests/test_recipes/recipe_nonWF_X.slim create mode 100644 treerec/tests/test_recipes/recipe_nonWF_Y.slim create mode 100644 treerec/tests/test_recipes/recipe_nonWF_substitutions.slim create mode 100644 treerec/tests/test_trait_consistency.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 177dcc99a..eeb1c9b94 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,37 +62,24 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Setup Miniforge - uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + + - name: Install uv and set the python version + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - miniforge-version: latest - activate-environment: anaconda-client-env python-version: ${{ matrix.python }} + version: "0.10.0" + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT shell: bash - - name: Cache Conda env - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ env.CONDA }}/envs - key: - conda-${{ runner.os }}--python-${{ matrix.python }}--${{ runner.arch }}--${{ - steps.get-date.outputs.today }}-${{ - hashFiles('treerec/tests/environment.yml') }}-${{ env.CACHE_NUMBER}} - env: - # Increase this value to reset cache if - # treerec/tests/environment.yml has not changed - CACHE_NUMBER: 0 - id: cache - - name: Update environment - run: conda env update -n anaconda-client-env -f treerec/tests/environment.yml - if: steps.cache.outputs.cache-hit != 'true' + - name: Workaround for gcc-11 if: startsWith(matrix.os, 'ubuntu') && matrix.gcc == 11 run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ sudo apt-get update -y + - name: Set up default GCC (in Ubuntu) if: startsWith(matrix.os, 'ubuntu') run: | @@ -100,6 +87,7 @@ jobs: sudo update-alternatives \ --install /usr/bin/gcc gcc /usr/bin/gcc-${{ matrix.gcc }} 100 \ --slave /usr/bin/g++ g++ /usr/bin/g++-${{ matrix.gcc }} + - name: Build and test (Debug) run: | mkdir Debug && \ @@ -108,6 +96,7 @@ jobs: make -j 2 && \ # Show any output from the test program whenever the test fails env CTEST_OUTPUT_ON_FAILURE=1 make test + - name: Build and test (Release) run: | mkdir Release && \ @@ -116,13 +105,13 @@ jobs: make -j 2 && \ # Show any output from the test program whenever the test fails env CTEST_OUTPUT_ON_FAILURE=1 make test + - name: Treesequence tests shell: bash -el {0} run: | - conda activate anaconda-client-env && \ export PATH=$PATH:$PWD/Release && \ echo $PATH && \ - cd treerec/tests && python -m pytest -xv + cd treerec/tests && uv run pytest -xv tests-Windows-CLI: if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'messerlab/slim') @@ -137,8 +126,10 @@ jobs: run: shell: msys2 {0} steps: + - name: Check out repository code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup MSYS2 ${{matrix.sys}} uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: @@ -150,40 +141,18 @@ jobs: msys2-devel mingw-w64-${{matrix.env}}-toolchain mingw-w64-${{matrix.env}}-cmake - - name: Setup Miniforge - uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 - with: - miniforge-version: latest - activate-environment: anaconda-client-env - auto-update-conda: true - python-version: ${{ matrix.python }} + - name: Get Date id: get-date run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT shell: bash - - name: Cache Conda env - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - # Increase this value to reset cache if treerec/tests/environment.yml has not changed - CACHE_NUMBER: 0 - with: - # Use faster GNU tar - enableCrossOsArchive: true - path: D:\conda_pkgs_dir - key: - conda-${{ runner.os }}--python-${{ matrix.python }}--${{ runner.arch }}--${{ - steps.get-date.outputs.today }}-${{ - hashFiles('treerec/tests/environment.yml') }}-${{ env.CACHE_NUMBER}} - id: cache - - uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + + - name: Install uv and set the python version + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - activate-environment: anaconda-client-env - environment-file: treerec/tests/environment.yml - pkgs-dirs: D:\conda_pkgs_dir - - name: Update environment - shell: bash -el {0} - run: conda env update -n anaconda-client-env -f treerec/tests/environment.yml - if: steps.cache.outputs.cache-hit != 'true' + python-version: ${{ matrix.python }} + version: "0.10.0" + - name: Build and test (Debug) run: | cd windows_compat/gnulib && \ @@ -195,6 +164,7 @@ jobs: make -j 2 && \ # Show any output from the test program whenever the test fails env CTEST_OUTPUT_ON_FAILURE=1 make test + - name: Build and test (Release) run: | cd windows_compat/gnulib && \ @@ -206,13 +176,13 @@ jobs: make -j 2 && \ # Show any output from the test program whenever the test fails env CTEST_OUTPUT_ON_FAILURE=1 make test + - name: Treesequence tests shell: bash -el {0} run: | - conda activate anaconda-client-env && \ export PATH=$PATH:$PWD/Release && \ echo $PATH && \ - cd treerec/tests && python -m pytest -xv + cd treerec/tests && uv run pytest -xv tests-Unix-GUI: if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'messerlab/slim') diff --git a/treerec/tests/README.md b/treerec/tests/README.md index ad71e4862..59078dd95 100644 --- a/treerec/tests/README.md +++ b/treerec/tests/README.md @@ -72,5 +72,8 @@ To add a new test: add individuals to the saved list by calling `addIndividuals(individuals)` at any point in the simulation. + * Ensure that tree sequences are written to subdirectories! If there are no subdirectories + then nothing will happen. + To temporarily turn off a test just comment out the appropriate line in `recipe_specs.py` diff --git a/treerec/tests/conftest.py b/treerec/tests/conftest.py index ffb08b974..a70c1764b 100644 --- a/treerec/tests/conftest.py +++ b/treerec/tests/conftest.py @@ -40,12 +40,11 @@ def get_ts(self): # binary print("******** Binary input.") bin_ts = load_file_or_dir(os.path.join(self.dir, "test_output.trees")) - yield bin_ts # and nonsimplified binary print("******** Unsimplified binary.") bin_nonsimp_ts = load_file_or_dir( os.path.join(self.dir, "test_output.unsimplified.trees")) - yield bin_nonsimp_ts + return (bin_ts, bin_nonsimp_ts) @staticmethod def get_slim_ids(ts): @@ -154,11 +153,13 @@ def make_result(run_dir): SLiM recipes are expected to output their results into a set of directories within the run_dir: return a list of results, one for each subdirectory """ - return [ + out = [ OutputResult(os.path.join(run_dir, f.name)) for f in os.scandir(run_dir) if f.is_dir() ] + assert len(out) > 0 + return out def run_slim(recipe, run_dir, recipe_dir="test_recipes"): """ diff --git a/treerec/tests/init.slim b/treerec/tests/init.slim index 52fa390ee..c84d42ec0 100644 --- a/treerec/tests/init.slim +++ b/treerec/tests/init.slim @@ -30,6 +30,19 @@ function (void)initializeMarks(i$ num_marks) return; } +function (void)saveTreeSeqTraits(void) { + MD = Dictionary(); + for (mt in sim.mutationTypes) { + MD.setValue("m" + mt.id, Dictionary( + "convertToSubstitution", mt.convertToSubstitution + )); + } + sim.demandPhenotype(sim.subpopulations); + setwdForGen(); + sim.treeSeqOutput("test_output.trees", metadata=Dictionary("mutation_types", MD)); + sim.treeSeqOutput("test_output.unsimplified.trees", simplify=F); +} + function (void)saveTreeSeq(void) { // create tree seq representation in various formats: this could overwrite any // existing ones output in the same cycle, but they should be identical diff --git a/treerec/tests/pyproject.toml b/treerec/tests/pyproject.toml index ed2fd28ab..67954ac9d 100644 --- a/treerec/tests/pyproject.toml +++ b/treerec/tests/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "SLiM_python_tests" version = "0.1.0" -requires-python = ">=3.13" +requires-python = ">=3.11" dependencies = [ "filelock>=3.32.4", - "msprime>=1.2.0", + "msprime>=1.4.4", "numpy>=2.5.2", - "pyslim>=1.0.4", + "pyslim @ git+https://github.com/tskit-dev/pyslim@main", "pytest>=9.1.1", "pytest-xdist>=3.8.0", "setuptools>=84.0.0", diff --git a/treerec/tests/recipe_specs.py b/treerec/tests/recipe_specs.py index 6b1e3a7a3..5d7d32bd1 100644 --- a/treerec/tests/recipe_specs.py +++ b/treerec/tests/recipe_specs.py @@ -50,6 +50,21 @@ "test_836_output from multiple-chromosome models.slim": {"mutations": True, "individuals": True}, "test_169_complex multi-chromosome inheritance.slim": {"mutations": True, "individuals": True}, "test_1610_modeling pseudo-autosomal regions.slim": {"mutations": True, "individuals": True}, + "recipe_WF.slim": {"traits": True}, + "recipe_WF_adds.slim": {"traits": True}, + "recipe_WF_no_substitutions.slim": {"traits": True}, + "recipe_WF_H.slim": {"traits": True}, + "recipe_WF_HF.slim": {"traits": True}, + "recipe_WF_HM.slim": {"traits": True}, + "recipe_WF_W.slim": {"traits": True}, + "recipe_WF_X.slim": {"traits": True}, + "recipe_WF_Y.slim": {"traits": True}, + "recipe_WF_Z.slim": {"traits": True}, + "recipe_nonWF.slim": {"traits": True}, + "recipe_nonWF_substitutions.slim": {"traits": True}, + "recipe_nonWF_H.slim": {"traits": True}, + "recipe_nonWF_X.slim": {"traits": True}, + "recipe_nonWF_Y.slim": {"traits": True}, } diff --git a/treerec/tests/test_recipes/recipe_WF.slim b/treerec/tests/test_recipes/recipe_WF.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_H.slim b/treerec/tests/test_recipes/recipe_WF_H.slim new file mode 100644 index 000000000..a73464c64 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_H.slim @@ -0,0 +1,23 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_HF.slim b/treerec/tests/test_recipes/recipe_WF_HF.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_HF.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_HM.slim b/treerec/tests/test_recipes/recipe_WF_HM.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_HM.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_W.slim b/treerec/tests/test_recipes/recipe_WF_W.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_W.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_X.slim b/treerec/tests/test_recipes/recipe_WF_X.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_X.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_Y.slim b/treerec/tests/test_recipes/recipe_WF_Y.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_Y.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_Z.slim b/treerec/tests/test_recipes/recipe_WF_Z.slim new file mode 100644 index 000000000..0471c8916 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_Z.slim @@ -0,0 +1,24 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_adds.slim b/treerec/tests/test_recipes/recipe_WF_adds.slim new file mode 100644 index 000000000..b17598909 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_adds.slim @@ -0,0 +1,32 @@ +initialize() +{ + setSeed(37); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + defineConstant("L", 1); + initializeChromosome(1, L, "A", "A"); + initializeMutationRate(0.99); + initializeMutationType("m1", NAN, "f", 0.01); + initializeMutationType("m2", NAN, "f", 0.01); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, L-1); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 7); +} + +1 late() { + hap = sample(p1.haplosomes, 1); + loc = sample(0:(L-1), 1); + hap.addNewDrawnMutation(m2, loc); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_WF_no_substitutions.slim b/treerec/tests/test_recipes/recipe_WF_no_substitutions.slim new file mode 100644 index 000000000..2138f6146 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_WF_no_substitutions.slim @@ -0,0 +1,25 @@ +initialize() +{ + setSeed(23); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + m1.convertToSubstitution = F; + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +30 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_nonWF.slim b/treerec/tests/test_recipes/recipe_nonWF.slim new file mode 100644 index 000000000..e6693ea6c --- /dev/null +++ b/treerec/tests/test_recipes/recipe_nonWF.slim @@ -0,0 +1,34 @@ +initialize() +{ + setSeed(23); + initializeSLiMModelType("nonWF"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(); + source("init.slim"); + initializeChromosome(1, 1000, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", 0.001); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 999); + initializeRecombinationRate(1e-2); + defineConstant("K", 10); +} + +reproduction(NULL) { + mate = subpop.sampleIndividuals(1); + subpop.addCrossed(individual, mate); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + p1.fitnessScaling = K / p1.individualCount; +} + +100 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_nonWF_H.slim b/treerec/tests/test_recipes/recipe_nonWF_H.slim new file mode 100644 index 000000000..bb9ea23c5 --- /dev/null +++ b/treerec/tests/test_recipes/recipe_nonWF_H.slim @@ -0,0 +1,34 @@ +initialize() +{ + setSeed(23); + initializeSLiMModelType("nonWF"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(); + source("init.slim"); + initializeChromosome(1, 100, "H", "H"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.001); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); + defineConstant("K", 10); +} + +reproduction() { + mate = subpop.sampleIndividuals(1); + subpop.addCrossed(individual, mate); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + p1.fitnessScaling = K / p1.individualCount; +} + +100 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_nonWF_X.slim b/treerec/tests/test_recipes/recipe_nonWF_X.slim new file mode 100644 index 000000000..2b488b5ba --- /dev/null +++ b/treerec/tests/test_recipes/recipe_nonWF_X.slim @@ -0,0 +1,35 @@ +initialize() +{ + setSeed(23); + initializeSLiMModelType("nonWF"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "X", "X"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.001); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); + defineConstant("K", 10); +} + +reproduction(NULL, "F") { + mate = subpop.sampleIndividuals(1, sex="M"); + subpop.addCrossed(individual, mate); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + p1.fitnessScaling = K / p1.individualCount; +} + +100 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_nonWF_Y.slim b/treerec/tests/test_recipes/recipe_nonWF_Y.slim new file mode 100644 index 000000000..2567b6c2f --- /dev/null +++ b/treerec/tests/test_recipes/recipe_nonWF_Y.slim @@ -0,0 +1,35 @@ +initialize() +{ + setSeed(23); + initializeSLiMModelType("nonWF"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "Y", "Y"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.001); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); + defineConstant("K", 20); +} + +reproduction(NULL, "F") { + mate = subpop.sampleIndividuals(1, sex="M"); + subpop.addCrossed(individual, mate); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + p1.fitnessScaling = K / p1.individualCount; +} + +100 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_recipes/recipe_nonWF_substitutions.slim b/treerec/tests/test_recipes/recipe_nonWF_substitutions.slim new file mode 100644 index 000000000..3827044fc --- /dev/null +++ b/treerec/tests/test_recipes/recipe_nonWF_substitutions.slim @@ -0,0 +1,36 @@ +initialize() +{ + setSeed(23); + initializeSLiMModelType("nonWF"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(); + source("init.slim"); + initializeSex(); + initializeChromosome(1, 100, "A", "A"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.001); + m1.convertToSubstitution = T; + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); + defineConstant("K", 10); +} + +reproduction(NULL, "F") { + mate = subpop.sampleIndividuals(1, sex="M"); + subpop.addCrossed(individual, mate); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + p1.fitnessScaling = K / p1.individualCount; +} + +100 late() { + saveTreeSeqTraits(); + catn("Done."); + sim.simulationFinished(); +} diff --git a/treerec/tests/test_trait_consistency.py b/treerec/tests/test_trait_consistency.py new file mode 100644 index 000000000..b6008710f --- /dev/null +++ b/treerec/tests/test_trait_consistency.py @@ -0,0 +1,235 @@ +import tskit, pyslim +import numpy as np +import pytest +from collections import Counter + +from recipe_specs import recipe_eq + +class TraitCalculator: + """ + Calculating phenotypes by hand from metadata takes a good bit of + internal state, especially if it's from a multi-chromosome simulation, + so here's a class that does it. See below for how to use this. + """ + + def __init__(self): + self.ts_metadata = None + self.slim_phenotypes = None + self.phenotypes = None + self.offsets = None + + def add_ts(self, ts): + ts_metadata = ts.metadata + if self.ts_metadata is not None: + assert ts_metadata["SLiM"]["traits"] == self.ts_metadata["SLiM"]["traits"] + self.ts_metadata = ts_metadata + self.ts = ts + chrom_type = self.ts_metadata["SLiM"]["this_chromosome"]["type"] + self.haploid_chromosome = chrom_type in ("H", "HF", "HM") + self.mut_metadata = pyslim.mutation_metadata(ts) + self.nodes_vacant = pyslim.nodes_vacant(ts) + self.sexes = self.ts.tables.individuals.metadata_vector("sex") + self.num_traits = len(self.ts_metadata["SLiM"]["traits"]) + self.types = [m["type"] for m in self.ts_metadata["SLiM"]["traits"]] + self.accumulates = [ + m["substitutionAccumulation"] for m in self.ts_metadata["SLiM"]["traits"] + ] + # to properly do traits in the case where substitutionAccumulation=F + # and there are mutation types with convertToSubstitution=T then we + # need to store this in metadata: (so, this code won't work if this is + # not present for such traits, which are probably only the default trait!) + if ( + "user_metadata" in ts_metadata["SLiM"] + and "mutation_types" in ts_metadata["SLiM"]["user_metadata"] + ): + self.mutation_types = { + k: v[0]["convertToSubstitution"][0] + for k, v in ts_metadata["SLiM"]["user_metadata"]["mutation_types"][ + 0 + ].items() + } + else: + # guess T if there just a single trait without substitution accumulation + # whose name is simT and this is a WF model + # since that describes "default trait" + convert = ( + self.ts_metadata["SLiM"]["model_type"] == "WF" + and len(self.ts_metadata["SLiM"]["traits"]) == 1 + and self.ts_metadata["SLiM"]["traits"][0]["name"] == "simT" + and self.ts_metadata["SLiM"]["traits"][0]["substitutionAccumulation"] + == False + ) + mtypes = set(md["mutation_type"] for md in self.mut_metadata.values()) + self.mutation_types = {f"m{k}": convert for k in mtypes} + sp = np.column_stack( + [ + self.ts.tables.individuals.metadata_vector(["per_trait", j, "phenotype"]) + for j in range(self.num_traits) + ] + ) + if self.slim_phenotypes is None: + self.slim_phenotypes = sp + assert np.all(self.slim_phenotypes == sp) + if self.phenotypes is None: + # initialize phenotypes where we'll accumulate changes + self.phenotypes = np.full( + self.slim_phenotypes.shape, + [1.0 if t == "multiplicative" else 0.0 for t in self.types], + dtype="float", + ) + offsets = np.column_stack( + [ + self.ts.tables.individuals.metadata_vector(["per_trait", j, "offset"]) + for j in range(self.num_traits) + ] + ) + if self.offsets is None: + self.offsets = offsets + # only do offsets if this is the first ts! + self.do_offsets() + else: + assert np.all(offsets == self.offsets) + self.get_frequencies() + self.do_effects() + + def transform(self): + for k, t in enumerate(self.types): + if t == "logistic": + self.phenotypes[:, k] = 1 / (1 + np.exp(-self.phenotypes[:, k])) + + def is_converted(self, a): + # is allele a from a convert-to-substitution mutation type? + # if we haven't recorded whether or not it's converted, assume T + md = self.mut_metadata[int(a)] + return self.mutation_types[f"m{md['mutation_type']}"] + + def get_frequencies(self): + # Fixed mutations count unless they are converted to substitutions + # and substitution accumulation is off, so we need to look up both these things: + # (frequency, whether they're converted to substitutions) for each mutation. + self.frequencies = {} + for v in self.ts.variants(isolated_as_missing=False): + freqs = {} + for ds, f in v.counts().items(): + for a in ds.split(","): + if a != "": + if a not in freqs: + convert = self.is_converted(a) + freqs[a] = [0, convert] + freqs[a][0] += f + self.frequencies[v.site.id] = freqs + + def do_offsets(self): + okeys = { + pyslim.INDIVIDUAL_TYPE_HERMAPHRODITE: "baselineOffsetH", + pyslim.INDIVIDUAL_TYPE_FEMALE: "baselineOffsetF", + pyslim.INDIVIDUAL_TYPE_MALE: "baselineOffsetM", + } + for k, t in enumerate(self.types): + bO = np.array( + [self.ts_metadata["SLiM"]["traits"][k][okeys[s]] for s in self.sexes] + ) + if t == "multiplicative": + self.phenotypes[:, k] *= bO * self.offsets[:, k] + else: + self.phenotypes[:, k] += bO + self.offsets[:, k] + + def do_effects(self): + for ind in self.ts.individuals(): + ploidy = np.sum(~self.nodes_vacant[ind.nodes]) + if ploidy == 0: + continue + else: + hemizygous = ploidy == 1 + for k, t in enumerate(self.types): + if t == "multiplicative": + g = self.multiplicative_effect(ind, k, hemizygous) + self.phenotypes[ind.id, k] *= g + else: + g = self.additive_effect(ind, k, hemizygous) + self.phenotypes[ind.id, k] += g + + def skip_mut(self, m, freqs, trait_id): + if m == "": + out = True + else: + f, convert = freqs[m] + out = f == 0 or ( + f == self.ts.num_samples and convert and not self.accumulates[trait_id] + ) + return out + + def additive_effect(self, ind, trait_id, hemizygous): + out = 0.0 + for v in self.ts.variants(samples=ind.nodes, isolated_as_missing=False): + freqs = self.frequencies[v.site.id] + a = ",".join([v.alleles[g] for g in v.genotypes]) + muts = Counter(a.split(",")) + for m in muts: + if self.skip_mut(m, freqs, trait_id): + continue + md = self.mut_metadata[int(m)]["per_trait"][trait_id] + s = md["effect_size"] + if muts[m] == 2: + h = 1 + else: + assert muts[m] == 1 + if self.haploid_chromosome: + h = 1 + else: + if hemizygous: + h = md["hemizygous_dominance"] + else: + h = md["dominance"] + if np.isnan(h): + h = 1 / 2 + out += 2 * h * s + return out + + def multiplicative_effect(self, ind, trait_id, hemizygous): + out = 1.0 + for v in self.ts.variants(samples=ind.nodes, isolated_as_missing=False): + freqs = self.frequencies[v.site.id] + a = ",".join([v.alleles[g] for g in v.genotypes]) + muts = Counter(a.split(",")) + for m in muts: + if self.skip_mut(m, freqs, trait_id): + continue + assert muts[m] > 0 and muts[m] <= 2 + md = self.mut_metadata[int(m)]["per_trait"][trait_id] + s = md["effect_size"] + if muts[m] == 2: + h = 1 + else: + assert muts[m] == 1 + if self.haploid_chromosome: + h = 1 + elif hemizygous: + h = md["hemizygous_dominance"] + else: + h = md["dominance"] + if np.isnan(h): + # "independent dominance occurs when (1+hs)(1+hs) equals 1+s, + # which occurs when h=(sqrt(1+s)−1)/s" + h = (np.sqrt(1 + s) - 1) / s if s != 0 else 0 + out *= 1 + h * s + return out + + +class TestTraits: + + @pytest.mark.parametrize("recipe", recipe_eq("traits"), indirect=True) + def test_traits_consistency(self, recipe): + assert len(recipe["results"]) > 0 + for result in recipe["results"]: + # note we don't have multi-chromosome sims here! + # see pyslim tests for that setup + ts = result.get_normal_ts() + assert ts.num_mutations > 0 + assert ts.num_individuals > 0 + # phenotypes as computed by us + tc = TraitCalculator() + tc.add_ts(ts) + tc.transform() + assert np.allclose(tc.phenotypes, tc.slim_phenotypes) + From 577e2ac3986f3ff8df028257d44ba7bb703626fa Mon Sep 17 00:00:00 2001 From: peter Date: Sun, 13 Sep 2026 08:14:02 -0700 Subject: [PATCH 2/2] . --- treerec/tests/pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/treerec/tests/pyproject.toml b/treerec/tests/pyproject.toml index 67954ac9d..6d2587dbc 100644 --- a/treerec/tests/pyproject.toml +++ b/treerec/tests/pyproject.toml @@ -3,12 +3,12 @@ name = "SLiM_python_tests" version = "0.1.0" requires-python = ">=3.11" dependencies = [ - "filelock>=3.32.4", + "filelock", "msprime>=1.4.4", - "numpy>=2.5.2", + "numpy", "pyslim @ git+https://github.com/tskit-dev/pyslim@main", - "pytest>=9.1.1", - "pytest-xdist>=3.8.0", - "setuptools>=84.0.0", + "pytest", + "pytest-xdist", + "setuptools", "tskit>=0.4.1", ]