From 23e5133c353c00f39257b853f5c9a4a69d039009 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 18 Jan 2023 10:27:35 -0800 Subject: [PATCH] update for lm-eval==0.3.0 (#2713) Co-authored-by: Jeff Rasley --- .github/workflows/nv-nightly.yml | 6 ------ requirements/requirements-inf.txt | 2 +- tests/unit/inference/test_inference.py | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nv-nightly.yml b/.github/workflows/nv-nightly.yml index 4229bee3..04d545da 100644 --- a/.github/workflows/nv-nightly.yml +++ b/.github/workflows/nv-nightly.yml @@ -38,12 +38,6 @@ jobs: pip install .[dev,1bit,autotuning,inf] ds_report - - name: Install lm-eval - run: | - pip install git+https://github.com/EleutherAI/lm-evaluation-harness - # This is required until lm-eval makes a new release. v0.2.0 is - # broken for latest version of transformers - - name: Python environment run: | pip list diff --git a/requirements/requirements-inf.txt b/requirements/requirements-inf.txt index afaa0e40..848a7f7a 100644 --- a/requirements/requirements-inf.txt +++ b/requirements/requirements-inf.txt @@ -1,5 +1,5 @@ google -lm-eval>=0.2.0 +lm-eval==0.3.0 protobuf transformers transformers[sentencepiece] diff --git a/tests/unit/inference/test_inference.py b/tests/unit/inference/test_inference.py index e6415cb1..3bfef205 100644 --- a/tests/unit/inference/test_inference.py +++ b/tests/unit/inference/test_inference.py @@ -427,7 +427,7 @@ class TestInjectionPolicy(DistributedTest): "gpt2-xl"], ), ) -@pytest.mark.parametrize("task", ["lambada"]) +@pytest.mark.parametrize("task", ["lambada_standard"]) class TestLMCorrectness(DistributedTest): world_size = 1 -- GitLab