diff --git a/.github/workflows/nv-nightly.yml b/.github/workflows/nv-nightly.yml
index 4229bee32893ec2b2f11b523891d4739c8dc7c9b..04d545dadd6bd5c2aceb65e5d48bead1ecc51886 100644
--- a/.github/workflows/nv-nightly.yml
+++ b/.github/workflows/nv-nightly.yml
@@ -38,12 +38,6 @@ jobs:
           pip install .[dev,1bit,autotuning,inf]
           ds_report
 
-      - name: Install lm-eval
-        run: |
-          pip install git+https://github.com/EleutherAI/lm-evaluation-harness
-          # This is required until lm-eval makes a new release. v0.2.0 is
-          # broken for latest version of transformers
-
       - name: Python environment
         run: |
           pip list
diff --git a/requirements/requirements-inf.txt b/requirements/requirements-inf.txt
index afaa0e4080738c6dca50fb62e85f7bbf32d1f2d3..848a7f7a485de46d9941b3de6919b42724f76409 100644
--- a/requirements/requirements-inf.txt
+++ b/requirements/requirements-inf.txt
@@ -1,5 +1,5 @@
 google
-lm-eval>=0.2.0
+lm-eval==0.3.0
 protobuf
 transformers
 transformers[sentencepiece]
diff --git a/tests/unit/inference/test_inference.py b/tests/unit/inference/test_inference.py
index e6415cb163495f4e41bd14fee23d62244855642a..3bfef2051c39a339c54e885aaee4e78d31f36b4b 100644
--- a/tests/unit/inference/test_inference.py
+++ b/tests/unit/inference/test_inference.py
@@ -427,7 +427,7 @@ class TestInjectionPolicy(DistributedTest):
          "gpt2-xl"],
     ),
 )
-@pytest.mark.parametrize("task", ["lambada"])
+@pytest.mark.parametrize("task", ["lambada_standard"])
 class TestLMCorrectness(DistributedTest):
     world_size = 1