diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8176be2683e8ddd7a2165592e900ed832275c99e..1cb9facbfc2a46712b2e0108ccbfbd6c8d5e3770 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -17,48 +17,31 @@ jobs:
           - '3.7'
           - '3.8'
           - '3.9'
+        numpy-version:
+          - '1.18.0'
+          - '1.20.0'
+          - '1.22.0'
         torch-version:
-          - '1.1.0'
           - '1.2.0'
-          - '1.3.0'
           - '1.4.0'
-          - '1.5.0'
           - '1.6.0'
-          - '1.7.0'
           - '1.8.0'
-          - '1.9.0'
           - '1.10.0'
         exclude:
-          - os: 'ubuntu-18.04'
-            python-version: '3.8'
-            torch-version: '1.1.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.8'
+          - python-version: '3.6'
+            numpy-version: '1.20.0'
+          - python-version: '3.6'
+            numpy-version: '1.22.0'
+          - python-version: '3.7'
+            numpy-version: '1.22.0'
+          - python-version: '3.8'
             torch-version: '1.2.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.8'
-            torch-version: '1.3.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
-            torch-version: '1.1.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
+          - python-version: '3.9'
             torch-version: '1.2.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
-            torch-version: '1.3.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
+          - python-version: '3.9'
             torch-version: '1.4.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
-            torch-version: '1.5.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
+          - python-version: '3.9'
             torch-version: '1.6.0'
-          - os: 'ubuntu-18.04'
-            python-version: '3.9'
-            torch-version: '1.7.0'
 
     steps:
       - name: Checkout code
@@ -80,6 +63,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade flake8 setuptools wheel twine
+      - name: Install latest numpy
+        if: ${{ matrix.numpy-version == 'latest' }}
+        run: |
+          pip install 'numpy'
+      - name: Install numpy v${{ matrix.numpy-version }}
+        if: ${{ matrix.numpy-version != 'latest' }}
+        run: |
+          pip install 'numpy==${{ matrix.numpy-version }}'
       - name: Install latest pytorch
         if: ${{ matrix.torch-version == 'latest' }}
         run: |