diff --git a/labelme/app.py b/labelme/app.py
index d2e88ed9ce3bc1f7d632789b1af5f63763ef7b1f..04da136cc9c29f948e8e5c8be5abfd6f5e7f4d46 100644
--- a/labelme/app.py
+++ b/labelme/app.py
@@ -6,18 +6,11 @@ import re
 import subprocess
 import sys
 
-try:
-    from PyQt5 import QtCore
-    from PyQt5.QtCore import Qt
-    from PyQt5 import QtGui
-    from PyQt5 import QtWidgets
-    PYQT5 = True
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4.QtCore import Qt
-    from PyQt4 import QtGui
-    from PyQt4 import QtGui as QtWidgets
-    PYQT5 = False
+from qtpy import PYQT5
+from qtpy import QtCore
+from qtpy.QtCore import Qt
+from qtpy import QtGui
+from qtpy import QtWidgets
 
 from labelme.canvas import Canvas
 from labelme.colorDialog import ColorDialog
@@ -979,7 +972,7 @@ class MainWindow(QtWidgets.QMainWindow, WindowMixin):
         filename = QtWidgets.QFileDialog.getOpenFileName(
             self, '%s - Choose Image or Label file' % __appname__,
             path, filters)
-        if PYQT5:
+        if qtpy.PYQT5:
             filename, _ = filename
         filename = str(filename)
         if filename:
@@ -1015,7 +1008,7 @@ class MainWindow(QtWidgets.QMainWindow, WindowMixin):
         filename = dlg.getSaveFileName(
             self, 'Choose File', default_labelfile_name,
             'Label files (*%s)' % LabelFile.suffix)
-        if PYQT5:
+        if qtpy.PYQT5:
             filename, _ = filename
         filename = str(filename)
         return filename
diff --git a/labelme/canvas.py b/labelme/canvas.py
index 33b74ebae725a012c5afaecd3572b5858289c58e..919e131de6f5573eb673534d295cc761061d798c 100644
--- a/labelme/canvas.py
+++ b/labelme/canvas.py
@@ -2,16 +2,10 @@ from __future__ import print_function
 
 import sys
 
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtGui
-    from PyQt5 import QtWidgets
-    PYQT5 = True
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui
-    from PyQt4 import QtGui as QtWidgets
-    PYQT5 = False
+from qtpy import PYQT5
+from qtpy import QtCore
+from qtpy import QtGui
+from qtpy import QtWidgets
 
 from labelme.lib import distance
 from labelme.shape import Shape
@@ -29,12 +23,12 @@ CURSOR_GRAB = QtCore.Qt.OpenHandCursor
 
 
 class Canvas(QtWidgets.QWidget):
-    zoomRequest = QtCore.pyqtSignal(int, QtCore.QPoint)
-    scrollRequest = QtCore.pyqtSignal(int, int)
-    newShape = QtCore.pyqtSignal()
-    selectionChanged = QtCore.pyqtSignal(bool)
-    shapeMoved = QtCore.pyqtSignal()
-    drawingPolygon = QtCore.pyqtSignal(bool)
+    zoomRequest = QtCore.Signal(int, QtCore.QPoint)
+    scrollRequest = QtCore.Signal(int, int)
+    newShape = QtCore.Signal()
+    selectionChanged = QtCore.Signal(bool)
+    shapeMoved = QtCore.Signal()
+    drawingPolygon = QtCore.Signal(bool)
 
     CREATE, EDIT = 0, 1
 
diff --git a/labelme/colorDialog.py b/labelme/colorDialog.py
index b10beadf0a94f3dfc04436e0ac3465e998ef9e00..e56143d8e3afe43fd97b6937cfe3d739ea56f6bd 100644
--- a/labelme/colorDialog.py
+++ b/labelme/colorDialog.py
@@ -1,7 +1,4 @@
-try:
-    from PyQt5 import QtWidgets
-except ImportError:
-    from PyQt4 import QtGui as QtWidgets
+from qtpy import QtWidgets
 
 
 class ColorDialog(QtWidgets.QColorDialog):
diff --git a/labelme/labelDialog.py b/labelme/labelDialog.py
index 2e5136ae10ffb43ef8ef5a9f3ac995660b2023d2..6e42c28734b01d389f1984a71538ef86078e3952 100644
--- a/labelme/labelDialog.py
+++ b/labelme/labelDialog.py
@@ -1,13 +1,7 @@
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtGui
-    from PyQt5 import QtWidgets
-    PYQT5 = True
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui
-    from PyQt4 import QtGui as QtWidgets
-    PYQT5 = False
+from qtpy import PYQT5
+from qtpy import QtCore
+from qtpy import QtGui
+from qtpy import QtWidgets
 
 from .lib import labelValidator
 from .lib import newIcon
diff --git a/labelme/lib.py b/labelme/lib.py
index aa2d14431045e5b1e3da3bc1e8c3046937f1a16b..1394715e69e254ecaaf4bdee5565b2122e4dcd17 100644
--- a/labelme/lib.py
+++ b/labelme/lib.py
@@ -1,14 +1,9 @@
 from math import sqrt
 import os.path as osp
 
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtGui
-    from PyQt5 import QtWidgets
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui
-    from PyQt4 import QtGui as QtWidgets
+from qtpy import QtCore
+from qtpy import QtGui
+from qtpy import QtWidgets
 
 
 here = osp.dirname(osp.abspath(__file__))
diff --git a/labelme/shape.py b/labelme/shape.py
index 8935195c1cf3c20b4b7315e8c30825dec7e6b8b2..f89bf16b9981e703541fcfba8d156d87a871a7ba 100644
--- a/labelme/shape.py
+++ b/labelme/shape.py
@@ -1,7 +1,4 @@
-try:
-    from PyQt5 import QtGui
-except ImportError:
-    from PyQt4 import QtGui
+from qtpy import QtGui
 
 from .lib import distance
 
diff --git a/labelme/toolBar.py b/labelme/toolBar.py
index 119cb3f3b103d8f7c1df801932800e1bee37d94f..9bfe1764aff4f4ee6b69e6db2f1144eabf3a079d 100644
--- a/labelme/toolBar.py
+++ b/labelme/toolBar.py
@@ -1,9 +1,5 @@
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtWidgets
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui as QtWidgets
+from qtpy import QtCore
+from qtpy import QtWidgets
 
 
 class ToolBar(QtWidgets.QToolBar):
diff --git a/labelme/zoomWidget.py b/labelme/zoomWidget.py
index 64305bb3470be3b2ca1d92b3ed69b4eff55b8c20..7a224b91572fb038a4601cf45dd7b2c6cce3048a 100644
--- a/labelme/zoomWidget.py
+++ b/labelme/zoomWidget.py
@@ -1,11 +1,6 @@
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtGui
-    from PyQt5 import QtWidgets
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui
-    from PyQt4 import QtGui as QtWidgets
+from qtpy import QtCore
+from qtpy import QtGui
+from qtpy import QtWidgets
 
 
 class ZoomWidget(QtWidgets.QSpinBox):
diff --git a/setup.py b/setup.py
index 4ddbf85c2a75d78bc33bf56214c417513278360f..69fdef2a495c002a6bd7a1b37193eebea964be04 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@ install_requires = [
     'numpy',
     'Pillow>=2.8.0',
     'PyYAML',
+    'qtpy',
 ]
 
 
diff --git a/tests/test_labelDialog.py b/tests/test_labelDialog.py
index 15648e1f6d3d79b236413628d9b892059bd6c9e9..1a877664338d457d38fec193c75ab31f4dade2d8 100644
--- a/tests/test_labelDialog.py
+++ b/tests/test_labelDialog.py
@@ -1,9 +1,5 @@
-try:
-    from PyQt5 import QtCore
-    from PyQt5 import QtWidgets
-except ImportError:
-    from PyQt4 import QtCore
-    from PyQt4 import QtGui as QtWidgets
+from qtpy import QtCore
+from qtpy import QtWidgets
 
 from labelme import labelDialog