diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index 950a4827507f9fe0e289481f1dafdc696e198eed..652dced0f3ec99f9e3d9b3c50bcb946969c70c35 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -164,7 +164,6 @@ struct ieee80211_low_level_stats {
 /* Transmit control fields. This data structure is passed to low-level driver
  * with each TX frame. The low-level driver is responsible for configuring
  * the hardware to use given values (depending on what is supported). */
-#define HW_KEY_IDX_INVALID -1
 
 struct ieee80211_tx_control {
 	int tx_rate; /* Transmit rate, given as the hw specific value for the
@@ -197,13 +196,13 @@ struct ieee80211_tx_control {
 						  * long retry value */
 	u32 flags;			       /* tx control flags defined
 						* above */
+	u8 key_idx;		/* keyidx from hw->set_key(), undefined if
+				 * IEEE80211_TXCTL_DO_NOT_ENCRYPT is set */
 	u8 retry_limit;		/* 1 = only first attempt, 2 = one retry, ..
 				 * This could be used when set_retry_limit
 				 * is not implemented by the driver */
 	u8 power_level;		/* per-packet transmit power level, in dBm */
 	u8 antenna_sel_tx; 	/* 0 = default/diversity, 1 = Ant0, 2 = Ant1 */
-	s8 key_idx;		/* HW_KEY_IDX_INVALID = do not encrypt,
-				 * other values: keyidx from hw->set_key() */
 	u8 icv_len;		/* length of the ICV/MIC field in octets */
 	u8 iv_len;		/* length of the IV field in octets */
 	u8 tkip_key[16];	/* generated phase2/phase1 key for hw TKIP */
@@ -467,8 +466,7 @@ enum ieee80211_key_flags {
  *
  * @hw_key_idx: To be set by the driver, this is the key index the driver
  *	wants to be given when a frame is transmitted and needs to be
- *	encrypted in hardware. It defaults to %HW_KEY_IDX_INVALID which
- *	the driver may not use.
+ *	encrypted in hardware.
  * @alg: The key algorithm.
  * @flags: key flags, see &enum ieee80211_key_flags.
  * @keyidx: the key index (0-3)
@@ -476,8 +474,8 @@ enum ieee80211_key_flags {
  * @key: key material
  */
 struct ieee80211_key_conf {
-	int hw_key_idx;
 	ieee80211_key_alg alg;
+	u8 hw_key_idx;
 	u8 flags;
 	s8 keyidx;
 	u8 keylen;
@@ -686,9 +684,10 @@ struct ieee80211_ops {
 	 * selected by the low-level driver.
 	 *
 	 * Return 0 if the key is now in use, -EOPNOTSUPP or -ENOSPC if it
-	 * couldn't be added; if you return 0 then hw_key_idx must be
-	 * assigned to something other than HW_KEY_IDX_INVALID. When the cmd
-	 * is DISABLE_KEY then it must succeed.
+	 * couldn't be added; if you return 0 then hw_key_idx must be assigned
+	 * to the hardware key index, you are free to use the full u8 range.
+	 *
+	 * When the cmd is DISABLE_KEY then it must succeed.
 	 *
 	 * Note that it is permissible to not decrypt a frame even if a key
 	 * for it has been uploaded to hardware, the stack will not make any
diff --git a/net/mac80211/key.c b/net/mac80211/key.c
index b10e33deb63ec8e052548a18f4c4674d408ca460..dd6fc4aa3ff506c92540b88ccfc6068cb0b3ee61 100644
--- a/net/mac80211/key.c
+++ b/net/mac80211/key.c
@@ -73,8 +73,6 @@ static void ieee80211_key_enable_hw_accel(struct ieee80211_key *key)
 				       key->sdata->dev->dev_addr, addr,
 				       &key->conf);
 
-	WARN_ON(!ret && (key->conf.hw_key_idx == HW_KEY_IDX_INVALID));
-
 	if (!ret)
 		key->flags |= KEY_FLAG_UPLOADED_TO_HARDWARE;
 
@@ -109,7 +107,6 @@ static void ieee80211_key_disable_hw_accel(struct ieee80211_key *key)
 		       key->conf.keyidx, MAC_ARG(addr), ret);
 
 	key->flags &= ~KEY_FLAG_UPLOADED_TO_HARDWARE;
-	key->conf.hw_key_idx = HW_KEY_IDX_INVALID;
 }
 
 struct ieee80211_key *ieee80211_key_alloc(struct ieee80211_sub_if_data *sdata,
@@ -132,7 +129,6 @@ struct ieee80211_key *ieee80211_key_alloc(struct ieee80211_sub_if_data *sdata,
 	 * Default to software encryption; we'll later upload the
 	 * key to the hardware if possible.
 	 */
-	key->conf.hw_key_idx = HW_KEY_IDX_INVALID;
 	key->conf.flags = 0;
 	key->flags = 0;
 
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index e2ae1e1fcc7b6fed6b7afba427933cc5877f040d..3d57e6d80f6db7b2adfc720793a3fe28319fe1bf 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -430,8 +430,6 @@ ieee80211_tx_h_select_key(struct ieee80211_txrx_data *tx)
 {
 	struct ieee80211_key *key;
 
-	tx->u.tx.control->key_idx = HW_KEY_IDX_INVALID;
-
 	if (unlikely(tx->u.tx.control->flags & IEEE80211_TXCTL_DO_NOT_ENCRYPT))
 		tx->key = NULL;
 	else if (tx->sta && (key = rcu_dereference(tx->sta->key)))
@@ -442,8 +440,10 @@ ieee80211_tx_h_select_key(struct ieee80211_txrx_data *tx)
 		 !(tx->sdata->eapol && ieee80211_is_eapol(tx->skb))) {
 		I802_DEBUG_INC(tx->local->tx_handlers_drop_unencrypted);
 		return TXRX_DROP;
-	} else
+	} else {
 		tx->key = NULL;
+		tx->u.tx.control->flags |= IEEE80211_TXCTL_DO_NOT_ENCRYPT;
+	}
 
 	if (tx->key) {
 		tx->key->tx_rx_count++;
@@ -724,6 +724,15 @@ ieee80211_tx_h_misc(struct ieee80211_txrx_data *tx)
 		}
 	}
 
+	/*
+	 * Tell hardware to not encrypt when we had sw crypto.
+	 * Because we use the same flag to internally indicate that
+	 * no (software) encryption should be done, we have to set it
+	 * after all crypto handlers.
+	 */
+	if (tx->key && !(tx->key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE))
+		tx->u.tx.control->flags |= IEEE80211_TXCTL_DO_NOT_ENCRYPT;
+
 	return TXRX_CONTINUE;
 }
 
@@ -833,7 +842,6 @@ __ieee80211_parse_tx_radiotap(
 	 */
 
 	control->retry_limit = 1; /* no retry */
-	control->key_idx = HW_KEY_IDX_INVALID;
 	control->flags &= ~(IEEE80211_TXCTL_USE_RTS_CTS |
 			    IEEE80211_TXCTL_USE_CTS_PROTECT);
 	control->flags |= IEEE80211_TXCTL_DO_NOT_ENCRYPT |