evp/evp_enc.c: check for partially[!] overlapping buffers

in EVP_EncryptUpdate and EVP_DecryptUpdate. It is argued that in
general case it's impossible to provide guarantee that partially[!]
overlapping buffers can be tolerated.
Reviewed-by: NMatt Caswell <matt@openssl.org>

crypto/evp/evp_enc.c

@@ -8,6 +8,7 @@
  */
 
 #include <stdio.h>
+#include <assert.h>
 #include "internal/cryptlib.h"
 #include <openssl/evp.h>
 #include <openssl/err.h>
@@ -252,11 +253,48 @@ int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
     return EVP_CipherInit_ex(ctx, cipher, impl, key, iv, 0);
 }
 
+/*
+ * According to the letter of standard difference between pointers
+ * is specified to be valid only within same object. This makes
+ * it formally challenging to determine if input and output buffers
+ * are not partially overlapping with standard pointer arithmetic.
+ */
+#ifdef PTRDIFF_T
+# undef PTRDIFF_T
+#endif
+#if defined(OPENSSL_SYS_VMS) && __INITIAL_POINTER_SIZE==64
+/*
+ * Then we have VMS that distinguishes itself by adhering to
+ * sizeof(size_t)==4 even in 64-bit builds...
+ */
+# define PTRDIFF_T uint64_t
+#else
+# define PTRDIFF_T size_t
+#endif
+
+static int is_partially_overlapping(const void *ptr1, const void *ptr2,
+                                    int len)
+{
+    PTRDIFF_T diff = (PTRDIFF_T)ptr1-(PTRDIFF_T)ptr2;
+    /*
+     * Check for partially overlapping buffers. [Binary logical
+     * operations are used instead of boolean to minimize number
+     * of conditional branches.]
+     */
+    int condition = (len > 0) & (diff != 0) & ((diff < (PTRDIFF_T)len) |
+                                               (diff > (0 - (PTRDIFF_T)len)));
+    assert(!condition);
+    return condition;
+}
+
 int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
     int i, j, bl;
 
+    if (is_partially_overlapping(out, in, inl))
+        return 0;
+
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
         i = ctx->cipher->do_cipher(ctx, out, in, inl);
         if (i < 0)
@@ -370,6 +408,9 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     int fix_len;
     unsigned int b;
 
+    if (is_partially_overlapping(out, in, inl))
+        return 0;
+
     if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
         fix_len = ctx->cipher->do_cipher(ctx, out, in, inl);
         if (fix_len < 0) {

doc/crypto/EVP_EncryptInit.pod

@@ -137,7 +137,9 @@ multiple times to encrypt successive blocks of data. The amount
 of data written depends on the block alignment of the encrypted data:
 as a result the amount of data written may be anything from zero bytes
 to (inl + cipher_block_size - 1) so B<out> should contain sufficient
-room. The actual number of bytes written is placed in B<outl>.
+room. The actual number of bytes written is placed in B<outl>. It also
+checks if B<in> and B<out> are partially overlapping, and if they are
+0 is returned to indicate failure.
 
 If padding is enabled (the default) then EVP_EncryptFinal_ex() encrypts
 the "final" data, that is any data that remains in a partial block.