reject out of range memory in SetMemory APIs

The APIs take the memory value in KiB and we store it in KiB
internally, but we cannot parse the whole ULONG_MAX range
on 64-bit systems, because virDomainParseScaledValue
needs to fit the value in bytes in an unsigned long long.

https://bugzilla.redhat.com/show_bug.cgi?id=1176739

src/conf/domain_conf.c

@@ -7098,12 +7098,7 @@ virDomainParseMemory(const char *xpath,
     int ret = -1;
     unsigned long long bytes, max;
 
-    /* On 32-bit machines, our bound is 0xffffffff * KiB. On 64-bit
-     * machines, our bound is off_t (2^63).  */
-    if (capped && sizeof(unsigned long) < sizeof(long long))
-        max = 1024ull * ULONG_MAX;
-    else
-        max = LLONG_MAX;
+    max = virMemoryMaxValue(capped);
 
     ret = virDomainParseScaledValue(xpath, units_xpath, ctxt,
                                     &bytes, 1024, max, required);

src/libvirt-domain.c

@@ -1850,6 +1850,12 @@ virDomainSetMaxMemory(virDomainPtr domain, unsigned long memory)
     virCheckReadOnlyGoto(conn->flags, error);
     virCheckNonZeroArgGoto(memory, error);
 
+    if (virMemoryMaxValue(true) / 1024 <= memory) {
+        virReportError(VIR_ERR_OVERFLOW, _("input too large: %lu"),
+                       memory);
+        goto error;
+    }
+
     if (conn->driver->domainSetMaxMemory) {
         int ret;
         ret = conn->driver->domainSetMaxMemory(domain, memory);
@@ -1896,6 +1902,12 @@ virDomainSetMemory(virDomainPtr domain, unsigned long memory)
     virCheckReadOnlyGoto(conn->flags, error);
     virCheckNonZeroArgGoto(memory, error);
 
+    if (virMemoryMaxValue(true) / 1024 <= memory) {
+        virReportError(VIR_ERR_OVERFLOW, _("input too large: %lu"),
+                       memory);
+        goto error;
+    }
+
     if (conn->driver->domainSetMemory) {
         int ret;
         ret = conn->driver->domainSetMemory(domain, memory);
@@ -1953,6 +1965,12 @@ virDomainSetMemoryFlags(virDomainPtr domain, unsigned long memory,
     virCheckReadOnlyGoto(conn->flags, error);
     virCheckNonZeroArgGoto(memory, error);
 
+    if (virMemoryMaxValue(true) / 1024 <= memory) {
+        virReportError(VIR_ERR_OVERFLOW, _("input too large: %lu"),
+                       memory);
+        goto error;
+    }
+
     if (conn->driver->domainSetMemoryFlags) {
         int ret;
         ret = conn->driver->domainSetMemoryFlags(domain, memory, flags);

src/libvirt_private.syms

@@ -2334,6 +2334,7 @@ virIsSUID;
 virManageVport;
 virMemoryLimitIsSet;
 virMemoryLimitTruncate;
+virMemoryMaxValue;
 virParseNumber;
 virParseOwnershipIds;
 virParseVersionString;

src/util/virutil.c

@@ -2598,3 +2598,23 @@ virMemoryLimitIsSet(unsigned long long value)
 {
     return value < VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
 }
+
+
+/**
+ * virMemoryMaxValue
+ *
+ * @ulong: whether the value must fit into unsigned long
+ *   (long long is assumed otherwise)
+ *
+ * Returns the maximum possible memory value in bytes.
+ */
+unsigned long long
+virMemoryMaxValue(bool ulong)
+{
+    /* On 32-bit machines, our bound is 0xffffffff * KiB. On 64-bit
+     * machines, our bound is off_t (2^63).  */
+    if (ulong && sizeof(unsigned long) < sizeof(long long))
+        return 1024ull * ULONG_MAX;
+    else
+        return LLONG_MAX;
+}

src/util/virutil.h

@@ -245,5 +245,6 @@ long virGetSystemPageSizeKB(void);
 
 unsigned long long virMemoryLimitTruncate(unsigned long long value);
 bool virMemoryLimitIsSet(unsigned long long value);
+unsigned long long virMemoryMaxValue(bool ulong);
 
 #endif /* __VIR_UTIL_H__ */