From 51defd927427b5b74c3a0c0f0b5c161929547cfc Mon Sep 17 00:00:00 2001
From: Sebastian Huber <sebastian.huber@embedded-brains.de>
Date: Tue, 20 Apr 2021 19:30:35 +0200
Subject: Fix calloc() behaviour in case of overflow

The multiplication to calculate the length of the memory area to
allocate may overflow.  Return NULL in case of an overflow.

Close #4389.
---
 cpukit/libcsupport/src/calloc.c      | 13 ++++++++++++-
 cpukit/libcsupport/src/rtemscalloc.c |  9 ++++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

(limited to 'cpukit/libcsupport/src')

diff --git a/cpukit/libcsupport/src/calloc.c b/cpukit/libcsupport/src/calloc.c
index e015f30d6c..693aa21453 100644
--- a/cpukit/libcsupport/src/calloc.c
+++ b/cpukit/libcsupport/src/calloc.c
@@ -20,7 +20,10 @@
 
 #if defined(RTEMS_NEWLIB) && !defined(HAVE_CALLOC)
 #include <stdlib.h>
+
+#include <errno.h>
 #include <string.h>
+
 #include <rtems/score/basedefs.h>
 
 void *calloc(
@@ -31,7 +34,15 @@ void *calloc(
   void   *cptr;
   size_t  length;
 
-  length = nelem * elsize;
+  if ( nelem == 0 ) {
+    length = 0;
+  } else if ( elsize > SIZE_MAX / nelem ) {
+    errno = ENOMEM;
+    return NULL;
+  } else {
+    length = nelem * elsize;
+  }
+
   cptr = malloc( length );
   RTEMS_OBFUSCATE_VARIABLE( cptr );
   if ( RTEMS_PREDICT_FALSE( cptr == NULL ) ) {
diff --git a/cpukit/libcsupport/src/rtemscalloc.c b/cpukit/libcsupport/src/rtemscalloc.c
index 4e189e8367..836f1da64d 100644
--- a/cpukit/libcsupport/src/rtemscalloc.c
+++ b/cpukit/libcsupport/src/rtemscalloc.c
@@ -46,7 +46,14 @@ void *rtems_calloc( size_t nelem, size_t elsize )
   size_t  length;
   void   *p;
 
-  length = nelem * elsize;
+  if ( nelem == 0 ) {
+    length = 0;
+  } else if ( elsize > SIZE_MAX / nelem ) {
+    return NULL;
+  } else {
+    length = nelem * elsize;
+  }
+
   p = rtems_malloc( length );
   RTEMS_OBFUSCATE_VARIABLE( p );
   if ( RTEMS_PREDICT_FALSE( p == NULL ) ) {
-- 
cgit v1.2.3