Fix calloc() behaviour in case of overflow

The multiplication to calculate the length of the memory area to
allocate may overflow.  Return NULL in case of an overflow.

Close #4389.

3 files changed, 33 insertions, 12 deletions

diff --git a/cpukit/libcsupport/src/calloc.c b/cpukit/libcsupport/src/calloc.c
index e015f30d6c..693aa21453 100644
--- a/cpukit/libcsupport/src/calloc.c
+++ b/cpukit/libcsupport/src/calloc.c
@@ -20,7 +20,10 @@
 
 #if defined(RTEMS_NEWLIB) && !defined(HAVE_CALLOC)
 #include <stdlib.h>
+
+#include <errno.h>
 #include <string.h>
+
 #include <rtems/score/basedefs.h>
 
 void *calloc(
@@ -31,7 +34,15 @@ void *calloc(
   void   *cptr;
   size_t  length;
 
-  length = nelem * elsize;
+  if ( nelem == 0 ) {
+    length = 0;
+  } else if ( elsize > SIZE_MAX / nelem ) {
+    errno = ENOMEM;
+    return NULL;
+  } else {
+    length = nelem * elsize;
+  }
+
   cptr = malloc( length );
   RTEMS_OBFUSCATE_VARIABLE( cptr );
   if ( RTEMS_PREDICT_FALSE( cptr == NULL ) ) {
diff --git a/cpukit/libcsupport/src/rtemscalloc.c b/cpukit/libcsupport/src/rtemscalloc.c
index 4e189e8367..836f1da64d 100644
--- a/cpukit/libcsupport/src/rtemscalloc.c
+++ b/cpukit/libcsupport/src/rtemscalloc.c
@@ -46,7 +46,14 @@ void *rtems_calloc( size_t nelem, size_t elsize )
   size_t  length;
   void   *p;
 
-  length = nelem * elsize;
+  if ( nelem == 0 ) {
+    length = 0;
+  } else if ( elsize > SIZE_MAX / nelem ) {
+    return NULL;
+  } else {
+    length = nelem * elsize;
+  }
+
   p = rtems_malloc( length );
   RTEMS_OBFUSCATE_VARIABLE( p );
   if ( RTEMS_PREDICT_FALSE( p == NULL ) ) {
diff --git a/testsuites/libtests/malloctest/init.c b/testsuites/libtests/malloctest/init.c
index 1d91385683..4d0f421c02 100644
--- a/testsuites/libtests/malloctest/init.c
+++ b/testsuites/libtests/malloctest/init.c
@@ -1190,6 +1190,14 @@ static void test_rtems_calloc(void)
   rtems_test_assert(p == NULL);
   rtems_test_assert(errno == 0);
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Walloc-size-larger-than=N"
+  errno = 0;
+  p = rtems_calloc(SIZE_MAX, SIZE_MAX);
+  rtems_test_assert(p == NULL);
+  rtems_test_assert(errno == 0);
+#pragma GCC diagnostic pop
+
   i = rtems_calloc(1, sizeof(*i));
   rtems_test_assert(i != NULL);
   rtems_test_assert(*i == 0);
@@ -1313,22 +1321,17 @@ rtems_task Init(
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Walloc-size-larger-than=N"
   p1 = calloc( 1, SIZE_MAX );
+  rtems_test_assert( p1 == NULL );
+
+  p1 = calloc( SIZE_MAX, SIZE_MAX );
+  rtems_test_assert( p1 == NULL );
 #pragma GCC diagnostic pop
-  if (p1) {
-    printf("ERROR on attempt to calloc SIZE_MAX block expected failure.");
-    free( p1 );
-  }
 
   /*
    * Verify error case where malloc of size 0.
    */
   p1 = malloc( 0 );
-  if (p1) {
-    printf("ERROR on attempt to malloc size 0 block expected failure.");
-    free( p1 );
-  }
-
-
+  rtems_test_assert( p1 == NULL );
 
   test_heap_initialize();
   test_heap_block_allocate();