From 51defd927427b5b74c3a0c0f0b5c161929547cfc Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 20 Apr 2021 19:30:35 +0200 Subject: Fix calloc() behaviour in case of overflow The multiplication to calculate the length of the memory area to allocate may overflow. Return NULL in case of an overflow. Close #4389. --- testsuites/libtests/malloctest/init.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'testsuites') diff --git a/testsuites/libtests/malloctest/init.c b/testsuites/libtests/malloctest/init.c index 1d91385683..4d0f421c02 100644 --- a/testsuites/libtests/malloctest/init.c +++ b/testsuites/libtests/malloctest/init.c @@ -1190,6 +1190,14 @@ static void test_rtems_calloc(void) rtems_test_assert(p == NULL); rtems_test_assert(errno == 0); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Walloc-size-larger-than=N" + errno = 0; + p = rtems_calloc(SIZE_MAX, SIZE_MAX); + rtems_test_assert(p == NULL); + rtems_test_assert(errno == 0); +#pragma GCC diagnostic pop + i = rtems_calloc(1, sizeof(*i)); rtems_test_assert(i != NULL); rtems_test_assert(*i == 0); @@ -1313,22 +1321,17 @@ rtems_task Init( #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Walloc-size-larger-than=N" p1 = calloc( 1, SIZE_MAX ); + rtems_test_assert( p1 == NULL ); + + p1 = calloc( SIZE_MAX, SIZE_MAX ); + rtems_test_assert( p1 == NULL ); #pragma GCC diagnostic pop - if (p1) { - printf("ERROR on attempt to calloc SIZE_MAX block expected failure."); - free( p1 ); - } /* * Verify error case where malloc of size 0. */ p1 = malloc( 0 ); - if (p1) { - printf("ERROR on attempt to malloc size 0 block expected failure."); - free( p1 ); - } - - + rtems_test_assert( p1 == NULL ); test_heap_initialize(); test_heap_block_allocate(); -- cgit v1.2.3