libmagic: fix ASR padding for memory mapping

The code could not decide whether to apply the padding to the start
or the end of the region, resulting in strange behavior because part
of the returned range might not have the right properties.  With this
patch, padding is now consistently applied at the end of the region,
since virtual mmap addresses are allocated from high to low.

Also fix a few small related bugs in error handling code.

Change-Id: I327814bb3921d7982ffc1296a5bf365d9c79c1d9

diff --git a/minix/llvm/static/magic/magic_mem.c b/minix/llvm/static/magic/magic_mem.c
index c9a7ac910d26dab08055e5aab6226ad58cf454e9..b25e1c8c16a81c6079ccb7201be2d17db91f4184 100644 (file)
--- a/minix/llvm/static/magic/magic_mem.c
+++ b/minix/llvm/static/magic/magic_mem.c
@@ -695,7 +695,7 @@ PRIVATE int magic_free_dsentry(struct _magic_dsentry *dsentry)
             if (!MAGIC_STATE_FLAG(sentry, MAGIC_STATE_DETACHED)) {
                 size_t padding_size = (size_t) dsentry->ext;
                 MAGIC_MEM_DEBUG_FREE(ptr);
-                ret = munmap(((char *)aligned_ptr) - padding_size, page_size + size + padding_size);
+                ret = munmap((char *)aligned_ptr, page_size + size + padding_size);
             }
             else {
 #ifndef __MINIX
@@ -1708,7 +1708,7 @@ PUBLIC void *magic_mmap_positioned(__MA_ARGS__ void *start, size_t length, int p
                 new_start = ((char *)ptr) + page_size;
                 new_ptr = mmap(new_start, length, prot, flags | MAP_FIXED, fd, offset);
                 if (new_ptr == MAP_FAILED) {
-                    munmap(ptr, page_size + alloc_length);
+                    munmap(ptr, page_size + alloc_length + padding_size);
                     ptr = MAP_FAILED;
                 }
             }
@@ -1716,7 +1716,7 @@ PUBLIC void *magic_mmap_positioned(__MA_ARGS__ void *start, size_t length, int p
         aligned_ptr = ptr;
         MAGIC_MEM_PRINTF("magic_mmap: ptr = mmap(start, length, prot, flags, fd, offset) <-> 0x%08x = mmap(0x%08x, %d, 0x%08x, 0x%08x, %d, %d)\n", (unsigned) aligned_ptr, aligned_start, page_size + length, prot, flags, fd, offset);
         if (ptr != MAP_FAILED) {
-            ptr = ((char *)ptr) + padding_size + page_size - MAGIC_SIZE_TO_REAL(0);
+            ptr = ((char *)ptr) + page_size - MAGIC_SIZE_TO_REAL(0);
         }
         else {
             ptr = NULL;
@@ -1725,14 +1725,14 @@ PUBLIC void *magic_mmap_positioned(__MA_ARGS__ void *start, size_t length, int p
             dsentry_flags |= MAGIC_STATE_SHM;
         data_ptr = magic_alloc(__MA_VALUES__ ptr, alloc_length, dsentry_flags);
         if (data_ptr == MAGIC_MEM_FAILED) {
-            munmap(aligned_ptr, page_size + length);
+            munmap(aligned_ptr, page_size + alloc_length + padding_size);
             data_ptr = NULL;
-            errno = ENOMEM;
         }
         if (!data_ptr) {
             errno = ENOMEM;
             data_ptr = MAP_FAILED;
         } else {
+            assert(data_ptr == (char *)aligned_ptr + page_size);
             MAGIC_PTR_TO_DSENTRY(MAGIC_PTR_FROM_DATA(data_ptr))->alloc_mmap_flags = flags;
             MAGIC_PTR_TO_DSENTRY(MAGIC_PTR_FROM_DATA(data_ptr))->alloc_mmap_prot = prot;
             MAGIC_PTR_TO_DSENTRY(MAGIC_PTR_FROM_DATA(data_ptr))->ext = (void *) padding_size;

diff --git a/minix/llvm/static/magic/magic_st.c b/minix/llvm/static/magic/magic_st.c
index 82ca4249279a1e21300b90f502a1f69a9b312282..419caecf3a7f08e84d3b6c7daef778d8545057b4 100644 (file)
--- a/minix/llvm/static/magic/magic_st.c
+++ b/minix/llvm/static/magic/magic_st.c
@@ -3374,7 +3374,7 @@ PRIVATE int allocate_pair_metadata_dsentries(st_init_info_t *info,
              * in order to free the preceding page, containing the dsentry struct, too.
              */
             MAGIC_MEM_WRAPPER_BLOCK(
-                res = munmap((char *)sentry->address - magic_get_sys_pagesize() - padding, size + magic_get_sys_pagesize() + padding);
+                res = munmap((char *)sentry->address - magic_get_sys_pagesize(), size + magic_get_sys_pagesize() + padding);
             );
             if (res != 0) {
                 printf("ERROR, munmap returned NULL.\n");