Disable malloc instrumentation for VM (#2)

When the malloc code is instrumented, the global _brksize variable
should not be transferred.  However, when the malloc code is not
instrumented, failing to transfer _brksize would reset the heap
upon state transfer.  In this patch, the magic pass stores the flag
indicating whether memory function instrumentation is disabled, in
the target process.  This allows libmagic to check this flag during
state transfer, to see whether it should transfer _brksize or not.

Change-Id: Ia004651e21e08b0ed3f5305865c53c6659e18f38

diff --git a/minix/llvm/include/magic.h b/minix/llvm/include/magic.h
index 4e55c2ab156d3573575e1b2d51edb50dafb2c950..c31db4358f8b10def2ed93cb87c2796542219f94 100644 (file)
--- a/minix/llvm/include/magic.h
+++ b/minix/llvm/include/magic.h
@@ -869,6 +869,9 @@ EXTERN void* __stop_magic_functions_st;
 #define _magic_asr_map_max_padding_pages (                                     \
     _magic_vars->asr_map_max_padding_pages)
 
+/* Runtime flags. */
+#define _magic_no_mem_inst              (_magic_vars->no_mem_inst)
+
 /* Magic type array. */
 #define _magic_types                    (_magic_vars->types)
 #define _magic_types_num                (_magic_vars->types_num)

diff --git a/minix/llvm/include/magic_common.h b/minix/llvm/include/magic_common.h
index f2e147ba1d430dba2e2ace6dd2fdf5c55fbdb308..7a3d22dcda03bf9f57fbe9f099ebd5706fd72db8 100644 (file)
--- a/minix/llvm/include/magic_common.h
+++ b/minix/llvm/include/magic_common.h
@@ -179,6 +179,7 @@
 #define MAGIC_RSTRUCT_FIELD_ASR_HEAP_MAX_PADDING      "asr_heap_max_padding"
 #define MAGIC_RSTRUCT_FIELD_ASR_MAP_MAX_OFFSET_PAGES  "asr_map_max_offset_pages"
 #define MAGIC_RSTRUCT_FIELD_ASR_MAP_MAX_PADDING_PAGES "asr_map_max_padding_pages"
+#define MAGIC_RSTRUCT_FIELD_NO_MEM_INST         "no_mem_inst"
 #define MAGIC_RSTRUCT_FIELD_TYPES               "types"
 #define MAGIC_RSTRUCT_FIELD_TYPES_NUM           "types_num"
 #define MAGIC_RSTRUCT_FIELD_TYPES_NEXT_ID       "types_next_id"
@@ -199,6 +200,7 @@
         MAGIC_RSTRUCT_FIELD_ASR_HEAP_MAX_PADDING,                              \
         MAGIC_RSTRUCT_FIELD_ASR_MAP_MAX_OFFSET_PAGES,                          \
         MAGIC_RSTRUCT_FIELD_ASR_MAP_MAX_PADDING_PAGES,                         \
+        MAGIC_RSTRUCT_FIELD_NO_MEM_INST,                                       \
         MAGIC_RSTRUCT_FIELD_TYPES,                                             \
         MAGIC_RSTRUCT_FIELD_TYPES_NUM,                                         \
         MAGIC_RSTRUCT_FIELD_TYPES_NEXT_ID,                                     \

diff --git a/minix/llvm/include/magic_structs.h b/minix/llvm/include/magic_structs.h
index d7cedd60cddc2c3e62d116baaacdd0589ced3739..ac763e0c0f723b44b88bd660925a3760685e097a 100644 (file)
--- a/minix/llvm/include/magic_structs.h
+++ b/minix/llvm/include/magic_structs.h
@@ -234,6 +234,9 @@ struct _magic_vars_t {
     int asr_map_max_offset_pages;
     int asr_map_max_padding_pages;
 
+    /* Runtime flags. */
+    int no_mem_inst;
+
     /* Magic type array. */
     struct _magic_type *types;
     int types_num;

diff --git a/minix/llvm/include/st/typedefs.h b/minix/llvm/include/st/typedefs.h
index 739be1f8114ff56420d487c4dcbebb94e7db074e..29d9e35c032284527b67b56a4f5d614cae07991e 100644 (file)
--- a/minix/llvm/include/st/typedefs.h
+++ b/minix/llvm/include/st/typedefs.h
@@ -41,11 +41,12 @@
     ST_DECLARE_STD_PTRINT_TYPEDEFS(pxfer_);
 #define ST_TYPENAME_STRUCT_TRANSFER_NAMES       "sxfer_*"
 #ifdef __MINIX
-#define ST_SENTRYNAME_NO_TRANSFER_NAMES         "noxfer_*", "sef_*", "st_*", "_brksize", "etext"
+#define ST_SENTRYNAME_NO_TRANSFER_NAMES         "noxfer_*", "sef_*", "st_*", "etext"
 #else
-#define ST_SENTRYNAME_NO_TRANSFER_NAMES         "noxfer_*", "st_*", "_brksize", "etext", "allocatedDescs*", "ep.*" /* nginx specific */
+#define ST_SENTRYNAME_NO_TRANSFER_NAMES         "noxfer_*", "st_*", "etext", "allocatedDescs*", "ep.*" /* nginx specific */
 #define ST_DSENTRYLIB_NO_TRANSFER_NAMES         "*/libst.so", "*/libcommon.so", "*/libtaskctl.so"
 #endif
+#define ST_SENTRYNAME_NO_TRANSFER_MEM_NAMES     "_brksize"
 #define ST_SENTRYNAME_IDENTITY_TRANSFER_NAMES   "ixfer_*"
 #define ST_SENTRYNAME_CIDENTITY_TRANSFER_NAMES  "cixfer_*"
 #define ST_SENTRYNAME_PTR_TRANSFER_NAMES        "pxfer_*"

diff --git a/minix/llvm/passes/magic/MagicPass.cpp b/minix/llvm/passes/magic/MagicPass.cpp
index 0e0dbf7406286006c5d6879be69567c03390c829..d06740f84e12ae14cf5c8b0c61074e65441d6df6 100644 (file)
--- a/minix/llvm/passes/magic/MagicPass.cpp
+++ b/minix/llvm/passes/magic/MagicPass.cpp
@@ -193,6 +193,13 @@ bool MagicPass::runOnModule(Module &M) {
     }
     Instruction *magicArrayBuildFuncInst = magicDataInitFunc->back().getTerminator();
 
+    //look up pointer to magic memory instrumentation flag
+    Value* magicNoMemInst = MagicUtil::getMagicRStructFieldPtr(M, magicArrayBuildFuncInst, magicRootVar, MAGIC_RSTRUCT_FIELD_NO_MEM_INST);
+    if(!magicNoMemInst) {
+        magicPassErr("Error: no " << MAGIC_RSTRUCT_FIELD_NO_MEM_INST << " field found");
+        exit(1);
+    }
+
     //look up pointer to magic array and magic struct type
     Value* magicArrayPtr = MagicUtil::getMagicRStructFieldPtr(M, magicArrayBuildFuncInst, magicRootVar, MAGIC_RSTRUCT_FIELD_SENTRIES);
     if(!magicArrayPtr) {
@@ -1684,6 +1691,9 @@ bool MagicPass::runOnModule(Module &M) {
     //set pointer to magic type array in build function
     new StoreInst(MagicUtil::getArrayPtr(M, magicTypeArray), magicTypeArrayPtr, false, magicArrayBuildFuncInst);
 
+    // set runtime flags
+    new StoreInst(ConstantInt::get(M.getContext(), APInt(32, DisableMemFunctions ? 1 : 0)), magicNoMemInst, false, magicArrayBuildFuncInst);
+
     //set magic type array size in build function
     new StoreInst(ConstantInt::get(M.getContext(), APInt(32, globalTypeInfos.size())), magicTypeArraySize, false, magicArrayBuildFuncInst);
 

diff --git a/minix/llvm/static/magic/magic.c b/minix/llvm/static/magic/magic.c
index 7c34264ff695232baf4a627a89e277795557db4f..8b345cc1ef0c0d5135a71c4ad0cad7798f039a63 100644 (file)
--- a/minix/llvm/static/magic/magic.c
+++ b/minix/llvm/static/magic/magic.c
@@ -57,6 +57,9 @@ MAGIC_VAR struct _magic_vars_t _magic_vars_buff = {
     0, /* asr_map_max_offset_pages */
     0, /* asr_map_max_padding_pages */
 
+    /* Runtime flags. */
+    0,    /* no_mem_inst */
+
     /* Magic type array. */
     NULL, /* types */
     0,    /* types_num */

diff --git a/minix/llvm/static/magic/magic_st.c b/minix/llvm/static/magic/magic_st.c
index 1bd9a95e16986e48ee225450346b159971b6f121..dd0cd6fd3c096b6de10b4f51116f7e2d82987649 100644 (file)
--- a/minix/llvm/static/magic/magic_st.c
+++ b/minix/llvm/static/magic/magic_st.c
@@ -214,6 +214,7 @@ char *st_sentryname_noxfers[] = {
 #endif
 #undef __X
     NULL };
+char *st_sentryname_noxfers_mem[] = { ST_SENTRYNAME_NO_TRANSFER_MEM_NAMES, NULL };
 
 /* Exclude the data segments of certain libs from state transfer. */
 char *st_dsentry_lib_noxfer[] = {
@@ -730,6 +731,14 @@ PUBLIC int st_cb_transfer_sentry_default(_magic_selement_t *selement, _magic_sel
         return MAGIC_SENTRY_ANALYZE_SKIP_PATH;
     }
 
+    /* Skip memory management related sentries only when memory functions have
+     * been instrumented (which is *not* the case for the MINIX3 VM service).
+     */
+    if (_magic_no_mem_inst == 0 && ST_SENTRY_NAME_MATCH_ANY(st_sentryname_noxfers_mem, sentry_name)) {
+        ST_CB_PRINT(ST_CB_DBG, "sentry name matches noxfer", selement, sel_analyzed, sel_stats, cb_info);
+        return MAGIC_SENTRY_ANALYZE_SKIP_PATH;
+    }
+
     if (ST_SENTRY_NAME_MATCH_ANY(st_sentryname_pxfers, sentry_name)) {
         ST_CB_PRINT(ST_CB_DBG, "sentry name matches pxfer", selement, sel_analyzed, sel_stats, cb_info);
         return transfer_ptr_sel_cb(selement, sel_analyzed, sel_stats, cb_info);