Subject: 25.1.50; [PATCH] Integrate the musl hybrid malloc patch for elf systems

From: Wolfgang Jenkner <[email-masked]>
Date: Thu, 03 Dec 2015 18:57:47 +0100

In the thread around

http://permalink.gmane.org/gmane.emacs.devel/182469

Rich Felker announced that he had found that a small addition to
unexelf.c allowed emacs to extend its hybrid malloc implementation
(previously only used for cygwin) to musl libc, but without using
features specific to that malloc implementation.

A corresponding patch was tested in the "Alpine Linux" distribution, see

http://git.alpinelinux.org/cgit/aports/plain/testing/emacs/musl.patch?id=d6f211ec868df4657c745b8ba2bae77b2a7fb7f6

This patch series proposes the relevant part of that change and adds
some (rather trivial) groundwork to make it reliably work on all systems
which use unexelf.c for dumping.

Personally, I've been using hybrid malloc on top of the libc malloc
(jemalloc) on FreeBSD 10 for several months now without problems.

I also tested it with other malloc libraries, in particular some of
those derived from Doug Lea's malloc.  It also works on GNU/Linux, on
top of glibc.  This might be of some interest as, IIRC, some glibc
people were not too happy about keeping malloc_get_state and stuff.

If you get an error about exceeding "static heap size" during dumping
just bump STATIC_HEAP_SIZE in src/sheap.c a bit.

If the commit messages in the following patches don't explain enough,
please ask.  Also, please apply the patch from

http://debbugs.gnu.org/cgi/bugreport.cgi?bug=22085

beforehand.

>From 0ee89d5aa9eae1bf72e5cc715d8b2e3afce37703 Mon Sep 17 00:00:00 2001
From: Wolfgang Jenkner <[email-masked]>
Date: Sun, 29 Nov 2015 17:39:56 +0100
Subject: [PATCH 2/5] For HYBRID_MALLOC, give most gmalloc symbols internal
 linkage.

This avoids clashes with symbols if the after-dump malloc is
derived from Doug Lea's implementation.

* src/gmalloc.c (emacs_abort, __morecore, __default_morecore):
Move declarations up.  For HYBRID_MALLOC, turn all `extern'
declarations below to `static' ones.
(aligned_alloc): Declare for !MSDOS as well.
(heapsize, _fraghead): Move resp. copy declaration downwards.
For HYBRID_MALLOC, conditionalize out the other definitions,
since the previous `static' declarations double as tentative
definitions, anyway.
(_malloc, _free, _realloc, __free_hook, _aligned_blocks)
(__realloc_hook, __memalign_hook): Conditionalize out.
(cfree, memalign, valloc): Ditto.
---
 src/gmalloc.c | 58 +++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 17 deletions(-)

diff --git a/src/gmalloc.c b/src/gmalloc.c
index 90a52a1..240408e 100644
--- a/src/gmalloc.c
+++ b/src/gmalloc.c
@@ -87,6 +87,20 @@ extern "C"
 
 #include <stddef.h>
 
+#ifdef emacs
+extern void emacs_abort (void);
+#endif
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern void *(*__morecore) (ptrdiff_t size);
+
+/* Default value of `__morecore'.  */
+extern void *__default_morecore (ptrdiff_t size);
+
+#ifdef HYBRID_MALLOC
+#define extern static
+#endif
 
 /* Allocate SIZE bytes of memory.  */
 extern void *malloc (size_t size) ATTRIBUTE_MALLOC_SIZE ((1));
@@ -99,8 +113,8 @@ extern void *calloc (size_t nmemb, size_t size) ATTRIBUTE_MALLOC_SIZE ((1,2));
 extern void free (void *ptr);
 
 /* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
-#ifdef MSDOS
 extern void *aligned_alloc (size_t, size_t);
+#ifdef MSDOS
 extern void *memalign (size_t, size_t);
 extern int posix_memalign (void **, size_t, size_t);
 #endif
@@ -110,10 +124,6 @@ extern int posix_memalign (void **, size_t, size_t);
 extern void malloc_enable_thread (void);
 #endif
 
-#ifdef emacs
-extern void emacs_abort (void);
-#endif
-
 /* The allocator divides the heap into blocks of fixed size; large
    requests receive one or more whole blocks, and small requests
    receive a fragment of a block.  Fragment sizes are powers of two,
@@ -249,13 +259,6 @@ extern int _malloc_thread_enabled_p;
    return the address of the beginning of the object.  */
 extern void *malloc_find_object_address (void *ptr);
 
-/* Underlying allocation function; successive calls should
-   return contiguous pieces of memory.  */
-extern void *(*__morecore) (ptrdiff_t size);
-
-/* Default value of `__morecore'.  */
-extern void *__default_morecore (ptrdiff_t size);
-
 /* If not NULL, this function is called after each time
    `__morecore' is called to increase the data size.  */
 extern void (*__after_morecore_hook) (void);
@@ -318,6 +321,8 @@ extern struct mstats mstats (void);
 /* Call WARNFUN with a warning message when memory usage is high.  */
 extern void memory_warnings (void *start, void (*warnfun) (const char *));
 
+#undef extern
+
 #ifdef	__cplusplus
 }
 #endif
@@ -346,6 +351,8 @@ License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 void *(*__morecore) (ptrdiff_t size) = __default_morecore;
 
+#ifndef HYBRID_MALLOC
+
 /* Debugging hook for `malloc'.  */
 void *(*__malloc_hook) (size_t size);
 
@@ -355,9 +362,6 @@ char *_heapbase;
 /* Block information table.  Allocated with align/__free (not malloc/free).  */
 malloc_info *_heapinfo;
 
-/* Number of info entries.  */
-static size_t heapsize;
-
 /* Search index in the info table.  */
 size_t _heapindex;
 
@@ -381,6 +385,15 @@ size_t __malloc_extra_blocks;
 void (*__malloc_initialize_hook) (void);
 void (*__after_morecore_hook) (void);
 
+#else
+
+static struct list _fraghead[BLOCKLOG];
+
+#endif /* HYBRID_MALLOC */
+
+/* Number of info entries.  */
+static size_t heapsize;
+
 #if defined GC_MALLOC_CHECK && defined GC_PROTECT_MALLOC_STATE
 
 /* Some code for hunting a bug writing into _heapinfo.
@@ -946,7 +959,7 @@ malloc (size_t size)
   return (hook != NULL ? *hook : _malloc_internal) (size);
 }
 
-#ifndef _LIBC
+#if !(defined (_LIBC) || defined (HYBRID_MALLOC))
 
 /* On some ANSI C systems, some libc functions call _malloc, _free
    and _realloc.  Make them use the GNU functions.  */
@@ -995,11 +1008,13 @@ License along with this library.  If not, see <http://www.gnu.org/licenses/>.
    or (US mail) as Mike Haertel c/o Free Software Foundation.  */

+#ifndef HYBRID_MALLOC
 /* Debugging hook for free.  */
 void (*__free_hook) (void *__ptr);
 
 /* List of blocks allocated by aligned_alloc.  */
 struct alignlist *_aligned_blocks = NULL;
+#endif
 
 /* Return memory to the heap.
    Like `_free_internal' but don't lock mutex.  */
@@ -1270,6 +1285,7 @@ free (void *ptr)
     _free_internal (ptr);
 }
 
+#ifndef HYBRID_MALLOC
 /* Define the `cfree' alias for `free'.  */
 #ifdef weak_alias
 weak_alias (free, cfree)
@@ -1280,6 +1296,7 @@ cfree (void *ptr)
   free (ptr);
 }
 #endif
+#endif
 /* Change the size of a block allocated by `malloc'.
    Copyright 1990, 1991, 1992, 1993, 1994, 1995 Free Software Foundation, Inc.
 		     Written May 1989 by Mike Haertel.
@@ -1304,8 +1321,10 @@ License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 #define min(a, b) ((a) < (b) ? (a) : (b))
 #endif
 
+#ifndef HYBRID_MALLOC
 /* Debugging hook for realloc.  */
 void *(*__realloc_hook) (void *ptr, size_t size);
+#endif
 
 /* Resize the given region to the new size, returning a pointer
    to the (possibly moved) region.  This is optimized for speed;
@@ -1549,7 +1568,9 @@ General Public License for more details.
 You should have received a copy of the GNU General Public
 License along with this library.  If not, see <http://www.gnu.org/licenses/>.  */
 
+#ifndef HYBRID_MALLOC
 void *(*__memalign_hook) (size_t size, size_t alignment);
+#endif
 
 void *
 aligned_alloc (size_t alignment, size_t size)
@@ -1638,6 +1659,8 @@ aligned_alloc (size_t alignment, size_t size)
   return result;
 }
 
+/* Note that memalign and posix_memalign are not used in Emacs.  */
+#ifndef HYBRID_MALLOC
 /* An obsolete alias for aligned_alloc, for any old libraries that use
    this alias.  */
 
@@ -1649,7 +1672,6 @@ memalign (size_t alignment, size_t size)
 
 /* If HYBRID_MALLOC is defined, we may want to use the system
    posix_memalign below.  */
-#ifndef HYBRID_MALLOC
 int
 posix_memalign (void **memptr, size_t alignment, size_t size)
 {
@@ -1689,6 +1711,7 @@ License along with this library.  If not, see <http://www.gnu.org/licenses/>.
    The author may be reached (Email) at the address [email-masked],
    or (US mail) as Mike Haertel c/o Free Software Foundation.  */
 
+#ifndef HYBRID_MALLOC
 /* Allocate SIZE bytes on a page boundary.  */
 extern void *valloc (size_t);
 
@@ -1708,6 +1731,7 @@ valloc (size_t size)
 
   return aligned_alloc (pagesize, size);
 }
+#endif /* HYBRID_MALLOC */
 
 #undef malloc
 #undef realloc
-- 
2.6.3

>From 9610f313480b8e0da810c38aee0fbc07c88b0904 Mon Sep 17 00:00:00 2001
From: Wolfgang Jenkner <[email-masked]>
Date: Sun, 22 Feb 2015 04:15:34 +0100
Subject: [PATCH 3/5] Link temacs with gnulib objects compiled with -Demacs.

* lib/Makefile.am: Add incantation to install libegnu.a.
* src/Makefile.in ($(lib)/libgnu.a): Replace with libegnu.a
(temacs$(EXEEXT)): Use it.

This is done to support HYBRID_MALLOC, since some static variables
(e.g., last_environ in putenv.c) hold pointers to memory malloced
before dumping.
---
 lib/Makefile.am | 9 +++++++++
 src/Makefile.in | 6 +++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/Makefile.am b/lib/Makefile.am
index cda9681..a1dd6a4 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -17,6 +17,15 @@ include gnulib.mk
 libgnu_a_SOURCES += openat-die.c save-cwd.c
 endif
 
+noinst_LIBRARIES += libegnu.a
+
+libegnu_a_SOURCES = $(libgnu_a_SOURCES)
+libegnu_a_LIBADD = $(patsubst %.o,e-%.o,$(libgnu_a_LIBADD))
+EXTRA_libegnu_a_SOURCES = $(EXTRA_libgnu_a_SOURCES)
+libegnu_a_SHORTNAME = e
+
+libegnu_a_CPPFLAGS = $(AM_CPPFLAGS) -Demacs
+
 .PHONY: bootstrap-clean
 
 bootstrap-clean: maintainer-clean
diff --git a/src/Makefile.in b/src/Makefile.in
index d7ad395..f1d780f 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -557,7 +557,7 @@ globals.h: gl-stamp; @true
 
 $(ALLOBJS): globals.h
 
-$(lib)/libgnu.a: $(config_h)
+$(lib)/libegnu.a: $(config_h)
 	$(MAKE) -C $(lib) all
 
 ## We have to create $(etc) here because init_cmdargs tests its
@@ -565,9 +565,9 @@ $(lib)/libgnu.a: $(config_h)
 ## This goes on to affect various things, and the emacs binary fails
 ## to start if Vinstallation_directory has the wrong value.
 temacs$(EXEEXT): $(LIBXMENU) $(ALLOBJS) \
-	         $(lib)/libgnu.a $(EMACSRES) ${charsets} ${charscript}
+	         $(lib)/libegnu.a $(EMACSRES) ${charsets} ${charscript}
 	$(AM_V_CCLD)$(CC) $(ALL_CFLAGS) $(TEMACS_LDFLAGS) $(LDFLAGS) \
-	  -o temacs $(ALLOBJS) $(lib)/libgnu.a $(W32_RES_LINK) $(LIBES)
+	  -o temacs $(ALLOBJS) $(lib)/libegnu.a $(W32_RES_LINK) $(LIBES)
 	$(MKDIR_P) $(etc)
 ifneq ($(CANNOT_DUMP),yes)
 	$(PAXCTL_if_present) -r $@
-- 
2.6.3

>From f65fff5bb5f2f2eee3b258538f1abb70eeb911c6 Mon Sep 17 00:00:00 2001
From: Rich Felker <[email-masked]>
Date: Sun, 22 Feb 2015 04:25:03 +0100
Subject: [PATCH 4/5] Add musl patch to support HYBRID_MALLOC on elf systems.

* src/unexelf.c (unexec): Define bss_sbrk_did_unexec.
* src/gmalloc.c: Adjust for HYBRID_MALLOC in the non CYGWIN case.
(__default_morecore): Here, in particular.

* configure.ac: Define HYBRID_MALLOC when unexelf.o is used.
New variable SHEAP_OBJ.
* src/Makefile.in: Use it.

Except for build system fixes this is essentially the same as

http://git.alpinelinux.org/cgit/aports/tree/testing/emacs/musl.patch?id=d6f211ec868df4657c745b8ba2bae77b2a7fb7f6

The original patch contains one unrelated hunk, which will be
committed separately.
---
 configure.ac    | 12 ++++++++++--
 src/Makefile.in |  6 +++++-
 src/gmalloc.c   |  4 ++--
 src/unexelf.c   |  5 +++++
 4 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/configure.ac b/configure.ac
index 63e354e..56e2f12 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2117,7 +2117,13 @@ case "$opsys" in
   cygwin) hybrid_malloc=yes;;
 esac
 
+if test "${system_malloc}" != yes && test "${doug_lea_malloc}" != yes \
+   && test "${UNEXEC_OBJ}" = unexelf.o && test "${hybrid_malloc}" != no; then
+  hybrid_malloc=yes
+fi
+
 GMALLOC_OBJ=
+SHEAP_OBJ=
 if test "${system_malloc}" = "yes"; then
   AC_DEFINE([SYSTEM_MALLOC], 1,
     [Define to 1 to use the system memory allocator, even if it is not
@@ -2132,6 +2138,7 @@ elif test "$hybrid_malloc" = yes; then
   GNU_MALLOC=
   GNU_MALLOC_reason="only before dumping"
   GMALLOC_OBJ=gmalloc.o
+  SHEAP_OBJ=sheap.o
   VMLIMIT_OBJ=
 else
   test "$doug_lea_malloc" != "yes" && GMALLOC_OBJ=gmalloc.o
@@ -2152,9 +2159,10 @@ else
   fi
 fi
 AC_SUBST(GMALLOC_OBJ)
+AC_SUBST(SHEAP_OBJ)
 AC_SUBST(VMLIMIT_OBJ)
 
-if test "$doug_lea_malloc" = "yes" ; then
+if test "$doug_lea_malloc" = "yes" && test "$hybrid_malloc" != yes; then
   if test "$GNU_MALLOC" = yes ; then
     GNU_MALLOC_reason="
       (Using Doug Lea's new malloc from the GNU C Library.)"
@@ -4970,7 +4978,7 @@ fi
 AC_SUBST(RALLOC_OBJ)
 
 if test "$opsys" = "cygwin"; then
-  CYGWIN_OBJ="sheap.o cygw32.o"
+  CYGWIN_OBJ="cygw32.o"
   ## Cygwin differs because of its unexec().
   PRE_ALLOC_OBJ=
   POST_ALLOC_OBJ=lastfile.o
diff --git a/src/Makefile.in b/src/Makefile.in
index f1d780f..ece8858 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -243,7 +243,10 @@ XFIXES_CFLAGS = @XFIXES_CFLAGS@
 ## widget.o if USE_X_TOOLKIT, otherwise empty.
 WIDGET_OBJ=@WIDGET_OBJ@
 
-## sheap.o if CYGWIN, otherwise empty.
+## sheap.o if HYBRID_MALLOC, otherwise empty.
+SHEAP_OBJ=@SHEAP_OBJ@
+
+## cygw32.o if CYGWIN, otherwise empty.
 CYGWIN_OBJ=@CYGWIN_OBJ@
 
 ## fontset.o fringe.o image.o if we have any window system
@@ -384,6 +387,7 @@ base_obj = dispnew.o frame.o scroll.o xdisp.o menu.o $(XMENU_OBJ) window.o \
 	region-cache.o sound.o atimer.o \
 	doprnt.o intervals.o textprop.o composite.o xml.o $(NOTIFY_OBJ) \
 	profiler.o decompress.o \
+	$(SHEAP_OBJ) \
 	$(MSDOS_OBJ) $(MSDOS_X_OBJ) $(NS_OBJ) $(CYGWIN_OBJ) $(FONT_OBJ) \
 	$(W32_OBJ) $(WINDOW_SYSTEM_OBJ) $(XGSELOBJ)
 obj = $(base_obj) $(NS_OBJC_OBJ)
diff --git a/src/gmalloc.c b/src/gmalloc.c
index 240408e..530b3b2 100644
--- a/src/gmalloc.c
+++ b/src/gmalloc.c
@@ -70,7 +70,7 @@ extern void emacs_abort (void);
 #define aligned_alloc galigned_alloc
 #define free gfree
 
-#ifdef CYGWIN
+#ifdef HYBRID_MALLOC
 extern void *bss_sbrk (ptrdiff_t size);
 extern int bss_sbrk_did_unexec;
 extern char bss_sbrk_buffer[];
@@ -1542,7 +1542,7 @@ void *
 __default_morecore (ptrdiff_t increment)
 {
   void *result;
-#if defined (CYGWIN)
+#ifdef HYBRID_MALLOC
   if (!DUMPED)
     {
       return bss_sbrk (increment);
diff --git a/src/unexelf.c b/src/unexelf.c
index c10c7f2..3d10aa0 100644
--- a/src/unexelf.c
+++ b/src/unexelf.c
@@ -227,6 +227,11 @@ unexec (const char *new_name, const char *old_name)
   off_t new_file_size;
   void *new_break;
 
+#ifdef HYBRID_MALLOC
+  extern int bss_sbrk_did_unexec;
+  bss_sbrk_did_unexec = 1;
+#endif
+
   /* Pointers to the base of the image of the two files.  */
   caddr_t old_base, new_base;
 
-- 
2.6.3

>From d042d56201d1ab00fcc405976954d0d0585c70bd Mon Sep 17 00:00:00 2001
From: Wolfgang Jenkner <[email-masked]>
Date: Sun, 22 Feb 2015 04:26:36 +0100
Subject: [PATCH 5/5] * configure.ac: Stop using mmap for buffers for FreeBSD.

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 56e2f12..af8de47 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2186,7 +2186,7 @@ fi
 
 use_mmap_for_buffers=no
 case "$opsys" in
-  cygwin|mingw32|freebsd|irix6-5) use_mmap_for_buffers=yes ;;
+  cygwin|mingw32|irix6-5) use_mmap_for_buffers=yes ;;
 esac
 
 AC_FUNC_MMAP
-- 
2.6.3