Fix severe slowdown on certain strings

The default "fast" string hash function samples only a few positions in
a string, the remaining bytes don't affect the function's result. The
function performs well for short strings; however long strings can yield
extremely high collision rates.

An adaptive schema was implemented. Two hash functions are used
simultaneously. A bucket is picked based on the output of the fast hash
function. If an item is to be inserted in a collision chain longer than
a certain threshold, another bucket is picked based on the stronger hash
function. Since two hash functions are used simultaneously, insert
should consider two buckets. The second bucket is often NOT considered
thanks to the bloom filter. The filter is rebuilt during GC cycle.

Author: funny-falcon

src/Makefile

@@ -148,6 +148,10 @@ XCFLAGS=
 # everything. Use only if you suspect a problem with LuaJIT itself.
 #XCFLAGS+= -DLUA_USE_ASSERT
 #
+# Switch to harder (and slower) hash function when a collision chain in
+# the string hash table exceeds certain length.
+XCFLAGS+= -DLUAJIT_SMART_STRINGS=1
+#
 ##############################################################################
 # You probably don't need to change anything below this line!
 ##############################################################################

src/lj_cparse.c

@@ -1058,6 +1058,17 @@ static void cp_decl_gccattribute(CPState *cp, CPDecl *decl)
   while (cp->tok != ')') {
     if (cp->tok == CTOK_IDENT) {
       GCstr *attrstr = cp->str;
+#if LUAJIT_SMART_STRINGS
+      /*
+       * Sadly, several option __name__-s exceed 12 bytes hence they
+       * could've been interned using the full hash.  Strip "__" to stay
+       * within limits.
+       */
+      const char *c = strdata(cp->str);
+      if (attrstr->len > 12 && c[0]=='_' && c[1]=='_' && c[2] != '_' &&
+	  c[attrstr->len-2]=='_' && c[attrstr->len-1]=='_')
+	attrstr = lj_str_new(cp->L, c+2, attrstr->len-4);
+#endif
       cp_next(cp);
       switch (attrstr->hash) {
       case H_(64a9208e,8ce14319): case H_(8e6331b2,95a282af):  /* aligned */

src/lj_gc.c

@@ -409,6 +409,33 @@ static GCRef *gc_sweep(global_State *g, GCRef *p, uint32_t lim)
   return p;
 }
 
+/* Full sweep of a string chain. */
+static GCRef *gc_sweep_str_chain(global_State *g, GCRef *p)
+{
+  /* Mask with other white and LJ_GC_FIXED. Or LJ_GC_SFIXED on shutdown. */
+  int ow = otherwhite(g);
+  GCobj *o;
+  while ((o = gcref(*p)) != NULL) {
+    if (((o->gch.marked ^ LJ_GC_WHITES) & ow)) {  /* Black or current white? */
+      lua_assert(!isdead(g, o) || (o->gch.marked & LJ_GC_FIXED));
+      makewhite(g, o);  /* Value is alive, change to the current white. */
+#if LUAJIT_SMART_STRINGS
+      if (strsmart(&o->str)) {
+	/* must match lj_str_new */
+	bloomset(g->strbloom.new[0], o->str.hash >> (sizeof(o->str.hash)*8-6));
+	bloomset(g->strbloom.new[1], o->str.strflags);
+      }
+#endif
+      p = &o->gch.nextgc;
+    } else {  /* Otherwise value is dead, free it. */
+      lua_assert(isdead(g, o) || ow == LJ_GC_SFIXED);
+      setgcrefr(*p, o->gch.nextgc);
+      lj_str_free(g, &o->str);
+    }
+  }
+  return p;
+}
+
 /* Check whether we can clear a key or a value slot from a table. */
 static int gc_mayclear(cTValue *o, int val)
 {
@@ -622,12 +649,21 @@ static size_t gc_onestep(lua_State *L)
     atomic(g, L);
     g->gc.state = GCSsweepstring;  /* Start of sweep phase. */
     g->gc.sweepstr = 0;
+#if LUAJIT_SMART_STRINGS
+    g->strbloom.new[0] = 0;
+    g->strbloom.new[1] = 0;
+#endif
     return 0;
   case GCSsweepstring: {
     GCSize old = g->gc.total;
-    gc_fullsweep(g, &g->strhash[g->gc.sweepstr++]);  /* Sweep one chain. */
-    if (g->gc.sweepstr > g->strmask)
+    gc_sweep_str_chain(g, &g->strhash[g->gc.sweepstr++]);  /* Sweep one chain. */
+    if (g->gc.sweepstr > g->strmask) {
       g->gc.state = GCSsweep;  /* All string hash chains sweeped. */
+#if LUAJIT_SMART_STRINGS
+      g->strbloom.cur[0] = g->strbloom.new[0];
+      g->strbloom.cur[1] = g->strbloom.new[1];
+#endif
+    }
     lua_assert(old >= g->gc.total);
     g->gc.estimate -= old - g->gc.total;
     return GCSWEEPCOST;

src/lj_obj.h

@@ -291,7 +291,7 @@ typedef const TValue cTValue;
 typedef struct GCstr {
   GCHeader;
   uint8_t reserved;	/* Used by lexer for fast lookup of reserved words. */
-  uint8_t unused;
+  uint8_t strflags;	/* If LUAJIT_SMART_STRINGS: hash function used(+). */
   MSize hash;		/* Hash of string. */
   MSize len;		/* Size of string. */
 } GCstr;
@@ -301,6 +301,7 @@ typedef struct GCstr {
 #define strdatawr(s)	((char *)((s)+1))
 #define strVdata(o)	strdata(strV(o))
 #define sizestring(s)	(sizeof(struct GCstr)+(s)->len+1)
+#define strsmart(s)	((s)->strflags >= 0xc0)
 
 /* -- Userdata object ----------------------------------------------------- */
 
@@ -595,6 +596,12 @@ typedef struct global_State {
   GCRef *strhash;	/* String hash table (hash chain anchors). */
   MSize strmask;	/* String hash mask (size of hash table - 1). */
   MSize strnum;		/* Number of strings in hash table. */
+#if LUAJIT_SMART_STRINGS
+  struct {
+    BloomFilter cur[2];
+    BloomFilter new[2];
+  } strbloom;
+#endif
   lua_Alloc allocf;	/* Memory allocator. */
   void *allocd;		/* Memory allocator data. */
   GCState gc;		/* Garbage collector. */

src/lj_str.c

@@ -130,6 +130,40 @@ void lj_str_resize(lua_State *L, MSize newmask)
   g->strhash = newhash;
 }
 
+#if LUAJIT_SMART_STRINGS
+static LJ_AINLINE uint32_t
+lj_fullhash(const uint8_t *v, MSize len)
+{
+  MSize a = 0, b = 0;
+  MSize c = 0xcafedead;
+  MSize d = 0xdeadbeef;
+  MSize h = len;
+  lua_assert(len >= 12);
+  for(; len>8; len-=8, v+=8) {
+    a ^= lj_getu32(v);
+    b ^= lj_getu32(v+4);
+    c += a;
+    d += b;
+    a = lj_rol(a, 5) - d;
+    b = lj_rol(b, 7) - c;
+    c = lj_rol(c, 24) ^ a;
+    d = lj_rol(d, 1) ^ b;
+  }
+  a ^= lj_getu32(v+len-8);
+  b ^= lj_getu32(v+len-4);
+  c += b; c -= lj_rol(a, 9);
+  d += a; d -= lj_rol(b, 18);
+  h -= lj_rol(a^b,7);
+  h += c; h += lj_rol(d,13);
+  d ^= c;  d -= lj_rol(c,25);
+  h ^= d; h -= lj_rol(d,16);
+  c ^= h; c -= lj_rol(h,4);
+  d ^= c;  d -= lj_rol(c,14);
+  h ^= d; h -= lj_rol(d,24);
+  return h;
+}
+#endif
+
 /* Intern a string and return string object. */
 GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
 {
@@ -138,6 +172,10 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
   GCobj *o;
   MSize len = (MSize)lenx;
   MSize a, b, h = len;
+  uint8_t strflags = 0;
+#if LUAJIT_SMART_STRINGS
+  unsigned collisions = 0;
+#endif
   if (lenx >= LJ_MAX_STR)
     lj_err_msg(L, LJ_ERR_STROV);
   g = G(L);
@@ -161,34 +199,123 @@ GCstr *lj_str_new(lua_State *L, const char *str, size_t lenx)
   h ^= b; h -= lj_rol(b, 16);
   /* Check if the string has already been interned. */
   o = gcref(g->strhash[h & g->strmask]);
+#if LUAJIT_SMART_STRINGS
+/*
+** The default "fast" string hash function samples only a few positions
+** in a string, the remaining bytes don't affect the function's result.
+** The function performs well for short strings; however long strings
+** can yield extremely high collision rates.
+**
+** An adaptive schema was implemented. Two hash functions are used
+** simultaneously. A bucket is picked based on the output of the fast
+** hash function. If an item is to be inserted in a collision chain
+** longer than a certain threshold, another bucket is picked based on
+** the stronger hash function. Since two hash functions are used
+** simultaneously, insert should consider two buckets. The second bucket
+** is often NOT considered thanks to the bloom filter. The filter is
+** rebuilt during GC cycle.
+**
+** Parameters below were tuned on a set of benchmarks. Max_collisions is
+** also backed by theory: the expected maximum length of a collision
+** chain in a hash table with the fill factor of 1.0 is
+** O(log(N)/log(log(N))), assuming uniformly distributed random keys.
+** The upper bound for N=65,000 is 10, hence 40 is a clear indication of
+** an anomaly.
+**/
+#define max_collisions 40
+#define inc_collision_soft() (collisions++)
+/* If different strings yield the same hash sum, grow counter faster. */
+#define inc_collision_hard() (collisions+=1+(len>>4), 1)
+#else
+#define inc_collision_hard() (1)
+#define inc_collision_soft()
+#endif
   if (LJ_LIKELY((((uintptr_t)str+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4)) {
     while (o != NULL) {
       GCstr *sx = gco2str(o);
-      if (sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) {
+      if (sx->hash == h && sx->len == len && inc_collision_hard() &&
+                      str_fastcmp(str, strdata(sx), len) == 0) {
 	/* Resurrect if dead. Can only happen with fixstring() (keywords). */
 	if (isdead(g, o)) flipwhite(o);
 	return sx;  /* Return existing string. */
       }
       o = gcnext(o);
+      inc_collision_soft();
     }
   } else {  /* Slow path: end of string is too close to a page boundary. */
     while (o != NULL) {
       GCstr *sx = gco2str(o);
-      if (sx->len == len && memcmp(str, strdata(sx), len) == 0) {
+      if (sx->hash == h && sx->len == len && inc_collision_hard() &&
+                      memcmp(str, strdata(sx), len) == 0) {
 	/* Resurrect if dead. Can only happen with fixstring() (keywords). */
 	if (isdead(g, o)) flipwhite(o);
 	return sx;  /* Return existing string. */
       }
       o = gcnext(o);
+      inc_collision_soft();
     }
   }
+#if LUAJIT_SMART_STRINGS
+  /* "Fast" hash function consumes all bytes of a string <= 12 bytes. */
+  if (len > 12) {
+    /*
+    ** The bloom filter is keyed with the high 12 bits of the fast
+    ** hash sum. The filter is rebuilt during GC cycle. It's beneficial
+    ** to have these bits readily available and avoid hash sum
+    ** recalculation during GC. High 6 bits are included in the "full"
+    ** hash sum, and bits 19-25 are stored in s->strflags.
+    **/
+    int search_fullh =
+       bloomtest(g->strbloom.cur[0], h>>(sizeof(h)*8- 6)) != 0 &&
+       bloomtest(g->strbloom.cur[1], h>>(sizeof(h)*8-12)) != 0;
+    if (LJ_UNLIKELY(search_fullh || collisions > max_collisions)) {
+      MSize fh = lj_fullhash((const uint8_t*)str, len);
+#define high6mask ((~(MSize)0)<<(sizeof(MSize)*8-6))
+      fh = (fh >> 6) | (h & high6mask);
+      if (search_fullh) {
+	/* Recheck if the string has already been interned with "harder" hash. */
+	o = gcref(g->strhash[fh & g->strmask]);
+	if (LJ_LIKELY((((uintptr_t)str+len-1) & (LJ_PAGESIZE-1)) <= LJ_PAGESIZE-4)) {
+	  while (o != NULL) {
+	    GCstr *sx = gco2str(o);
+	    if (sx->hash == fh && sx->len == len && str_fastcmp(str, strdata(sx), len) == 0) {
+	      /* Resurrect if dead. Can only happen with fixstring() (keywords). */
+	      if (isdead(g, o)) flipwhite(o);
+	      return sx;  /* Return existing string. */
+	    }
+	    o = gcnext(o);
+	  }
+	} else {  /* Slow path: end of string is too close to a page boundary. */
+	  while (o != NULL) {
+	    GCstr *sx = gco2str(o);
+	    if (sx->hash == fh && sx->len == len && memcmp(str, strdata(sx), len) == 0) {
+	      /* Resurrect if dead. Can only happen with fixstring() (keywords). */
+	      if (isdead(g, o)) flipwhite(o);
+	      return sx;  /* Return existing string. */
+	    }
+	    o = gcnext(o);
+	  }
+	}
+      }
+      if (collisions > max_collisions) {
+	strflags = 0xc0 | ((h>>(sizeof(h)*8-12))&0x3f);
+	bloomset(g->strbloom.cur[0], h>>(sizeof(h)*8- 6));
+	bloomset(g->strbloom.cur[1], h>>(sizeof(h)*8-12));
+	bloomset(g->strbloom.new[0], h>>(sizeof(h)*8- 6));
+	bloomset(g->strbloom.new[1], h>>(sizeof(h)*8-12));
+	h = fh;
+      }
+    }
+  }
+#endif
   /* Nope, create a new string. */
   s = lj_mem_newt(L, sizeof(GCstr)+len+1, GCstr);
   newwhite(g, s);
   s->gct = ~LJ_TSTR;
   s->len = len;
   s->hash = h;
   s->reserved = 0;
+  s->strflags = strflags;
   memcpy(strdatawr(s), str, len);
   strdatawr(s)[len] = '\0';  /* Zero-terminate string. */
   /* Add it to string hash table. */