Fix negative Fixnum encoding

Author: mtodd

lib/net/ber/core_ext/fixnum.rb

@@ -42,17 +42,21 @@ def to_ber_internal
     # Looks for the first byte in the fixnum that is not all zeroes. It does
     # this by masking one byte after another, checking the result for bits
     # that are left on.
-    size = Net::BER::MAX_FIXNUM_SIZE
-    while size > 1
-      break if (self & (0xff << (size - 1) * 8)) > 0
-      size -= 1
-    end
+    val   = (self < 0) ? ~self : self
+    size  = 1
+    size += 1 until (val >> (size * 8)).zero?
 
     # for positive integers, if most significant bit in an octet is set to one,
     # pad the result (otherwise it's decoded as a negative value)
     # See section 8.5 of ITU-T X.690:
     # http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
-    if self > 0 && (self & (0b10000000 << (size - 1) * 8)) > 0
+    if self > 0 && (self & (0x80 << (size - 1) * 8)) > 0
+      size += 1
+    end
+
+    # and for negative integers, pad if the most significant bit in the octet
+    # is not set to one.
+    if self < 0 && (self & (0x80 << (size - 1) * 8)) == 0
       size += 1
     end
 

test/ber/test_ber.rb

@@ -45,9 +45,9 @@ def test_false
     5_000_000_000 => "\x02\x05\x01\x2a\x05\xF2\x00",
 
     # negatives
-    # -1          => "\x02\x01\xFF",
-    # -127        => "\x02\x01\x81",
-    # -128        => "\x02\x01\x80"
+    -1          => "\x02\x01\xFF",
+    -127        => "\x02\x01\x81",
+    -128        => "\x02\x01\x80"
   }.each do |number, expected_encoding|
     define_method "test_encode_#{number}" do
       assert_equal expected_encoding.b, number.to_ber