new Char: up to 4 zero-padded UTF-8 bytes; basic change passes tests

Author: StefanKarpinski

base/char.jl

@@ -1,8 +1,25 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-convert(::Type{Char}, x::UInt32) = reinterpret(Char, x)
+function convert(::Type{UInt32}, c::Char)
+    u = reinterpret(UInt32, c)
+    u $= ifelse(
+        u <= 0x0000ffff,
+        ifelse(u <= 0x000000ff, 0x00000000, 0x0000c080),
+        ifelse(u <= 0x00ffffff, 0x00e08080, 0xf0808080),
+    )
+    ((u & 0x000000ff) >> 0) $ ((u & 0x0000ff00) >> 2) $
+    ((u & 0x00ff0000) >> 4) $ ((u & 0xff000000) >> 6)
+end
+
+function convert(::Type{Char}, u::UInt32)
+    u < 0x00200000 || throw(InexactError())
+    c = (u & 0x3f) | ((u << 2) & 0x3f00) | ((u << 4) & 0x3f0000) | ((u << 6) & 0x3f000000)
+    reinterpret(Char, ifelse(u <= 0x7f, u,
+        c | ifelse(u <= 0x000007ff, 0x0000c080,
+            ifelse(u <= 0x0000ffff, 0x00e08080, 0xf0808080))))
+end
+
 convert(::Type{Char}, x::Number) = Char(UInt32(x))
-convert(::Type{UInt32}, x::Char) = reinterpret(UInt32, x)
 convert{T<:Number}(::Type{T}, x::Char) = convert(T, UInt32(x))
 
 rem{T<:Number}(x::Char, ::Type{T}) = rem(UInt32(x), T)
@@ -29,11 +46,11 @@ done(c::Char, state) = state
 isempty(c::Char) = false
 in(x::Char, y::Char) = x == y
 
-==(x::Char, y::Char) = UInt32(x) == UInt32(y)
+==(x::Char, y::Char) = eq_int(unbox(Char,x), unbox(Char,y))
 ==(x::Char, y::Integer) = UInt32(x) == y
 ==(x::Integer, y::Char) = x == UInt32(y)
 
-isless(x::Char, y::Char)    = isless(UInt32(x), UInt32(y))
+isless(x::Char, y::Char)    = ult_int(unbox(Char,x), unbox(Char,y))
 isless(x::Char, y::Integer) = isless(UInt32(x), y)
 isless(x::Integer, y::Char) = isless(x, UInt32(y))
 
@@ -42,8 +59,6 @@ isless(x::Integer, y::Char) = isless(x, UInt32(y))
 +(x::Char, y::Integer) = Char(Int32(x) + Int32(y))
 +(x::Integer, y::Char) = y + x
 
-bswap(x::Char) = Char(bswap(UInt32(x)))
-
 print(io::IO, c::Char) = (write(io, c); nothing)
 
 const hex_chars = UInt8['0':'9';'a':'z']

base/io.jl

@@ -70,25 +70,15 @@ function write(s::IO, a::AbstractArray)
     return nb
 end
 
-function write(s::IO, ch::Char)
-    c = reinterpret(UInt32, ch)
-    if c < 0x80
-        return write(s, c%UInt8)
-    elseif c < 0x800
-        return (write(s, (( c >> 6          ) | 0xC0)%UInt8)) +
-               (write(s, (( c        & 0x3F ) | 0x80)%UInt8))
-    elseif c < 0x10000
-        return (write(s, (( c >> 12         ) | 0xE0)%UInt8)) +
-               (write(s, (((c >> 6)  & 0x3F ) | 0x80)%UInt8)) +
-               (write(s, (( c        & 0x3F ) | 0x80)%UInt8))
-    elseif c < 0x110000
-        return (write(s, (( c >> 18         ) | 0xF0)%UInt8)) +
-               (write(s, (((c >> 12) & 0x3F ) | 0x80)%UInt8)) +
-               (write(s, (((c >> 6)  & 0x3F ) | 0x80)%UInt8)) +
-               (write(s, (( c        & 0x3F ) | 0x80)%UInt8))
-    else
-        return write(s, '\ufffd')
+function write(io::IO, c::Char)
+    u = bswap(reinterpret(UInt32, c))
+    n = 24 & trailing_zeros(u)
+    u >>= n
+    while true
+        write(io, u % UInt8)
+        0 < (u >>= 8) || break
     end
+    4 - (n >> 3)
 end
 
 function write(s::IO, p::Ptr, n::Integer)
@@ -144,22 +134,22 @@ function read!{T}(s::IO, a::Array{T})
 end
 
 function read(s::IO, ::Type{Char})
-    ch = read(s, UInt8)
-    if ch < 0x80
-        return Char(ch)
-    end
-
-    # mimic utf8.next function
-    trailing = Base.utf8_trailing[ch+1]
-    c::UInt32 = 0
-    for j = 1:trailing
-        c += ch
-        c <<= 6
-        ch = read(s, UInt8)
+    b0 = read(s, UInt8)
+    n = leading_ones(b0)
+    c = UInt32(b0)
+    2 <= n <= 4 || return reinterpret(Char, c)
+    mark(s)
+    while n > 1
+        b = read(s, UInt8)
+        if b & 0xc0 != 0x80
+            reset(s)
+            return reinterpret(Char, UInt32(b0))
+        end
+        c <<= 8
+        c |= b
+        n -= 1
     end
-    c += ch
-    c -= Base.utf8_offset[trailing+1]
-    Char(c)
+    return reinterpret(Char, c)
 end
 
 function readuntil(s::IO, delim::Char)

base/unicode/types.jl

@@ -28,7 +28,7 @@ immutable UTF32String <: DirectIndexString
         new(data)
     end
 end
-UTF32String(data::Vector{Char}) = UTF32String(reinterpret(UInt32, data))
+UTF32String(data::Vector{Char}) = UTF32String(map(UInt32, data))
 
 isvalid{T<:Union{ASCIIString,UTF8String,UTF16String,UTF32String}}(str::T) = isvalid(T, str.data)
 isvalid{T<:Union{ASCIIString,UTF8String,UTF16String,UTF32String}}(::Type{T}, str::T) = isvalid(T, str.data)

src/ast.c

@@ -392,8 +392,12 @@ static jl_value_t *scm_to_julia_(value_t e, int eo)
         }
     }
     if (iscprim(e) && cp_class((cprim_t*)ptr(e))==wchartype) {
-        jl_value_t *wc =
-            jl_box32(jl_char_type, *(int32_t*)cp_data((cprim_t*)ptr(e)));
+        uint32_t u = *(uint32_t*)cp_data((cprim_t*)ptr(e));
+        if (u > 0x7f) {
+            u = (u <= 0x000007ff ? 0x0000c080 : u <= 0x0000ffff ? 0x00e08080 : 0xf0808080) |
+                (u & 0x3f) | ((u << 2) & 0x3f00) | ((u << 4) & 0x3f0000) | ((u << 6) & 0x3f000000);
+        }
+        jl_value_t *wc = jl_box32(jl_char_type, u);
         return wc;
     }
     if (iscvalue(e) && cv_class((cvalue_t*)ptr(e)) == jvtype) {

test/char.jl

@@ -9,9 +9,6 @@
 # This is current behavior, but it seems incorrect
 @test getindex('a',1,1,1) == 'a'
 @test_throws BoundsError getindex('a',1,1,2)
-# bswap of a Char should be removed, only the underlying codeunit (UInt32)
-# should be swapped
-@test bswap('\U10200') == '\U20100'
 
 let
 

test/strings/basic.jl

@@ -268,7 +268,7 @@ c = Char['0','\0']
 d = UTF32String(c)
 @test d == "0"
 c[1] = 'A'
-@test d == "A"
+@test d == "0"
 
 # iteration
 @test [c for c in "ḟøøƀäṙ"] == ['ḟ', 'ø', 'ø', 'ƀ', 'ä', 'ṙ']