throw UnicodeError when accessing an invalid UTF-8 indice, rather than attempting to adjust it (in getindex, next, search). fix #7811

Author: vtjnash

base/ascii.jl

@@ -19,7 +19,11 @@ sizeof(s::ASCIIString) = sizeof(s.data)
 getindex(s::ASCIIString, r::Vector) = ASCIIString(getindex(s.data,r))
 getindex(s::ASCIIString, r::UnitRange{Int}) = ASCIIString(getindex(s.data,r))
 getindex(s::ASCIIString, indx::AbstractVector{Int}) = ASCIIString(s.data[indx])
-search(s::ASCIIString, c::Char, i::Integer) = c < Char(0x80) ? search(s.data,c%UInt8,i) : 0
+function search(s::ASCIIString, c::Char, i::Integer)
+    i == sizeof(s) + 1 && return 0
+    (i < 1 || i > sizeof(s)) && throw(BoundsError(s, i))
+    return c < Char(0x80) ? search(s.data,c%UInt8,i) : 0
+end
 rsearch(s::ASCIIString, c::Char, i::Integer) = c < Char(0x80) ? rsearch(s.data,c%UInt8,i) : 0
 
 function string(c::ASCIIString...)

base/dates/io.jl

@@ -102,12 +102,12 @@ slotparse(slot::Slot{Millisecond},x) = !ismatch(r"[^0-9\s]",x) ? slot.period(Bas
 slotparse(slot::Slot{DayOfWeekSlot},x) = nothing
 
 function getslot(x,slot::DelimitedSlot,df,cursor)
-    endind = first(search(x,df.trans[slot.i],cursor+1))
+    endind = first(search(x,df.trans[slot.i],nextind(x,cursor)))
     if endind == 0 # we didn't find the next delimiter
         s = x[cursor:end]
         return (endof(x)+1, isdigit(s) ? slotparse(slot,s) : default(slot.period))
     end
-    return endind+1, slotparse(slot,x[cursor:(endind-1)])
+    return nextind(x,endind), slotparse(slot,x[cursor:(endind-1)])
 end
 getslot(x,slot,df,cursor) = (cursor+slot.width, slotparse(slot,x[cursor:(cursor+slot.width-1)]))
 

base/string.jl

@@ -188,10 +188,9 @@ function search(s::AbstractString, c::Chars, i::Integer)
         return 1 <= i <= nextind(s,endof(s)) ? i :
                throw(BoundsError(s, i))
     end
-    if i < 1
+    if i < 1 || i > nextind(s,endof(s))
         throw(BoundsError(s, i))
     end
-    i = nextind(s,i-1)
     while !done(s,i)
         d, j = next(s,i)
         if d in c
@@ -655,7 +654,7 @@ prevind(s::SubString, i::Integer) = prevind(s.string, i+s.offset)-s.offset
 
 convert{T<:AbstractString}(::Type{SubString{T}}, s::T) = SubString(s, 1, endof(s))
 
-bytestring{T <: ByteString}(p::SubString{T}) = bytestring(pointer(p.string.data)+p.offset, nextind(p, p.endof)-1)
+bytestring{T <: ByteString}(p::SubString{T}) = bytestring(p.string.data[1+p.offset:p.offset+nextind(p, p.endof)-1])
 
 function getindex(s::AbstractString, r::UnitRange{Int})
     if first(r) < 1 || endof(s) < last(r)

base/utf8.jl

@@ -66,16 +66,7 @@ function next(s::UTF8String, i::Int)
     d = s.data
     b = d[i]
     if !is_utf8_start(b)
-        j = i-1
-        while 0 < j && !is_utf8_start(d[j])
-            j -= 1
-        end
-        if 0 < j && i <= j+utf8_trailing[d[j]+1] <= length(d)
-            # b is a continuation byte of a valid UTF-8 character
-            throw(UnicodeError(UTF_ERR_CONT, i, d[j]))
-        end
-        # move past 1 byte in case the data is actually Latin-1
-        return '\ufffd', i+1
+        throw(UnicodeError(UTF_ERR_INVALID_INDEX, i, d[i]))
     end
     trailing = utf8_trailing[b+1]
     if length(d) < i + trailing
@@ -123,8 +114,11 @@ function getindex(s::UTF8String, r::UnitRange{Int})
     isempty(r) && return empty_utf8
     i, j = first(r), last(r)
     d = s.data
+    if i < 1 || i > length(s.data)
+        throw(BoundsError(s, i))
+    end
     if !is_utf8_start(d[i])
-        i = nextind(s,i)
+        throw(UnicodeError(UTF_ERR_INVALID_INDEX, i, d[i]))
     end
     if j > length(d)
         throw(BoundsError())
@@ -134,9 +128,17 @@ function getindex(s::UTF8String, r::UnitRange{Int})
 end
 
 function search(s::UTF8String, c::Char, i::Integer)
-    c < Char(0x80) && return search(s.data, c%UInt8, i)
+    if i < 1 || i > sizeof(s)
+        i == sizeof(s) + 1 && return 0
+        throw(BoundsError(s, i))
+    end
+    d = s.data
+    if !is_utf8_start(d[i])
+        throw(UnicodeError(UTF_ERR_INVALID_INDEX, i, d[i]))
+    end
+    c < Char(0x80) && return search(d, c%UInt8, i)
     while true
-        i = search(s.data, first_utf8_byte(c), i)
+        i = search(d, first_utf8_byte(c), i)
         (i==0 || s[i] == c) && return i
         i = next(s,i)[2]
     end

test/strings.jl

@@ -262,6 +262,8 @@ u8str = "∀ ε > 0, ∃ δ > 0: |x-y| < δ ⇒ |f(x)-f(y)| < ε"
 
 # ascii search
 for str in [astr, Base.GenericString(astr)]
+    @test_throws BoundsError search(str, 'z', 0)
+    @test_throws BoundsError search(str, '∀', 0)
     @test search(str, 'x') == 0
     @test search(str, '\0') == 0
     @test search(str, '\u80') == 0
@@ -275,6 +277,8 @@ for str in [astr, Base.GenericString(astr)]
     @test search(str, ',', 7) == 0
     @test search(str, '\n') == 14
     @test search(str, '\n', 15) == 0
+    @test_throws BoundsError search(str, 'ε', nextind(str,endof(str))+1)
+    @test_throws BoundsError search(str, 'a', nextind(str,endof(str))+1)
 end
 
 # ascii rsearch
@@ -297,23 +301,32 @@ end
 
 # utf-8 search
 for str in (u8str, Base.GenericString(u8str))
+    @test_throws BoundsError search(str, 'z', 0)
+    @test_throws BoundsError search(str, '∀', 0)
     @test search(str, 'z') == 0
     @test search(str, '\0') == 0
     @test search(str, '\u80') == 0
     @test search(str, '∄') == 0
     @test search(str, '∀') == 1
-    @test search(str, '∀', 2) == 0
+    @test_throws UnicodeError search(str, '∀', 2)
+    @test search(str, '∀', 4) == 0
     @test search(str, '∃') == 13
-    @test search(str, '∃', 14) == 0
+    @test_throws UnicodeError search(str, '∃', 15)
+    @test search(str, '∃', 16) == 0
     @test search(str, 'x') == 26
     @test search(str, 'x', 27) == 43
     @test search(str, 'x', 44) == 0
     @test search(str, 'δ') == 17
-    @test search(str, 'δ', 18) == 33
-    @test search(str, 'δ', 34) == 0
+    @test_throws UnicodeError search(str, 'δ', 18)
+    @test search(str, 'δ', nextind(str,17)) == 33
+    @test search(str, 'δ', nextind(str,33)) == 0
     @test search(str, 'ε') == 5
-    @test search(str, 'ε', 6) == 54
-    @test search(str, 'ε', 55) == 0
+    @test search(str, 'ε', nextind(str,5)) == 54
+    @test search(str, 'ε', nextind(str,54)) == 0
+    @test search(str, 'ε', nextind(str,endof(str))) == 0
+    @test search(str, 'a', nextind(str,endof(str))) == 0
+    @test_throws BoundsError search(str, 'ε', nextind(str,endof(str))+1)
+    @test_throws BoundsError search(str, 'a', nextind(str,endof(str))+1)
 end
 
 # utf-8 rsearch