Merge pull request #1149 from kll/feature/preserve_encoding

Preserve existing file encoding if it can be easily determined otherwise use UTF8 with BOM.

Author: JakeGinnivan

src/GitVersionCore.Tests/GitVersionCore.Tests.csproj

@@ -98,6 +98,7 @@
     <Compile Include="ConfigProviderTests.cs" />
     <Compile Include="GitVersionContextTests.cs" />
     <Compile Include="Helpers\DirectoryHelper.cs" />
+    <Compile Include="IntegrationTests\FileSystemTests.cs" />
     <Compile Include="IntegrationTests\MainlineDevelopmentMode.cs" />
     <Compile Include="LogMessages.cs" />
     <Compile Include="Mocks\MockThreadSleep.cs" />

src/GitVersionCore.Tests/IntegrationTests/FileSystemTests.cs

@@ -0,0 +1,68 @@
+using System.IO;
+using System.Text;
+
+using GitVersion.Helpers;
+
+using NUnit.Framework;
+
+using Shouldly;
+
+[TestFixture]
+public class FileSystemTests
+{
+    public string TempFilePath { get; set; }
+
+    [SetUp]
+    public void CreateTempFile()
+    {
+        TempFilePath = Path.GetTempFileName();
+    }
+
+    [TearDown]
+    public void Cleanup()
+    {
+        File.Delete(TempFilePath);
+    }
+
+    [TestCase("utf-32")]
+    [TestCase("utf-32BE")]
+    [TestCase("utf-16")]
+    [TestCase("utf-16BE")]
+    [TestCase("utf-8")]
+    public void WhenFileExistsWithEncodingPreamble_EncodingIsPreservedAfterWriteAll(string encodingName)
+    {
+        var encoding = Encoding.GetEncoding(encodingName);
+
+        File.WriteAllText(TempFilePath, "(－‸ლ)", encoding);
+
+        var fileSystem = new FileSystem();
+        fileSystem.WriteAllText(TempFilePath, @"¯\(◉◡◔)/¯");
+
+        using (var stream = File.OpenRead(TempFilePath))
+        {
+            var preamble = encoding.GetPreamble();
+            var bytes = new byte[preamble.Length];
+            stream.Read(bytes, 0, preamble.Length);
+
+            bytes.ShouldBe(preamble);
+        }
+    }
+
+    [Test]
+    public void WhenFileDoesNotExist_CreateWithUTF8WithPreamble()
+    {
+        var encoding = Encoding.UTF8;
+
+        var fileSystem = new FileSystem();
+        fileSystem.WriteAllText(TempFilePath, "╚(ಠ_ಠ)=┐");
+
+        using (var stream = File.OpenRead(TempFilePath))
+        {
+            var preamble = encoding.GetPreamble();
+            var bytes = new byte[preamble.Length];
+            stream.Read(bytes, 0, preamble.Length);
+
+            bytes.ShouldBe(preamble);
+        }
+    }
+}

src/GitVersionCore.Tests/TestFileSystem.cs

@@ -7,7 +7,7 @@
 
 public class TestFileSystem : IFileSystem
 {
-    Dictionary<string, string> fileSystem = new Dictionary<string, string>();
+    Dictionary<string, byte[]> fileSystem = new Dictionary<string, byte[]>();
 
     public void Copy(string @from, string to, bool overwrite)
     {
@@ -19,7 +19,7 @@ public void Copy(string @from, string to, bool overwrite)
                 throw new IOException("File already exists");
         }
 
-        string source;
+        byte[] source;
         if (!fileSystem.TryGetValue(from, out source))
             throw new FileNotFoundException(string.Format("The source file '{0}' was not found", from), from);
 
@@ -44,19 +44,25 @@ public void Delete(string path)
 
     public string ReadAllText(string path)
     {
-        return fileSystem[path];
+        byte[] content;
+        if (!fileSystem.TryGetValue(path, out content))
+            throw new FileNotFoundException(string.Format("The file '{0}' was not found", path), path);
+
+        var encoding = EncodingHelper.DetectEncoding(content) ?? Encoding.UTF8;
+        return encoding.GetString(content);
     }
 
     public void WriteAllText(string file, string fileContents)
     {
-        if (fileSystem.ContainsKey(file))
-        {
-            fileSystem[file] = fileContents;
-        }
-        else
-        {
-            fileSystem.Add(file, fileContents);
-        }
+        var encoding = fileSystem.ContainsKey(file)
+                           ? EncodingHelper.DetectEncoding(fileSystem[file]) ?? Encoding.UTF8
+                           : Encoding.UTF8;
+        WriteAllText(file, fileContents, encoding);
+    }
+
+    public void WriteAllText(string file, string fileContents, Encoding encoding)
+    {
+        fileSystem[file] = encoding.GetBytes(fileContents);
     }
 
     public IEnumerable<string> DirectoryGetFiles(string directory, string searchPattern, SearchOption searchOption)
@@ -74,7 +80,7 @@ public Stream OpenRead(string path)
         if (fileSystem.ContainsKey(path))
         {
             var content = fileSystem[path];
-            return new MemoryStream(Encoding.UTF8.GetBytes(content));
+            return new MemoryStream(content);
         }
 
         throw new FileNotFoundException("File not found.", path);
@@ -84,11 +90,11 @@ public void CreateDirectory(string path)
     {
         if (fileSystem.ContainsKey(path))
         {
-            fileSystem[path] = "";
+            fileSystem[path] = new byte[0];
         }
         else
         {
-            fileSystem.Add(path, "");
+            fileSystem.Add(path, new byte[0]);
         }
     }
 

src/GitVersionCore/GitVersionCore.csproj

@@ -126,6 +126,7 @@
     <Compile Include="GitVersionCache.cs" />
     <Compile Include="GitVersionCacheKey.cs" />
     <Compile Include="GitVersionCacheKeyFactory.cs" />
+    <Compile Include="Helpers\EncodingHelper.cs" />
     <Compile Include="Helpers\FileSystem.cs" />
     <Compile Include="Helpers\IFileSystem.cs" />
     <Compile Include="Helpers\IThreadSleep.cs" />

src/GitVersionCore/Helpers/EncodingHelper.cs

@@ -0,0 +1,102 @@
+namespace GitVersion.Helpers
+{
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Linq;
+    using System.Text;
+
+    public static class EncodingHelper
+    {
+        private static IList<Encoding> EncodingsWithPreambles;
+
+        private static int MaxPreambleLength;
+
+        /// <summary>
+        /// Detects the encoding of a file if and only if it includes a preamble .
+        /// </summary>
+        /// <param name="filename">The file name to check the encoding of.</param>
+        /// <returns>The encoding of the file if it has a preamble otherwise null.</returns>
+        public static Encoding DetectEncoding(string filename)
+        {
+            if (!File.Exists(filename))
+            {
+                return null;
+            }
+
+            if (EncodingsWithPreambles == null)
+            {
+                ScanEncodings();
+            }
+
+            using (var stream = File.OpenRead(filename))
+            {
+                // No bytes? No encoding!
+                if (stream.Length == 0)
+                {
+                    return null;
+                }
+
+                // Read the minimum amount necessary.
+                var length = stream.Length > MaxPreambleLength ? MaxPreambleLength : stream.Length;
+
+                var bytes = new byte[length];
+                stream.Read(bytes, 0, (int)length);
+                return DetectEncoding(bytes);
+            }
+        }
+
+        /// <summary>
+        /// Returns the first encoding where all the preamble bytes match exactly.
+        /// </summary>
+        /// <param name="bytes">The bytes to check for a matching preamble.</param>
+        /// <returns>The encoding that has a matching preamble or null if one was not found.</returns>
+        public static Encoding DetectEncoding(IList<byte> bytes)
+        {
+            if (bytes == null || bytes.Count == 0)
+            {
+                return null;
+            }
+
+            if (EncodingsWithPreambles == null)
+            {
+                ScanEncodings();
+            }
+
+            return EncodingsWithPreambles.FirstOrDefault(encoding => PreambleMatches(encoding, bytes));
+        }
+
+        /// <summary>
+        /// Returns an ordered list of encodings that have preambles ordered by the length of the
+        /// preamble longest to shortest. This prevents a short preamble masking a longer one
+        /// later in the list.
+        /// </summary>
+        /// <returns>An ordered list of encodings and corresponding preambles.</returns>
+        private static void ScanEncodings()
+        {
+            EncodingsWithPreambles = (from info in Encoding.GetEncodings()
+                                          let encoding = info.GetEncoding()
+                                          let preamble = encoding.GetPreamble()
+                                          where preamble.Length > 0
+                                          orderby preamble.Length descending
+                                          select encoding).ToList();
+            
+            var encodingWithLongestPreamble = EncodingsWithPreambles.FirstOrDefault();
+            MaxPreambleLength = encodingWithLongestPreamble == null ? 0 : encodingWithLongestPreamble.GetPreamble().Length;
+        }
+
+        /// <summary>
+        /// Verifies that all bytes of an encoding's preamble are present at the beginning of some sample data.
+        /// </summary>
+        /// <param name="encoding">The encoding to check against.</param>
+        /// <param name="data">The data to test.</param>
+        /// <returns>A boolean indicating if a preamble match was found.</returns>
+        private static bool PreambleMatches(Encoding encoding, IList<byte> data)
+        {
+            var preamble = encoding.GetPreamble();
+            if (preamble.Length > data.Count)
+                return false;
+
+            return !preamble.Where((preambleByte, index) => data[index] != preambleByte).Any();
+        }
+    }
+}

src/GitVersionCore/Helpers/FileSystem.cs

@@ -4,6 +4,7 @@ namespace GitVersion.Helpers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Text;
 
     public class FileSystem : IFileSystem
     {
@@ -36,7 +37,15 @@ public string ReadAllText(string path)
 
         public void WriteAllText(string file, string fileContents)
         {
-            File.WriteAllText(file, fileContents);
+            // Opinionated decision to use UTF8 with BOM when creating new files or when the existing
+            // encoding was not easily detected due to the file not having an encoding preamble.
+            var encoding = EncodingHelper.DetectEncoding(file) ?? Encoding.UTF8;
+            WriteAllText(file, fileContents, encoding);
+        }
+
+        public void WriteAllText(string file, string fileContents, Encoding encoding)
+        {
+            File.WriteAllText(file, fileContents, encoding);
         }
 
         public IEnumerable<string> DirectoryGetFiles(string directory, string searchPattern, SearchOption searchOption)

src/GitVersionCore/Helpers/IFileSystem.cs

@@ -2,6 +2,7 @@ namespace GitVersion.Helpers
 {
     using System.Collections.Generic;
     using System.IO;
+    using System.Text;
 
     public interface IFileSystem
     {
@@ -11,6 +12,7 @@ public interface IFileSystem
         void Delete(string path);
         string ReadAllText(string path);
         void WriteAllText(string file, string fileContents);
+        void WriteAllText(string file, string fileContents, Encoding encoding);
         IEnumerable<string> DirectoryGetFiles(string directory, string searchPattern, SearchOption searchOption);
         Stream OpenWrite(string path);
         Stream OpenRead(string path);

src/GitVersionTask/AssemblyInfoBuilder/UpdateAssemblyInfo.cs

@@ -6,6 +6,7 @@
     using System.Text;
 
     using GitVersion;
+    using GitVersion.Helpers;
 
     using Microsoft.Build.Framework;
 
@@ -95,14 +96,15 @@ void CreateTempAssemblyInfo(VersionVariables versionVariables)
             }
 
             var assemblyInfo = assemblyInfoBuilder.GetAssemblyInfoText(versionVariables, RootNamespace).Trim();
+            var encoding = EncodingHelper.DetectEncoding(AssemblyInfoTempFilePath) ?? Encoding.UTF8;
 
             // We need to try to read the existing text first if the file exists and see if it's the same
             // This is to avoid writing when there's no differences and causing a rebuild
             try
             {
                 if (File.Exists(AssemblyInfoTempFilePath))
                 {
-                    var content = File.ReadAllText(AssemblyInfoTempFilePath, Encoding.UTF8).Trim();
+                    var content = File.ReadAllText(AssemblyInfoTempFilePath, encoding).Trim();
                     if (string.Equals(assemblyInfo, content, StringComparison.Ordinal))
                     {
                         return; // nothign to do as the file matches what we'd create
@@ -114,7 +116,7 @@ void CreateTempAssemblyInfo(VersionVariables versionVariables)
                 // Something happened reading the file, try to overwrite anyway
             }
 
-            File.WriteAllText(AssemblyInfoTempFilePath, assemblyInfo, Encoding.UTF8);
+            File.WriteAllText(AssemblyInfoTempFilePath, assemblyInfo, encoding);
         }
     }
 }