diff --git a/tools/VersionController/Program.cs b/tools/VersionController/Program.cs
index 6e256a25493d..2d36aae9c375 100644
--- a/tools/VersionController/Program.cs
+++ b/tools/VersionController/Program.cs
@@ -34,6 +34,8 @@ public class Program
         private static List<string> _projectDirectories, _outputDirectories;
         private static string _rootDirectory, _moduleNameFilter;
 
+        private const string Psd1NameExtension = ".psd1";
+
         private static IList<string> ExceptionFileNames = new List<string>()
         {
             "AssemblyVersionConflict.csv",
@@ -70,7 +72,7 @@ public static void Main(string[] args)
             _moduleNameFilter = string.Empty;
             if (args != null && args.Length > 1)
             {
-                _moduleNameFilter = args[1] + ".psd1";
+                _moduleNameFilter = args[1] + Psd1NameExtension;
             }
 
             ConsolidateExceptionFiles(exceptionsDirectory);
@@ -175,6 +177,10 @@ private static void BumpVersions()
                 var file = File.ReadAllLines(miniVersionFile);
                 var header = file.First();
                 var lines = file.Skip(1).Where(c => !string.IsNullOrEmpty(c));
+
+                var bumpingModule = _moduleNameFilter.Replace(Psd1NameExtension, "");
+                List<string> _minimalVersionContent = new List<string>() { header };
+
                 foreach (var line in lines)
                 {
                     var cols = line.Split(",").Select(c => c.StartsWith("\"") ? c.Substring(1) : c)
@@ -183,11 +189,17 @@ private static void BumpVersions()
                     if (cols.Length >= 2)
                     {
                         _minimalVersion.Add(cols[0], new AzurePSVersion(cols[1]));
+
+                        // Bump one module, only remove its minimal version from MinimalVersion.csv content
+                        if (!string.IsNullOrEmpty(bumpingModule) && !cols[0].Equals(bumpingModule))
+                        {
+                            _minimalVersionContent.Add(line);
+                        }
                     }
                 }
 
                 // Clean MinimalVersion.csv
-                File.WriteAllLines(Path.Combine(_rootDirectory, @"tools\VersionController", "MinimalVersion.csv"), new string[]{ header});
+                File.WriteAllLines(Path.Combine(_rootDirectory, @"tools\VersionController", "MinimalVersion.csv"), _minimalVersionContent.ToArray());
             }
 
             //Make Az.Accounts as the last module to calculate