From d8953f5744a0033b6e75e6ee5ff08f3b8b09043a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Sep 2018 13:26:42 -0600 Subject: [PATCH 1/7] Compress the entries in the modified paths database when mounting This will search the modified paths for a parent folder and if one exists it will remove the entry from the modified paths list --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 28 +++++++++++++++++++ .../Common/ModifiedPathsDatabaseTests.cs | 26 ++++++++++++++++- .../FileSystemCallbacks.cs | 3 ++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 5a7e34c43..9fef5318b 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -52,6 +52,34 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } + public void Compress(ITracer tracer) + { + int startingCount = this.modifiedPaths.Count; + using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + { + foreach (var item in this.modifiedPaths) + { + int pathSeparatorIndex = item.IndexOf('/'); + while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + { + string folder = item.Substring(0, pathSeparatorIndex + 1); + if (this.modifiedPaths.Contains(folder)) + { + this.modifiedPaths.TryRemove(item); + break; + } + + pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); + } + } + + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(startingCount), startingCount); + metadata.Add("EndCount", this.modifiedPaths.Count); + activity.Stop(metadata); + } + } + public bool Contains(string path, bool isFolder) { string entry = this.NormalizeEntryString(path, isFolder); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 663a76bb0..59b52e921 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -1,6 +1,7 @@ using GVFS.Common; using GVFS.Tests.Should; using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; using System; @@ -17,6 +18,17 @@ public class ModifiedPathsDatabaseTests private const string ExistingEntries = @"A file.txt A dir/file2.txt A dir1/dir2/file3.txt +"; + private const string EntriesToCompress = @"A file.txt +A dir/file2.txt +A dir/dir3/dir4/ +A dir1/dir2/file3.txt +A dir/ +A dir1/dir2/ +A dir1/file.txt +A dir1/dir2/dir3/dir4/dir5/ +A dir/dir2/file3.txt +A dir/dir4/dir5/ "; [TestCase] @@ -119,7 +131,19 @@ public void EntryNotAddedIfParentDirectoryExists() modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); } - private static void TestAddingPath(string path, bool isFolder = false) + [TestCase] + public void CompressEntries() + { + ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); + mpd.Compress(new MockTracer()); + mpd.Count.ShouldEqual(4); + mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); + mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } + + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d7c0d57b4..44bce1687 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,6 +89,9 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); From d28270e8ea06f8f47a4ad827df3f6f04545663c9 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:15:26 -0600 Subject: [PATCH 2/7] Fix test for modified paths that was broken --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f8e42331e..f2005745e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -37,11 +37,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", $"A {FolderToCreateOutsideRepo}/", - $"A {FolderToDelete}/CreateCommonAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonVersionHeader.bat", - $"A {FolderToDelete}/RunFunctionalTests.bat", - $"A {FolderToDelete}/RunUnitTests.bat", $"A {FolderToDelete}/", }; From 36fef4d4335fead8127d68cb2f48c4ecc2ebd9c0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:20:18 -0600 Subject: [PATCH 3/7] Use better names in the modified paths test --- .../Common/ModifiedPathsDatabaseTests.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 59b52e921..bc3106da5 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -134,16 +134,16 @@ public void EntryNotAddedIfParentDirectoryExists() [TestCase] public void CompressEntries() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); - mpd.Compress(new MockTracer()); - mpd.Count.ShouldEqual(4); - mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); - } + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); + modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } - private static void TestAddingPath(string path, bool isFolder = false) + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } From 2bbca795df85e63392955a754c9dbe4dd9ea40bb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:24:23 -0600 Subject: [PATCH 4/7] Move code to remove modified path entries to the TryStart method --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 44bce1687..6ff4d40f3 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,9 +89,6 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } - this.modifiedPaths.Compress(this.context.Tracer); - this.modifiedPaths.WriteAllEntriesAndFlush(); - this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); @@ -174,6 +171,9 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + if (!this.fileSystemVirtualizer.TryStart(this, out error)) { return false; From f0bcb3425f4928848d7f4c4918c84d7ac1a76208 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 16:09:52 -0600 Subject: [PATCH 5/7] Rename method and use Split method for building parent paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 26 ++++++++++++------- .../Common/ModifiedPathsDatabaseTests.cs | 2 +- .../FileSystemCallbacks.cs | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 9fef5318b..6f40923b3 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -52,24 +54,30 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } - public void Compress(ITracer tracer) + /// + /// This method will examine the modified paths to check if there is already a parent folder entry in + /// the modified paths. If there is a parent folder the entry does not need to be in the modified paths + /// and will be removed because the parent folder is recursive and covers any children. + /// + public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) { - foreach (var item in this.modifiedPaths) + StringBuilder parentFolder = new StringBuilder(); + foreach (string modifiedPath in this.modifiedPaths) { - int pathSeparatorIndex = item.IndexOf('/'); - while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + parentFolder.Clear(); + foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) { - string folder = item.Substring(0, pathSeparatorIndex + 1); - if (this.modifiedPaths.Contains(folder)) + parentFolder.Append(pathPart + "/"); + if (this.modifiedPaths.Contains(parentFolder.ToString())) { - this.modifiedPaths.TryRemove(item); + this.modifiedPaths.TryRemove(modifiedPath); break; } - - pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); } } diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index bc3106da5..ced7ba585 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -135,7 +135,7 @@ public void EntryNotAddedIfParentDirectoryExists() public void CompressEntries() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); modifiedPathsDatabase.Count.ShouldEqual(4); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6ff4d40f3..627c895a5 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -171,7 +171,7 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { - this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.RemoveEntriesWithParentFolderEntry(this.context.Tracer); this.modifiedPaths.WriteAllEntriesAndFlush(); if (!this.fileSystemVirtualizer.TryStart(this, out error)) From 02442473d9c1fba9d8361f7a4305e4dee06f59b0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 27 Sep 2018 12:28:43 -0600 Subject: [PATCH 6/7] Add more tests for modified paths --- .../EnlistmentPerTestCase/ModifiedPathsTests.cs | 13 ++++++++++--- .../Common/ModifiedPathsDatabaseTests.cs | 11 ++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f2005745e..f2f255180 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -133,9 +133,16 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) folderToCreateOutsideRepoTargetPath.ShouldBeADirectory(fileSystem); folderToCreateOutsideRepoPath.ShouldNotExistOnDisk(fileSystem); - string folderToDelete = this.Enlistment.GetVirtualPathTo(FolderToDelete); - fileSystem.DeleteDirectory(folderToDelete); - folderToDelete.ShouldNotExistOnDisk(fileSystem); + string folderToDeleteFullPath = this.Enlistment.GetVirtualPathTo(FolderToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "NewFile.txt"), "Contents for new file"); + string newFileToDelete = Path.Combine(folderToDeleteFullPath, "NewFileToDelete.txt"); + fileSystem.WriteAllText(newFileToDelete, "Contents for new file"); + fileSystem.DeleteFile(newFileToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "CreateCommonVersionHeader.bat"), "Changing the file contents"); + fileSystem.DeleteFile(Path.Combine(folderToDeleteFullPath, "RunUnitTests.bat")); + + fileSystem.DeleteDirectory(folderToDeleteFullPath); + folderToDeleteFullPath.ShouldNotExistOnDisk(fileSystem); // Remount this.Enlistment.UnmountGVFS(); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index ced7ba585..40bacf08f 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -20,15 +20,19 @@ A dir/file2.txt A dir1/dir2/file3.txt "; private const string EntriesToCompress = @"A file.txt +D deleted.txt A dir/file2.txt A dir/dir3/dir4/ A dir1/dir2/file3.txt A dir/ +D deleted/ A dir1/dir2/ A dir1/file.txt A dir1/dir2/dir3/dir4/dir5/ A dir/dir2/file3.txt A dir/dir4/dir5/ +D dir/dir2/deleted.txt +A dir1/dir2 "; [TestCase] @@ -132,13 +136,14 @@ public void EntryNotAddedIfParentDirectoryExists() } [TestCase] - public void CompressEntries() + public void RemoveEntriesWithParentFolderEntry() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); - modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.RemoveEntriesWithParentFolderEntry(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(5); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); } From c3f5aaec9726d1a17c5595e33e741749ee57d426 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 23:07:21 -0600 Subject: [PATCH 7/7] Change to use ContainsParentDirectory for removing modified paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 6f40923b3..95fb18e73 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -62,22 +60,13 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; - using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + using (ITracer activity = tracer.StartActivity(nameof(this.RemoveEntriesWithParentFolderEntry), EventLevel.Informational)) { - StringBuilder parentFolder = new StringBuilder(); foreach (string modifiedPath in this.modifiedPaths) { - string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - - parentFolder.Clear(); - foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + if (this.ContainsParentDirectory(modifiedPath)) { - parentFolder.Append(pathPart + "/"); - if (this.modifiedPaths.Contains(parentFolder.ToString())) - { - this.modifiedPaths.TryRemove(modifiedPath); - break; - } + this.modifiedPaths.TryRemove(modifiedPath); } }