Merge pull request #7228 from ubsan/parallel-file-ops

Parallel file operations

toolsrc/CMakeLists.txt

@@ -35,7 +35,7 @@ if(GCC OR CLANG)
 endif()
 
 file(GLOB_RECURSE VCPKGLIB_SOURCES src/vcpkg/*.cpp)
-file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-tests/*.cpp)
+file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-test/*.cpp)
 
 if (DEFINE_DISABLE_METRICS)
     set(DISABLE_METRICS_VALUE "1")
@@ -52,7 +52,11 @@ add_executable(vcpkg-test EXCLUDE_FROM_ALL ${VCPKGTEST_SOURCES} ${VCPKGLIB_SOURC
 target_compile_definitions(vcpkg-test PRIVATE -DDISABLE_METRICS=${DISABLE_METRICS_VALUE})
 target_include_directories(vcpkg-test PRIVATE include)
 
-foreach(TEST_NAME arguments chrono dependencies paragraph plan specifier supports)
+foreach(TEST_NAME
+    arguments chrono dependencies files
+    paragraph plan specifier statusparagraphs
+    strings supports update
+)
     add_test(${TEST_NAME} vcpkg-test [${TEST_NAME}])
 endforeach()
 
@@ -91,3 +95,4 @@ endif()
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 target_link_libraries(vcpkg PRIVATE Threads::Threads)
+target_link_libraries(vcpkg-test PRIVATE Threads::Threads)

toolsrc/include/vcpkg-test/util.h

@@ -1,3 +1,4 @@
+#include <vcpkg/base/files.h>
 #include <vcpkg/statusparagraph.h>
 
 #include <memory>
@@ -30,4 +31,12 @@ T&& unwrap(vcpkg::Optional<T>&& opt)
     return std::move(*opt.get());
 }
 
+extern const bool SYMLINKS_ALLOWED;
+
+extern const fs::path TEMPORARY_DIRECTORY;
+
+void create_symlink(const fs::path& file, const fs::path& target, std::error_code& ec);
+
+void create_directory_symlink(const fs::path& file, const fs::path& target, std::error_code& ec);
+
 }

toolsrc/include/vcpkg/base/files.h

@@ -13,13 +13,60 @@ namespace fs
     using stdfs::file_status;
     using stdfs::file_type;
     using stdfs::path;
+    using stdfs::perms;
     using stdfs::u8path;
 
-    inline bool is_regular_file(file_status s) { return stdfs::is_regular_file(s); }
+    /*
-    inline bool is_directory(file_status s) { return stdfs::is_directory(s); }
+        std::experimental::filesystem's file_status and file_type are broken in
-    inline bool is_symlink(file_status s) { return stdfs::is_symlink(s); }
+        the presence of symlinks -- a symlink is treated as the object it points
+        to for `symlink_status` and `symlink_type`
+    */
+
+    using stdfs::status;
+
+    // we want to poison ADL with these niebloids
+
+    namespace detail
+    {
+        struct symlink_status_t
+        {
+            file_status operator()(const path& p, std::error_code& ec) const noexcept;
+            file_status operator()(const path& p, vcpkg::LineInfo li) const noexcept;
+        };
+        struct is_symlink_t
+        {
+            inline bool operator()(file_status s) const { return stdfs::is_symlink(s); }
+        };
+        struct is_regular_file_t
+        {
+            inline bool operator()(file_status s) const { return stdfs::is_regular_file(s); }
+        };
+        struct is_directory_t
+        {
+            inline bool operator()(file_status s) const { return stdfs::is_directory(s); }
+        };
+    }
+
+    constexpr detail::symlink_status_t symlink_status{};
+    constexpr detail::is_symlink_t is_symlink{};
+    constexpr detail::is_regular_file_t is_regular_file{};
+    constexpr detail::is_directory_t is_directory{};
 }
 
+/*
+    if someone attempts to use unqualified `symlink_status` or `is_symlink`,
+    they might get the ADL version, which is broken.
+    Therefore, put `symlink_status` in the global namespace, so that they get
+    our symlink_status.
+
+    We also want to poison the ADL on is_regular_file and is_directory, because
+    we don't want people calling these functions on paths
+*/
+using fs::is_directory;
+using fs::is_regular_file;
+using fs::is_symlink;
+using fs::symlink_status;
+
 namespace vcpkg::Files
 {
     struct Filesystem
@@ -44,7 +91,9 @@ namespace vcpkg::Files
                                     std::error_code& ec) = 0;
         bool remove(const fs::path& path, LineInfo linfo);
         virtual bool remove(const fs::path& path, std::error_code& ec) = 0;
-        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) = 0;
+
+        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) = 0;
+        std::uintmax_t remove_all(const fs::path& path, LineInfo li);
         virtual bool exists(const fs::path& path) const = 0;
         virtual bool is_directory(const fs::path& path) const = 0;
         virtual bool is_regular_file(const fs::path& path) const = 0;

toolsrc/include/vcpkg/base/strings.h

@@ -184,4 +184,9 @@ namespace vcpkg::Strings
     const char* search(StringView haystack, StringView needle);
 
     bool contains(StringView haystack, StringView needle);
+
+    // base 32 encoding, since base64 encoding requires lowercase letters,
+    // which are not distinct from uppercase letters on macOS or Windows filesystems.
+    // follows RFC 4648
+    std::string b32_encode(std::uint64_t x) noexcept;
 }

toolsrc/include/vcpkg/base/work_queue.h

@@ -0,0 +1,230 @@
+#pragma once
+
+#include <condition_variable>
+#include <memory>
+#include <queue>
+
+namespace vcpkg
+{
+    template<class Action, class ThreadLocalData>
+    struct WorkQueue;
+
+    namespace detail
+    {
+        // for SFINAE purposes, keep out of the class
+        template<class Action, class ThreadLocalData>
+        auto call_moved_action(Action& action,
+                               const WorkQueue<Action, ThreadLocalData>& work_queue,
+                               ThreadLocalData& tld) -> decltype(static_cast<void>(std::move(action)(tld, work_queue)))
+        {
+            std::move(action)(tld, work_queue);
+        }
+
+        template<class Action, class ThreadLocalData>
+        auto call_moved_action(Action& action, const WorkQueue<Action, ThreadLocalData>&, ThreadLocalData& tld)
+            -> decltype(static_cast<void>(std::move(action)(tld)))
+        {
+            std::move(action)(tld);
+        }
+    }
+
+    template<class Action, class ThreadLocalData>
+    struct WorkQueue
+    {
+        template<class F>
+        WorkQueue(std::uint16_t num_threads, LineInfo li, const F& tld_init) noexcept
+        {
+            m_line_info = li;
+
+            set_unjoined_workers(num_threads);
+            m_threads.reserve(num_threads);
+            for (std::size_t i = 0; i < num_threads; ++i)
+            {
+                m_threads.push_back(std::thread(Worker{this, tld_init()}));
+            }
+        }
+
+        WorkQueue(WorkQueue const&) = delete;
+        WorkQueue(WorkQueue&&) = delete;
+
+        ~WorkQueue()
+        {
+            auto lck = std::unique_lock<std::mutex>(m_mutex);
+            if (!is_joined(m_state))
+            {
+                Checks::exit_with_message(m_line_info, "Failed to call join() on a WorkQueue that was destroyed");
+            }
+        }
+
+        // should only be called once; anything else is an error
+        void run(LineInfo li)
+        {
+            // this should _not_ be locked before `run()` is called; however, we
+            // want to terminate if someone screws up, rather than cause UB
+            auto lck = std::unique_lock<std::mutex>(m_mutex);
+
+            if (m_state != State::BeforeRun)
+            {
+                Checks::exit_with_message(li, "Attempted to run() twice");
+            }
+
+            m_state = State::Running;
+        }
+
+        // runs all remaining tasks, and blocks on their finishing
+        // if this is called in an existing task, _will block forever_
+        // DO NOT DO THAT
+        // thread-unsafe
+        void join(LineInfo li)
+        {
+            {
+                auto lck = std::unique_lock<std::mutex>(m_mutex);
+                if (is_joined(m_state))
+                {
+                    Checks::exit_with_message(li, "Attempted to call join() more than once");
+                }
+                else if (m_state == State::Terminated)
+                {
+                    m_state = State::TerminatedJoined;
+                }
+                else
+                {
+                    m_state = State::Joined;
+                }
+            }
+
+            while (unjoined_workers())
+            {
+                if (!running_workers())
+                {
+                    m_cv.notify_one();
+                }
+            }
+
+            // wait for all threads to join
+            for (auto& thrd : m_threads)
+            {
+                thrd.join();
+            }
+        }
+
+        // useful in the case of errors
+        // doesn't stop any existing running tasks
+        // returns immediately, so that one can call this in a task
+        void terminate() const
+        {
+            {
+                auto lck = std::unique_lock<std::mutex>(m_mutex);
+                if (is_joined(m_state))
+                {
+                    m_state = State::TerminatedJoined;
+                }
+                else
+                {
+                    m_state = State::Terminated;
+                }
+            }
+            m_cv.notify_all();
+        }
+
+        void enqueue_action(Action a) const
+        {
+            {
+                auto lck = std::unique_lock<std::mutex>(m_mutex);
+                m_actions.push_back(std::move(a));
+
+                if (m_state == State::BeforeRun) return;
+            }
+            m_cv.notify_one();
+        }
+
+    private:
+        struct Worker
+        {
+            const WorkQueue* work_queue;
+            ThreadLocalData tld;
+
+            void operator()()
+            {
+                // unlocked when waiting, or when in the action
+                // locked otherwise
+                auto lck = std::unique_lock<std::mutex>(work_queue->m_mutex);
+
+                work_queue->m_cv.wait(lck, [&] { return work_queue->m_state != State::BeforeRun; });
+
+                work_queue->increment_running_workers();
+                for (;;)
+                {
+                    const auto state = work_queue->m_state;
+
+                    if (is_terminated(state))
+                    {
+                        break;
+                    }
+
+                    if (work_queue->m_actions.empty())
+                    {
+                        if (state == State::Running || work_queue->running_workers() > 1)
+                        {
+                            work_queue->decrement_running_workers();
+                            work_queue->m_cv.wait(lck);
+                            work_queue->increment_running_workers();
+                            continue;
+                        }
+
+                        // the queue is joining, and we are the only worker running
+                        // no more work!
+                        break;
+                    }
+
+                    Action action = std::move(work_queue->m_actions.back());
+                    work_queue->m_actions.pop_back();
+
+                    lck.unlock();
+                    work_queue->m_cv.notify_one();
+                    detail::call_moved_action(action, *work_queue, tld);
+                    lck.lock();
+                }
+
+                work_queue->decrement_running_workers();
+                work_queue->decrement_unjoined_workers();
+            }
+        };
+
+        enum class State : std::int16_t
+        {
+            // can only exist upon construction
+            BeforeRun = -1,
+
+            Running,
+            Joined,
+            Terminated,
+            TerminatedJoined,
+        };
+
+        static bool is_terminated(State st) { return st == State::Terminated || st == State::TerminatedJoined; }
+
+        static bool is_joined(State st) { return st == State::Joined || st == State::TerminatedJoined; }
+
+        mutable std::mutex m_mutex{};
+        // these are all under m_mutex
+        mutable State m_state = State::BeforeRun;
+        mutable std::vector<Action> m_actions{};
+        mutable std::condition_variable m_cv{};
+
+        mutable std::atomic<std::uint32_t> m_workers;
+        // = unjoined_workers << 16 | running_workers
+
+        void set_unjoined_workers(std::uint16_t threads) { m_workers = std::uint32_t(threads) << 16; }
+        void decrement_unjoined_workers() const { m_workers -= 1 << 16; }
+
+        std::uint16_t unjoined_workers() const { return std::uint16_t(m_workers >> 16); }
+
+        void increment_running_workers() const { ++m_workers; }
+        void decrement_running_workers() const { --m_workers; }
+        std::uint16_t running_workers() const { return std::uint16_t(m_workers); }
+
+        std::vector<std::thread> m_threads{};
+        LineInfo m_line_info;
+    };
+}

toolsrc/src/vcpkg-test/arguments.cpp

@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/vcpkgcmdarguments.h>
 

toolsrc/src/vcpkg-test/catch.cpp

@@ -1,5 +1,5 @@
 #define CATCH_CONFIG_RUNNER
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/system.debug.h>
 

toolsrc/src/vcpkg-test/chrono.cpp

@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/chrono.h>
 

toolsrc/src/vcpkg-test/dependencies.cpp

@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/sourceparagraph.h>
 

toolsrc/src/vcpkg-test/files.cpp

@@ -0,0 +1,121 @@
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/strings.h>
+
+#include <iostream>
+#include <random>
+
+#include <vector>
+
+using vcpkg::Test::SYMLINKS_ALLOWED;
+using vcpkg::Test::TEMPORARY_DIRECTORY;
+
+namespace
+{
+    using uid = std::uniform_int_distribution<std::uint64_t>;
+
+    std::mt19937_64 get_urbg(std::uint64_t index)
+    {
+        // smallest prime > 2**63 - 1
+        return std::mt19937_64{index + 9223372036854775837ULL};
+    }
+
+    std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b32_encode(uid{}(urbg)); }
+
+    void create_directory_tree(std::mt19937_64& urbg,
+                               vcpkg::Files::Filesystem& fs,
+                               std::uint64_t depth,
+                               const fs::path& base)
+    {
+        std::random_device rd;
+        constexpr std::uint64_t max_depth = 5;
+        constexpr std::uint64_t width = 5;
+
+        // we want ~70% of our "files" to be directories, and then a third
+        // each of the remaining ~30% to be regular files, directory symlinks,
+        // and regular symlinks
+        constexpr std::uint64_t directory_min_tag = 0;
+        constexpr std::uint64_t directory_max_tag = 6;
+        constexpr std::uint64_t regular_file_tag = 7;
+        constexpr std::uint64_t regular_symlink_tag = 8;
+        constexpr std::uint64_t directory_symlink_tag = 9;
+
+        // if we're at the max depth, we only want to build non-directories
+        std::uint64_t file_type;
+        if (depth < max_depth)
+        {
+            file_type = uid{directory_min_tag, regular_symlink_tag}(urbg);
+        }
+        else
+        {
+            file_type = uid{regular_file_tag, regular_symlink_tag}(urbg);
+        }
+
+        if (!SYMLINKS_ALLOWED && file_type > regular_file_tag)
+        {
+            file_type = regular_file_tag;
+        }
+
+        std::error_code ec;
+        if (file_type <= directory_max_tag)
+        {
+            fs.create_directory(base, ec);
+            if (ec) {
+                INFO("File that failed: " << base);
+                REQUIRE_FALSE(ec);
+            }
+
+            for (int i = 0; i < width; ++i)
+            {
+                create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg));
+            }
+        }
+        else if (file_type == regular_file_tag)
+        {
+            // regular file
+            fs.write_contents(base, "", ec);
+        }
+        else if (file_type == regular_symlink_tag)
+        {
+            // regular symlink
+            fs.write_contents(base, "", ec);
+            REQUIRE_FALSE(ec);
+            auto base_link = base;
+            base_link.replace_filename(base.filename().u8string() + "-link");
+            vcpkg::Test::create_symlink(base, base_link, ec);
+        }
+        else // type == directory_symlink_tag
+        {
+            // directory symlink
+            vcpkg::Test::create_directory_symlink(base / "..", base, ec);
+        }
+
+        REQUIRE_FALSE(ec);
+    }
+}
+
+TEST_CASE ("remove all", "[files]")
+{
+    auto urbg = get_urbg(0);
+
+    fs::path temp_dir = TEMPORARY_DIRECTORY / get_random_filename(urbg);
+
+    auto& fs = vcpkg::Files::get_real_filesystem();
+
+    std::error_code ec;
+    fs.create_directory(TEMPORARY_DIRECTORY, ec);
+
+    REQUIRE_FALSE(ec);
+
+    INFO("temp dir is: " << temp_dir);
+
+    create_directory_tree(urbg, fs, 0, temp_dir);
+
+    fs::path fp;
+    fs.remove_all(temp_dir, ec, fp);
+    REQUIRE_FALSE(ec);
+
+    REQUIRE_FALSE(fs.exists(temp_dir));
+}

toolsrc/src/vcpkg-test/paragraph.cpp

@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/strings.h>
 

toolsrc/src/vcpkg-test/plan.cpp

@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/dependencies.h>
 #include <vcpkg/sourceparagraph.h>

toolsrc/src/vcpkg-test/specifier.cpp

@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/util.h>
 #include <vcpkg/packagespec.h>

toolsrc/src/vcpkg-test/statusparagraphs.cpp

@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/util.h>
 #include <vcpkg/paragraphs.h>

toolsrc/src/vcpkg-test/strings.cpp

@@ -0,0 +1,33 @@
+#include <vcpkg-test/catch.h>
+
+#include <vcpkg/base/strings.h>
+
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+TEST_CASE ("b32 encoding", "[strings]")
+{
+    using u64 = std::uint64_t;
+
+    std::vector<std::pair<std::uint64_t, std::string>> map;
+
+    map.emplace_back(0, "AAAAAAAAAAAAA");
+    map.emplace_back(1, "BAAAAAAAAAAAA");
+
+    map.emplace_back(u64(1) << 32, "AAAAAAEAAAAAA");
+    map.emplace_back((u64(1) << 32) + 1, "BAAAAAEAAAAAA");
+
+    map.emplace_back(0xE4D0'1065'D11E'0229, "JRA4RIXMQAUJO");
+    map.emplace_back(0xA626'FE45'B135'07FF, "77BKTYWI6XJMK");
+    map.emplace_back(0xEE36'D228'0C31'D405, "FAVDDGAFSWN4O");
+    map.emplace_back(0x1405'64E7'FE7E'A88C, "MEK5H774ELBIB");
+    map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "777777777777P");
+
+    std::string result;
+    for (const auto& pr : map)
+    {
+        result = vcpkg::Strings::b32_encode(pr.first);
+        REQUIRE(vcpkg::Strings::b32_encode(pr.first) == pr.second);
+    }
+}

toolsrc/src/vcpkg-test/supports.cpp

@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/sourceparagraph.h>
 

toolsrc/src/vcpkg-test/update.cpp

@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/sortedvector.h>
 

toolsrc/src/vcpkg-test/util.cpp

@@ -0,0 +1,177 @@
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
+
+#include <vcpkg/base/checks.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/statusparagraph.h>
+
+// used to get the implementation specific compiler flags (i.e., __cpp_lib_filesystem)
+#include <ciso646>
+
+#include <iostream>
+#include <memory>
+
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
+#define FILESYSTEM_SYMLINK_STD 0
+#define FILESYSTEM_SYMLINK_UNIX 1
+#define FILESYSTEM_SYMLINK_NONE 2
+
+#if defined(__cpp_lib_filesystem)
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_STD
+#include <filesystem> // required for filesystem::create_{directory_}symlink
+
+#elif !defined(_MSC_VER)
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_UNIX
+#include <unistd.h>
+
+#else
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_NONE
+
+#endif
+
+namespace vcpkg::Test
+{
+    std::unique_ptr<vcpkg::StatusParagraph> make_status_pgh(const char* name,
+                                                            const char* depends,
+                                                            const char* default_features,
+                                                            const char* triplet)
+    {
+        using Pgh = std::unordered_map<std::string, std::string>;
+        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
+                                                     {"Version", "1"},
+                                                     {"Architecture", triplet},
+                                                     {"Multi-Arch", "same"},
+                                                     {"Depends", depends},
+                                                     {"Default-Features", default_features},
+                                                     {"Status", "install ok installed"}});
+    }
+
+    std::unique_ptr<StatusParagraph> make_status_feature_pgh(const char* name,
+                                                             const char* feature,
+                                                             const char* depends,
+                                                             const char* triplet)
+    {
+        using Pgh = std::unordered_map<std::string, std::string>;
+        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
+                                                     {"Version", "1"},
+                                                     {"Feature", feature},
+                                                     {"Architecture", triplet},
+                                                     {"Multi-Arch", "same"},
+                                                     {"Depends", depends},
+                                                     {"Status", "install ok installed"}});
+    }
+
+    PackageSpec unsafe_pspec(std::string name, Triplet t)
+    {
+        auto m_ret = PackageSpec::from_name_and_triplet(name, t);
+        REQUIRE(m_ret.has_value());
+        return m_ret.value_or_exit(VCPKG_LINE_INFO);
+    }
+
+    static bool system_allows_symlinks()
+    {
+#if defined(_WIN32)
+        if (!__cpp_lib_filesystem)
+        {
+            return false;
+        }
+
+        HKEY key;
+        bool allow_symlinks = true;
+
+        const auto status = RegOpenKeyExW(
+            HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, 0, &key);
+
+        if (status == ERROR_FILE_NOT_FOUND)
+        {
+            allow_symlinks = false;
+            std::clog << "Symlinks are not allowed on this system\n";
+        }
+
+        if (status == ERROR_SUCCESS) RegCloseKey(key);
+
+        return allow_symlinks;
+#else
+        return true;
+#endif
+    }
+
+    static fs::path internal_temporary_directory()
+    {
+#if defined(_WIN32)
+        wchar_t* tmp = static_cast<wchar_t*>(std::calloc(32'767, 2));
+
+        if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767))
+        {
+            std::cerr << "No temporary directory found.\n";
+            std::abort();
+        }
+
+        fs::path result = tmp;
+        std::free(tmp);
+
+        return result / L"vcpkg-test";
+#else
+        return "/tmp/vcpkg-test";
+#endif
+    }
+
+    const bool SYMLINKS_ALLOWED = system_allows_symlinks();
+    const fs::path TEMPORARY_DIRECTORY = internal_temporary_directory();
+
+#if FILESYSTEM_SYMLINK == FILSYSTEM_SYMLINK_NONE
+    constexpr inline char no_filesystem_message[] =
+        "<filesystem> doesn't exist; on windows, we don't attempt to use the win32 calls to create symlinks";
+#endif
+
+    void create_symlink(const fs::path& target, const fs::path& file, std::error_code& ec)
+    {
+#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD
+        if (SYMLINKS_ALLOWED)
+        {
+            std::filesystem::path targetp = target.native();
+            std::filesystem::path filep = file.native();
+
+            std::filesystem::create_symlink(targetp, filep);
+        }
+        else
+        {
+            vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system");
+        }
+#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX
+        if (symlink(target.c_str(), file.c_str()) != 0)
+        {
+            ec.assign(errno, std::system_category());
+        }
+#else
+        vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message);
+#endif
+    }
+
+    void create_directory_symlink(const fs::path& target, const fs::path& file, std::error_code& ec)
+    {
+#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD
+        if (SYMLINKS_ALLOWED)
+        {
+            std::filesystem::path targetp = target.native();
+            std::filesystem::path filep = file.native();
+
+            std::filesystem::create_symlink(targetp, filep);
+        }
+        else
+        {
+            vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system");
+        }
+#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX
+        ::vcpkg::Test::create_symlink(target, file, ec);
+#else
+        vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message);
+#endif
+    }
+}

toolsrc/src/vcpkg-tests/util.cpp

@@ -1,47 +0,0 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
-
-#include <vcpkg/statusparagraph.h>
-
-#include <memory>
-
-namespace vcpkg::Test
-{
-    std::unique_ptr<vcpkg::StatusParagraph> make_status_pgh(const char* name,
-                                                            const char* depends,
-                                                            const char* default_features,
-                                                            const char* triplet)
-    {
-        using Pgh = std::unordered_map<std::string, std::string>;
-        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
-                                                     {"Version", "1"},
-                                                     {"Architecture", triplet},
-                                                     {"Multi-Arch", "same"},
-                                                     {"Depends", depends},
-                                                     {"Default-Features", default_features},
-                                                     {"Status", "install ok installed"}});
-    }
-
-    std::unique_ptr<StatusParagraph> make_status_feature_pgh(const char* name,
-                                                             const char* feature,
-                                                             const char* depends,
-                                                             const char* triplet)
-    {
-        using Pgh = std::unordered_map<std::string, std::string>;
-        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
-                                                     {"Version", "1"},
-                                                     {"Feature", feature},
-                                                     {"Architecture", triplet},
-                                                     {"Multi-Arch", "same"},
-                                                     {"Depends", depends},
-                                                     {"Status", "install ok installed"}});
-    }
-
-    PackageSpec unsafe_pspec(std::string name, Triplet t)
-    {
-        auto m_ret = PackageSpec::from_name_and_triplet(name, t);
-        REQUIRE(m_ret.has_value());
-        return m_ret.value_or_exit(VCPKG_LINE_INFO);
-    }
-
-}

toolsrc/src/vcpkg/archives.cpp

@@ -15,9 +15,10 @@ namespace vcpkg::Archives
 #endif
             ;
 
+        fs.remove_all(to_path, VCPKG_LINE_INFO);
+        fs.remove_all(to_path_partial, VCPKG_LINE_INFO);
+        // TODO: check this error code
         std::error_code ec;
-        fs.remove_all(to_path, ec);
-        fs.remove_all(to_path_partial, ec);
         fs.create_directories(to_path_partial, ec);
         const auto ext = archive.extension();
 #if defined(_WIN32)

toolsrc/src/vcpkg/base/files.cpp

@@ -6,6 +6,7 @@
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/base/work_queue.h>
 
 #if defined(__linux__) || defined(__APPLE__)
 #include <fcntl.h>
@@ -20,6 +21,59 @@
 #include <copyfile.h>
 #endif
 
+namespace fs::detail
+{
+    file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept
+    {
+#if defined(_WIN32)
+        static_cast<void>(ec);
+
+        /*
+            do not find the permissions of the file -- it's unnecessary for the
+            things that vcpkg does.
+            if one were to add support for this in the future, one should look
+            into GetFileSecurityW
+        */
+        perms permissions = perms::unknown;
+
+        WIN32_FILE_ATTRIBUTE_DATA file_attributes;
+        file_type ft = file_type::unknown;
+        if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes))
+        {
+            ft = file_type::not_found;
+        }
+        else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+        {
+            // check for reparse point -- if yes, then symlink
+            ft = file_type::symlink;
+        }
+        else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+        {
+            ft = file_type::directory;
+        }
+        else
+        {
+            // otherwise, the file is a regular file
+            ft = file_type::regular;
+        }
+
+        return file_status(ft, permissions);
+
+#else
+        return stdfs::symlink_status(p, ec);
+#endif
+    }
+
+    file_status symlink_status_t::operator()(const path& p, vcpkg::LineInfo li) const noexcept
+    {
+        std::error_code ec;
+        auto result = symlink_status(p, ec);
+        if (ec) vcpkg::Checks::exit_with_message(li, "error getting status of path %s: %s", p.string(), ec.message());
+
+        return result;
+    }
+}
+
 namespace vcpkg::Files
 {
     static const std::regex FILESYSTEM_INVALID_CHARACTERS_REGEX = std::regex(R"([\/:*?"<>|])");
@@ -63,6 +117,25 @@ namespace vcpkg::Files
         if (ec) Checks::exit_with_message(linfo, "error writing lines: %s: %s", path.u8string(), ec.message());
     }
 
+    std::uintmax_t Filesystem::remove_all(const fs::path& path, LineInfo li)
+    {
+        std::error_code ec;
+        fs::path failure_point;
+
+        const auto result = this->remove_all(path, ec, failure_point);
+
+        if (ec)
+        {
+            Checks::exit_with_message(li,
+                                      "Failure to remove_all(%s) due to file %s: %s",
+                                      path.string(),
+                                      failure_point.string(),
+                                      ec.message());
+        }
+
+        return result;
+    }
+
     struct RealFilesystem final : Filesystem
     {
         virtual Expected<std::string> read_contents(const fs::path& file_path) const override
@@ -87,7 +160,7 @@ namespace vcpkg::Files
             file_stream.read(&output[0], length);
             file_stream.close();
 
-            return std::move(output);
+            return output;
         }
         virtual Expected<std::vector<std::string>> read_lines(const fs::path& file_path) const override
         {
@@ -105,7 +178,7 @@ namespace vcpkg::Files
             }
             file_stream.close();
 
-            return std::move(output);
+            return output;
         }
         virtual fs::path find_file_recursively_up(const fs::path& starting_dir,
                                                   const std::string& filename) const override
@@ -254,28 +327,163 @@ namespace vcpkg::Files
 #endif
         }
         virtual bool remove(const fs::path& path, std::error_code& ec) override { return fs::stdfs::remove(path, ec); }
-        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) override
+        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) override
         {
-            // Working around the currently buggy remove_all()
+            /*
-            std::uintmax_t out = fs::stdfs::remove_all(path, ec);
+                does not use the std::filesystem call since it is buggy, and can
+                have spurious errors before VS 2017 update 6, and on later versions
+                (as well as on macOS and Linux), this is just as fast and will have
+                fewer spurious errors due to locks.
+            */
 
-            for (int i = 0; i < 5 && this->exists(path); i++)
+            /*
+                `remove` doesn't actually remove anything -- it simply moves the
+                files into a parent directory (which ends up being at `path`),
+                and then inserts `actually_remove{current_path}` into the work
+                queue.
+            */
+            struct remove
             {
-                using namespace std::chrono_literals;
+                struct tld
-                std::this_thread::sleep_for(i * 100ms);
+                {
-                out += fs::stdfs::remove_all(path, ec);
+                    const fs::path& tmp_directory;
+                    std::uint64_t index;
+
+                    std::atomic<std::uintmax_t>& files_deleted;
+
+                    std::mutex& ec_mutex;
+                    std::error_code& ec;
+                    fs::path& failure_point;
+                };
+
+                struct actually_remove;
+                using queue = WorkQueue<actually_remove, tld>;
+
+                /*
+                    if `current_path` is a directory, first `remove`s all
+                    elements of the directory, then calls remove.
+
+                    else, just calls remove.
+                */
+                struct actually_remove
+                {
+                    fs::path current_path;
+
+                    void operator()(tld& info, const queue& queue) const
+                    {
+                        std::error_code ec;
+                        const auto path_type = fs::symlink_status(current_path, ec).type();
+
+                        if (check_ec(ec, info, queue, current_path)) return;
+
+                        if (path_type == fs::file_type::directory)
+                        {
+                            for (const auto& entry : fs::stdfs::directory_iterator(current_path))
+                            {
+                                remove{}(entry, info, queue);
+                            }
+                        }
+
+                        if (fs::stdfs::remove(current_path, ec))
+                        {
+                            info.files_deleted.fetch_add(1, std::memory_order_relaxed);
+                        }
+                        else
+                        {
+                            check_ec(ec, info, queue, current_path);
+                        }
+                    }
+                };
+
+                static bool check_ec(const std::error_code& ec,
+                                     tld& info,
+                                     const queue& queue,
+                                     const fs::path& failure_point)
+                {
+                    if (ec)
+                    {
+                        queue.terminate();
+
+                        auto lck = std::unique_lock<std::mutex>(info.ec_mutex);
+                        if (!info.ec)
+                        {
+                            info.ec = ec;
+                            info.failure_point = failure_point;
+                        }
+
+                        return true;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+
+                void operator()(const fs::path& current_path, tld& info, const queue& queue) const
+                {
+                    std::error_code ec;
+
+                    const auto tmp_name = Strings::b32_encode(info.index++);
+                    const auto tmp_path = info.tmp_directory / tmp_name;
+
+                    fs::stdfs::rename(current_path, tmp_path, ec);
+                    if (check_ec(ec, info, queue, current_path)) return;
+
+                    queue.enqueue_action(actually_remove{std::move(tmp_path)});
+                }
+            };
+
+            const auto path_type = fs::symlink_status(path, ec).type();
+
+            std::atomic<std::uintmax_t> files_deleted{0};
+
+            if (path_type == fs::file_type::directory)
+            {
+                std::uint64_t index = 0;
+                std::mutex ec_mutex;
+
+                auto const tld_gen = [&] {
+                    index += static_cast<std::uint64_t>(1) << 32;
+                    return remove::tld{path, index, files_deleted, ec_mutex, ec, failure_point};
+                };
+
+                remove::queue queue{4, VCPKG_LINE_INFO, tld_gen};
+
+                // note: we don't actually start the queue running until the
+                // `join()`. This allows us to rename all the top-level files in
+                // peace, so that we don't get collisions.
+                auto main_tld = tld_gen();
+                for (const auto& entry : fs::stdfs::directory_iterator(path))
+                {
+                    remove{}(entry, main_tld, queue);
+                }
+
+                queue.join(VCPKG_LINE_INFO);
             }
 
-            if (this->exists(path))
+            /*
+                we need to do backoff on the removal of the top level directory,
+                since we need to place all moved files into that top level
+                directory, and so we can only delete the directory after all the
+                lower levels have been deleted.
+            */
+            for (int backoff = 0; backoff < 5; ++backoff)
             {
-                System::print2(
+                if (backoff)
-                    System::Color::warning,
+                {
-                    "Some files in ",
+                    using namespace std::chrono_literals;
-                    path.u8string(),
+                    auto backoff_time = 100ms * backoff;
-                    " were unable to be removed. Close any editors operating in this directory and retry.\n");
+                    std::this_thread::sleep_for(backoff_time);
+                }
+
+                if (fs::stdfs::remove(path, ec))
+                {
+                    files_deleted.fetch_add(1, std::memory_order_relaxed);
+                    break;
+                }
             }
 
-            return out;
+            return files_deleted;
         }
         virtual bool exists(const fs::path& path) const override { return fs::stdfs::exists(path); }
         virtual bool is_directory(const fs::path& path) const override { return fs::stdfs::is_directory(path); }
@@ -307,11 +515,11 @@ namespace vcpkg::Files
 
         virtual fs::file_status status(const fs::path& path, std::error_code& ec) const override
         {
-            return fs::stdfs::status(path, ec);
+            return fs::status(path, ec);
         }
         virtual fs::file_status symlink_status(const fs::path& path, std::error_code& ec) const override
         {
-            return fs::stdfs::symlink_status(path, ec);
+            return fs::symlink_status(path, ec);
         }
         virtual void write_contents(const fs::path& file_path, const std::string& data, std::error_code& ec) override
         {

toolsrc/src/vcpkg/base/strings.cpp

@@ -288,3 +288,43 @@ bool Strings::contains(StringView haystack, StringView needle)
 {
     return Strings::search(haystack, needle) != haystack.end();
 }
+
+namespace vcpkg::Strings
+{
+    namespace
+    {
+        template<class Integral>
+        std::string b32_encode_implementation(Integral x)
+        {
+            static_assert(std::is_integral<Integral>::value, "b64url_encode must take an integer type");
+            using Unsigned = std::make_unsigned_t<Integral>;
+            auto value = static_cast<Unsigned>(x);
+
+            // 32 values, plus the implicit \0
+            constexpr static char map[33] = "ABCDEFGHIJKLMNOP"
+                                            "QRSTUVWXYZ234567";
+
+            // log2(32)
+            constexpr static int shift = 5;
+            // 32 - 1
+            constexpr static auto mask = 31;
+
+            // ceiling(bitsize(Integral) / log2(32))
+            constexpr static auto result_size = (sizeof(value) * 8 + shift - 1) / shift;
+
+            std::string result;
+            result.reserve(result_size);
+
+            for (std::size_t i = 0; i < result_size; ++i)
+            {
+                result.push_back(map[value & mask]);
+                value >>= shift;
+            }
+
+            return result;
+        }
+    }
+
+    std::string b32_encode(std::uint64_t x) noexcept { return b32_encode_implementation(x); }
+
+}

toolsrc/src/vcpkg/build.cpp

@@ -586,7 +586,8 @@ namespace vcpkg::Build
                 if (fs.is_directory(file)) // Will only keep the logs
                 {
                     std::error_code ec;
-                    fs.remove_all(file, ec);
+                    fs::path failure_point;
+                    fs.remove_all(file, ec, failure_point);
                 }
             }
         }
@@ -701,8 +702,8 @@ namespace vcpkg::Build
         auto& fs = paths.get_filesystem();
 
         auto pkg_path = paths.package_dir(spec);
+        fs.remove_all(pkg_path, VCPKG_LINE_INFO);
         std::error_code ec;
-        fs.remove_all(pkg_path, ec);
         fs.create_directories(pkg_path, ec);
         auto files = fs.get_files_non_recursive(pkg_path);
         Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());
@@ -886,7 +887,7 @@ namespace vcpkg::Build
                     fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);
 
                     // clean up temporary directory
-                    fs.remove_all(tmp_log_path, ec);
+                    fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);
                 }
             }
 
@@ -1053,7 +1054,7 @@ namespace vcpkg::Build
             {
                 switch (maybe_option->second)
                 {
-                case VcpkgTripletVar::TARGET_ARCHITECTURE : 
+                case VcpkgTripletVar::TARGET_ARCHITECTURE :
                     pre_build_info.target_architecture = variable_value;
                     break;
                 case VcpkgTripletVar::CMAKE_SYSTEM_NAME :

toolsrc/src/vcpkg/commands.exportifw.cpp

@@ -352,13 +352,15 @@ namespace vcpkg::Export::IFW
         System::print2("Generating repository ", repository_dir.generic_u8string(), "...\n");
 
         std::error_code ec;
+        fs::path failure_point;
         Files::Filesystem& fs = paths.get_filesystem();
 
-        fs.remove_all(repository_dir, ec);
+        fs.remove_all(repository_dir, ec, failure_point);
         Checks::check_exit(VCPKG_LINE_INFO,
                            !ec,
-                           "Could not remove outdated repository directory %s",
+                           "Could not remove outdated repository directory %s due to file %s",
-                           repository_dir.generic_u8string());
+                           repository_dir.generic_u8string(),
+                           failure_point.string());
 
         const auto cmd_line = Strings::format(R"("%s" --packages "%s" "%s" > nul)",
                                               repogen_exe.u8string(),
@@ -414,16 +416,18 @@ namespace vcpkg::Export::IFW
                    const VcpkgPaths& paths)
     {
         std::error_code ec;
+        fs::path failure_point;
         Files::Filesystem& fs = paths.get_filesystem();
 
         // Prepare packages directory
         const fs::path ifw_packages_dir_path = get_packages_dir_path(export_id, ifw_options, paths);
 
-        fs.remove_all(ifw_packages_dir_path, ec);
+        fs.remove_all(ifw_packages_dir_path, ec, failure_point);
         Checks::check_exit(VCPKG_LINE_INFO,
                            !ec,
-                           "Could not remove outdated packages directory %s",
+                           "Could not remove outdated packages directory %s due to file %s",
-                           ifw_packages_dir_path.generic_u8string());
+                           ifw_packages_dir_path.generic_u8string(),
+                           failure_point.string());
 
         fs.create_directory(ifw_packages_dir_path, ec);
         Checks::check_exit(

toolsrc/src/vcpkg/commands.portsdiff.cpp

@@ -105,7 +105,7 @@ namespace vcpkg::Commands::PortsDiff
         std::map<std::string, VersionT> names_and_versions;
         for (auto&& port : all_ports)
             names_and_versions.emplace(port->core_paragraph->name, port->core_paragraph->version);
-        fs.remove_all(temp_checkout_path, ec);
+        fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO);
         return names_and_versions;
     }
 

toolsrc/src/vcpkg/export.cpp

@@ -400,8 +400,10 @@ namespace vcpkg::Export
         Files::Filesystem& fs = paths.get_filesystem();
         const fs::path export_to_path = paths.root;
         const fs::path raw_exported_dir_path = export_to_path / export_id;
+        fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
+
+        // TODO: error handling
         std::error_code ec;
-        fs.remove_all(raw_exported_dir_path, ec);
         fs.create_directory(raw_exported_dir_path, ec);
 
         // execute the plan
@@ -476,7 +478,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
 
         if (!opts.raw)
         {
-            fs.remove_all(raw_exported_dir_path, ec);
+            fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
         }
     }
 

toolsrc/src/vcpkg/install.cpp

@@ -354,8 +354,7 @@ namespace vcpkg::Install
             {
                 auto& fs = paths.get_filesystem();
                 const fs::path package_dir = paths.package_dir(action.spec);
-                std::error_code ec;
+                fs.remove_all(package_dir, VCPKG_LINE_INFO);
-                fs.remove_all(package_dir, ec);
             }
 
             if (action.build_options.clean_downloads == Build::CleanDownloads::YES)

toolsrc/src/vcpkg/remove.cpp

@@ -179,8 +179,7 @@ namespace vcpkg::Remove
         {
             System::printf("Purging package %s...\n", display_name);
             Files::Filesystem& fs = paths.get_filesystem();
-            std::error_code ec;
+            fs.remove_all(paths.packages / action.spec.dir(), VCPKG_LINE_INFO);
-            fs.remove_all(paths.packages / action.spec.dir(), ec);
             System::printf(System::Color::success, "Purging package %s... done\n", display_name);
         }
     }

toolsrc/vcpkg/vcpkg.vcxproj

@@ -147,4 +147,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

toolsrc/vcpkglib/vcpkglib.vcxproj

@@ -274,4 +274,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj

@@ -144,4 +144,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

toolsrc/vcpkgtest/vcpkgtest.vcxproj

@@ -23,10 +23,12 @@
     <ClCompile Include="..\src\vcpkg-tests\catch.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\chrono.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\dependencies.cpp" />
+    <ClCompile Include="..\src\vcpkg-tests\files.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\paragraph.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\plan.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\specifier.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\statusparagraphs.cpp" />
+    <ClCompile Include="..\src\vcpkg-tests\strings.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\supports.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\update.cpp" />
     <ClCompile Include="..\src\vcpkg-tests\util.cpp" />

toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters

@@ -48,6 +48,12 @@
     <ClCompile Include="..\src\vcpkg-tests\util.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\src\tests.files.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\src\tests.strings.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\include\vcpkg-tests\catch.h">