diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 493409152a31..a723cdb81084 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -125,6 +125,7 @@ FileCache::FileCache(const std::string & cache_name, const FileCacheSettings & s , keep_current_size_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_size_ratio]) , keep_current_elements_to_max_ratio(1 - settings[FileCacheSetting::keep_free_space_elements_ratio]) , keep_up_free_space_remove_batch(settings[FileCacheSetting::keep_free_space_remove_batch]) + , name(cache_name) , log(getLogger("FileCache(" + cache_name + ")")) , metadata(settings[FileCacheSetting::path], settings[FileCacheSetting::background_download_queue_size_limit], diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index b9e6363a3ab7..3a92430429a8 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -209,6 +209,8 @@ class FileCache : private boost::noncopyable void freeSpaceRatioKeepingThreadFunc(); + const String & getName() const { return name; } + private: using KeyAndOffset = FileCacheKeyAndOffset; @@ -228,6 +230,7 @@ class FileCache : private boost::noncopyable const double keep_current_elements_to_max_ratio; const size_t keep_up_free_space_remove_batch; + String name; LoggerPtr log; std::exception_ptr init_exception; diff --git a/src/Interpreters/Cache/FileCacheFactory.cpp b/src/Interpreters/Cache/FileCacheFactory.cpp index 6088c1bdce04..f741c89e5793 100644 --- a/src/Interpreters/Cache/FileCacheFactory.cpp +++ b/src/Interpreters/Cache/FileCacheFactory.cpp @@ -52,6 +52,15 @@ FileCacheFactory::CacheByName FileCacheFactory::getAll() return caches_by_name; } +FileCacheFactory::Caches FileCacheFactory::getUniqueInstances() +{ + std::lock_guard lock(mutex); + Caches caches; + for (const auto & [_, cache_data] : caches_by_name) + caches.insert(cache_data); + return caches; +} + FileCachePtr FileCacheFactory::get(const std::string & cache_name) { std::lock_guard lock(mutex); diff --git a/src/Interpreters/Cache/FileCacheFactory.h b/src/Interpreters/Cache/FileCacheFactory.h index 8aae5ae74047..03d89ab76fbb 100644 --- a/src/Interpreters/Cache/FileCacheFactory.h +++ b/src/Interpreters/Cache/FileCacheFactory.h @@ -5,6 +5,7 @@ #include #include +#include #include namespace DB @@ -36,6 +37,7 @@ class FileCacheFactory final : private boost::noncopyable using FileCacheDataPtr = std::shared_ptr; using CacheByName = std::unordered_map; + using Caches = std::unordered_set; static FileCacheFactory & instance(); @@ -52,6 +54,7 @@ class FileCacheFactory final : private boost::noncopyable const std::string & config_path); CacheByName getAll(); + Caches getUniqueInstances(); FileCacheDataPtr getByName(const std::string & cache_name); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f102c660aa9c..3f774e2a1b97 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -882,9 +882,8 @@ struct ContextSharedPart : boost::noncopyable /// Background operations in cache use background schedule pool. /// Deactivate them before destructing it. LOG_TRACE(log, "Shutting down caches"); - const auto & caches = FileCacheFactory::instance().getAll(); - for (const auto & [_, cache] : caches) - cache->cache->deactivateBackgroundOperations(); + for (const auto & cache_data : FileCacheFactory::instance().getUniqueInstances()) + cache_data->cache->deactivateBackgroundOperations(); FileCacheFactory::instance().clear(); { diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 76a09ca9ff9e..f02a6e8ddd6c 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -461,8 +461,7 @@ BlockIO InterpreterSystemQuery::execute() if (query.filesystem_cache_name.empty()) { - auto caches = FileCacheFactory::instance().getAll(); - for (const auto & [_, cache_data] : caches) + for (const auto & cache_data : FileCacheFactory::instance().getUniqueInstances()) { if (!cache_data->cache->isInitialized()) continue; @@ -523,11 +522,10 @@ BlockIO InterpreterSystemQuery::execute() if (query.filesystem_cache_name.empty()) { - auto caches = FileCacheFactory::instance().getAll(); - for (const auto & [cache_name, cache_data] : caches) + for (const auto & cache_data : FileCacheFactory::instance().getUniqueInstances()) { auto file_segments = cache_data->cache->sync(); - fill_data(cache_name, cache_data->cache, file_segments); + fill_data(cache_data->cache->getName(), cache_data->cache, file_segments); } } else diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 930b0fcb4752..80af95332a12 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -73,12 +73,11 @@ ServerAsynchronousMetrics::~ServerAsynchronousMetrics() void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint current_time, bool force_update, bool first_run, AsynchronousMetricValues & new_values) { { - auto caches = FileCacheFactory::instance().getAll(); size_t total_bytes = 0; size_t max_bytes = 0; size_t total_files = 0; - for (const auto & [_, cache_data] : caches) + for (const auto & cache_data : FileCacheFactory::instance().getUniqueInstances()) { total_bytes += cache_data->cache->getUsedCacheSize(); max_bytes += cache_data->cache->getMaxCacheSize(); diff --git a/src/Storages/System/StorageSystemFilesystemCache.cpp b/src/Storages/System/StorageSystemFilesystemCache.cpp index 2e9ba4daafb7..fd3c10f2e5e2 100644 --- a/src/Storages/System/StorageSystemFilesystemCache.cpp +++ b/src/Storages/System/StorageSystemFilesystemCache.cpp @@ -45,9 +45,15 @@ StorageSystemFilesystemCache::StorageSystemFilesystemCache(const StorageID & tab void StorageSystemFilesystemCache::fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector) const { - auto caches = FileCacheFactory::instance().getAll(); + auto caches_by_name = FileCacheFactory::instance().getAll(); + std::unordered_set caches; + for (const auto & [_, cache_data] : caches_by_name) + caches.insert(cache_data); + std::unordered_map> caches_by_instance; + for (const auto & [cache_name, cache_data] : caches_by_name) + caches_by_instance[cache_data].push_back(cache_name); - for (const auto & [cache_name, cache_data] : caches) + for (const auto & cache_data : caches) { const auto & cache = cache_data->cache; if (!cache->isInitialized()) @@ -55,36 +61,39 @@ void StorageSystemFilesystemCache::fillData(MutableColumns & res_columns, Contex cache->iterate([&](const FileSegment::Info & file_segment) { - size_t i = 0; - res_columns[i++]->insert(cache_name); - res_columns[i++]->insert(cache->getBasePath()); + for (const auto & cache_name : caches_by_instance.at(cache_data)) + { + size_t i = 0; + res_columns[i++]->insert(cache_name); + res_columns[i++]->insert(cache->getBasePath()); - /// Do not use `file_segment->getPath` here because it will lead to nullptr dereference - /// (because file_segments in getSnapshot doesn't have `cache` field set) + /// Do not use `file_segment->getPath` here because it will lead to nullptr dereference + /// (because file_segments in getSnapshot doesn't have `cache` field set) - const auto path = cache->getFileSegmentPath( - file_segment.key, file_segment.offset, file_segment.kind, - FileCache::UserInfo(file_segment.user_id, file_segment.user_weight)); - res_columns[i++]->insert(path); - res_columns[i++]->insert(file_segment.key.toString()); - res_columns[i++]->insert(file_segment.range_left); - res_columns[i++]->insert(file_segment.range_right); - res_columns[i++]->insert(file_segment.size); - res_columns[i++]->insert(FileSegment::stateToString(file_segment.state)); - res_columns[i++]->insert(file_segment.download_finished_time); - res_columns[i++]->insert(file_segment.cache_hits); - res_columns[i++]->insert(file_segment.references); - res_columns[i++]->insert(file_segment.downloaded_size); - res_columns[i++]->insert(toString(file_segment.kind)); - res_columns[i++]->insert(file_segment.is_unbound); - res_columns[i++]->insert(file_segment.user_id); + const auto path = cache->getFileSegmentPath( + file_segment.key, file_segment.offset, file_segment.kind, + FileCache::UserInfo(file_segment.user_id, file_segment.user_weight)); + res_columns[i++]->insert(path); + res_columns[i++]->insert(file_segment.key.toString()); + res_columns[i++]->insert(file_segment.range_left); + res_columns[i++]->insert(file_segment.range_right); + res_columns[i++]->insert(file_segment.size); + res_columns[i++]->insert(FileSegment::stateToString(file_segment.state)); + res_columns[i++]->insert(file_segment.download_finished_time); + res_columns[i++]->insert(file_segment.cache_hits); + res_columns[i++]->insert(file_segment.references); + res_columns[i++]->insert(file_segment.downloaded_size); + res_columns[i++]->insert(toString(file_segment.kind)); + res_columns[i++]->insert(file_segment.is_unbound); + res_columns[i++]->insert(file_segment.user_id); - std::error_code ec; - auto size = fs::file_size(path, ec); - if (!ec) - res_columns[i++]->insert(size); - else - res_columns[i++]->insertDefault(); + std::error_code ec; + auto size = fs::file_size(path, ec); + if (!ec) + res_columns[i++]->insert(size); + else + res_columns[i++]->insertDefault(); + } }, FileCache::getCommonUser().user_id); } }