Fixed and polished SMP store stats collection. SMP shared memory cache stats were not collected. Mean disk object size stats were aggregated inaccurately for SMP. Moved Store-related stats into a dedicated StoreStats class, encapsulating memory cache-related (mem), disk cache-related (swap), and global store stats. Used consistent naming scheme to make memory and disk stats more alike (we could create a class to represent "store device" stats of sorts, but that seems like an overkill at this time). Moved Store stats collection into corresponding Store classes rather than forcing GetInfo() in stat.cc to know how to deal with all Store stats. TODO: Polish mgr:info display of Store stats along the same mem/disk symmetry lines. === modified file 'src/Makefile.am' --- src/Makefile.am 2011-10-04 17:28:03 +0000 +++ src/Makefile.am 2011-10-07 22:54:45 +0000 @@ -442,40 +442,42 @@ store_rebuild.cc \ store_swapin.cc \ store_swapmeta.cc \ store_swapout.cc \ StoreMeta.cc \ StoreMeta.h \ StoreMetaMD5.cc \ StoreMetaMD5.h \ StoreMetaSTD.cc \ StoreMetaSTD.h \ StoreMetaSTDLFS.cc \ StoreMetaSTDLFS.h \ StoreMetaObjSize.h \ StoreMetaUnpacker.cc \ StoreMetaUnpacker.h \ StoreMetaURL.cc \ StoreMetaURL.h \ StoreMetaVary.cc \ StoreMetaVary.h \ StoreSearch.h \ + StoreStats.cc \ + StoreStats.h \ StoreSwapLogData.cc \ StoreSwapLogData.h \ Server.cc \ Server.h \ structs.h \ swap_log_op.h \ SwapDir.cc \ SwapDir.h \ MemStore.cc \ MemStore.h \ time.cc \ TimeOrTag.h \ tools.cc \ tunnel.cc \ typedefs.h \ $(UNLINKDSOURCE) \ url.cc \ URL.h \ URLScheme.cc \ URLScheme.h \ @@ -1051,40 +1053,41 @@ HttpReply.cc \ HttpReply.h \ HttpStatusCode.h \ HttpStatusLine.cc \ HttpStatusLine.h \ mem.cc \ MemBuf.cc \ MemBuf.h \ mime_header.cc \ Packer.cc \ Packer.h \ SquidString.h \ SquidTime.h \ String.cc \ tests/stub_cache_cf.cc \ tests/stub_cache_manager.cc \ tests/stub_debug.cc \ tests/stub_HelperChildConfig.cc \ tests/stub_StatHist.cc \ tests/stub_store.cc \ + tests/stub_store_stats.cc \ tests/testHttpReply.cc \ tests/testHttpReply.h \ tests/testMain.cc \ time.cc \ wordlist.cc nodist_tests_testHttpReply_SOURCES=\ $(TESTSOURCES) tests_testHttpReply_LDFLAGS = $(LIBADD_DL) tests_testHttpReply_LDADD=\ acl/libapi.la \ acl/libstate.la \ $(AUTH_LIBS) \ ip/libip.la \ base/libbase.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ $(top_builddir)/lib/libmiscutil.la \ $(SQUID_CPPUNIT_LIBS) \ $(SQUID_CPPUNIT_LA) \ $(COMPAT_LIB) \ @@ -1153,40 +1156,41 @@ StoreMetaVary.cc \ StoreSwapLogData.cc \ store_key_md5.cc \ swap_log_op.cc \ swap_log_op.h \ SwapDir.cc \ SwapDir.h \ tests/stub_access_log.cc \ tests/stub_cache_cf.cc \ tests/stub_comm.cc \ tests/stub_debug.cc \ tests/stub_DelayId.cc \ tests/stub_DiskIOModule.cc \ tests/stub_fd.cc \ tests/stub_HttpRequest.cc \ tests/stub_MemObject.cc \ tests/stub_MemStore.cc \ tests/stub_mime.cc \ tests/stub_store.cc \ tests/stub_store_rebuild.cc \ + tests/stub_store_stats.cc \ tests/stub_store_swapout.cc \ tests/stub_tools.cc \ tests/stub_cache_manager.cc \ tests/testACLMaxUserIP.cc \ tests/testACLMaxUserIP.h \ tests/testMain.cc \ time.cc \ url.cc \ URL.h \ URLScheme.cc \ URLScheme.h \ mem.cc \ MemBuf.cc \ wordlist.cc nodist_tests_testACLMaxUserIP_SOURCES= \ $(TESTSOURCES) tests_testACLMaxUserIP_LDADD= \ $(AUTH_ACL_LIBS) \ ident/libident.la \ acl/libacls.la \ @@ -1227,40 +1231,41 @@ $(XTRA_LIBS) tests_testBoilerplate_LDFLAGS = $(LIBADD_DL) tests_testBoilerplate_DEPENDENCIES = \ $(SQUID_CPPUNIT_LA) ## Tests of the CacheManager module. tests_testCacheManager_SOURCES = \ $(ACL_REGISTRATION_SOURCES) \ debug.cc \ HttpParser.cc \ HttpParser.h \ HttpRequest.cc \ HttpRequestMethod.cc \ mem.cc \ String.cc \ tests/testCacheManager.cc \ tests/testCacheManager.h \ tests/testMain.cc \ tests/stub_main_cc.cc \ tests/stub_ipc_Forwarder.cc \ + tests/stub_store_stats.cc \ time.cc \ BodyPipe.cc \ cache_manager.cc \ cache_cf.cc \ ProtoPort.cc \ ProtoPort.h \ CacheDigest.cc \ carp.cc \ cbdata.cc \ ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ client_side_request.cc \ ClientInfo.h \ clientStream.cc \ ConfigOption.cc \ ConfigParser.cc \ CpuAffinityMap.cc \ CpuAffinityMap.h \ @@ -1475,40 +1480,41 @@ tests/stub_cache_cf.cc \ tests/stub_cache_manager.cc \ tests/stub_client_db.cc \ tests/stub_client_side_request.cc \ tests/stub_debug.cc \ tests/stub_errorpage.cc \ tests/stub_helper.cc \ tests/stub_HelperChildConfig.cc \ tests/stub_HttpRequest.cc \ tests/stub_http.cc \ tests/stub_icp.cc \ tests/stub_internal.cc \ tests/stub_ipc.cc \ tests/stub_ipcache.cc \ tests/stub_libicmp.cc \ tests/stub_MemStore.cc \ tests/stub_mime.cc \ tests/stub_pconn.cc \ tests/stub_Port.cc \ tests/stub_store_client.cc \ + tests/stub_store_stats.cc \ tests/stub_store_rebuild.cc \ tests/stub_tools.cc \ tests/stub_UdsOp.cc \ tests/testDiskIO.cc \ tests/testDiskIO.h \ tests/testMain.cc \ tests/testStoreSupport.cc \ tests/testStoreSupport.h \ time.cc \ $(UNLINKDSOURCE) \ url.cc \ URLScheme.cc \ $(WIN32_SOURCE) \ wordlist.cc nodist_tests_testDiskIO_SOURCES= \ $(TESTSOURCES) \ $(DISKIO_GEN_SOURCE) \ SquidMath.cc \ SquidMath.h \ swap_log_op.cc @@ -1663,40 +1669,41 @@ store_swapmeta.cc \ store_swapout.cc \ StoreFileSystem.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ String.cc \ SwapDir.cc \ tests/CapturingStoreEntry.h \ tests/testEvent.cc \ tests/testEvent.h \ tests/testMain.cc \ tests/stub_main_cc.cc \ tests/stub_ipc_Forwarder.cc \ + tests/stub_store_stats.cc \ time.cc \ tools.cc \ tunnel.cc \ MemStore.cc \ $(UNLINKDSOURCE) \ url.cc \ URLScheme.cc \ urn.cc \ wccp2.cc \ whois.cc \ $(WIN32_SOURCE) \ wordlist.cc nodist_tests_testEvent_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_testEvent_LDADD = \ $(AUTH_ACL_LIBS) \ ident/libident.la \ acl/libacls.la \ eui/libeui.la \ @@ -1847,40 +1854,41 @@ store_swapin.cc \ store_swapmeta.cc \ store_swapout.cc \ StoreFileSystem.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ String.cc \ SwapDir.cc \ tests/testEventLoop.cc \ tests/testEventLoop.h \ tests/testMain.cc \ tests/stub_main_cc.cc \ tests/stub_ipc_Forwarder.cc \ + tests/stub_store_stats.cc \ time.cc \ tools.cc \ tunnel.cc \ MemStore.cc \ $(UNLINKDSOURCE) \ url.cc \ URLScheme.cc \ urn.cc \ wccp2.cc \ whois.cc \ $(WIN32_SOURCE) \ wordlist.cc nodist_tests_testEventLoop_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_testEventLoop_LDADD = \ $(AUTH_ACL_LIBS) \ ident/libident.la \ acl/libacls.la \ eui/libeui.la \ @@ -2027,40 +2035,41 @@ store_rebuild.cc \ store_swapin.cc \ store_swapmeta.cc \ store_swapout.cc \ StoreFileSystem.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ String.cc \ SwapDir.cc \ tests/test_http_range.cc \ tests/stub_ipc_Forwarder.cc \ tests/stub_main_cc.cc \ tests/stub_MemStore.cc \ + tests/stub_store_stats.cc \ time.cc \ tools.cc \ tunnel.cc \ $(UNLINKDSOURCE) \ url.cc \ URLScheme.cc \ urn.cc \ wccp2.cc \ whois.cc \ $(WIN32_SOURCE) \ wordlist.cc nodist_tests_test_http_range_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_test_http_range_LDADD = \ $(AUTH_ACL_LIBS) \ ident/libident.la \ acl/libacls.la \ eui/libeui.la \ acl/libstate.la \ @@ -2132,40 +2141,41 @@ tests_testHttpParser_DEPENDENCIES = \ $(SQUID_CPPUNIT_LA) ## Tests of the HttpRequest module. tests_testHttpRequest_SOURCES = \ $(ACL_REGISTRATION_SOURCES) \ HttpParser.cc \ HttpParser.h \ HttpRequest.cc \ HttpRequestMethod.cc \ mem.cc \ String.cc \ tests/testHttpRequest.h \ tests/testHttpRequest.cc \ tests/testHttpRequestMethod.h \ tests/testHttpRequestMethod.cc \ tests/testMain.cc \ tests/stub_DiskIOModule.cc \ tests/stub_main_cc.cc \ tests/stub_ipc_Forwarder.cc \ + tests/stub_store_stats.cc \ time.cc \ BodyPipe.cc \ cache_manager.cc \ cache_cf.cc \ debug.cc \ ProtoPort.cc \ ProtoPort.h \ CacheDigest.cc \ carp.cc \ cbdata.cc \ ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ client_side_request.cc \ ClientInfo.h \ clientStream.cc \ ConfigOption.cc \ ConfigParser.cc \ CpuAffinityMap.cc \ @@ -2369,40 +2379,41 @@ tests/stub_acl.cc \ tests/stub_cache_cf.cc \ tests/stub_cache_manager.cc \ tests/stub_client_side_request.cc \ tests/stub_comm.cc \ tests/stub_debug.cc \ tests/stub_DiskIOModule.cc \ tests/stub_errorpage.cc \ tests/stub_fd.cc \ tests/stub_helper.cc \ tests/stub_HelperChildConfig.cc \ tests/stub_http.cc \ tests/stub_HttpReply.cc \ tests/stub_HttpRequest.cc \ tests/stub_libcomm.cc \ tests/stub_MemObject.cc \ tests/stub_MemStore.cc \ tests/stub_mime.cc \ tests/stub_Port.cc \ tests/stub_store_client.cc \ + tests/stub_store_stats.cc \ tests/stub_store_rebuild.cc \ tests/stub_store_swapout.cc \ tests/stub_tools.cc \ tests/stub_UdsOp.cc \ tests/testMain.cc \ tests/testStore.cc \ tests/testStore.h \ tests/testStoreEntryStream.cc \ tests/testStoreEntryStream.h \ tests/testStoreController.cc \ tests/testStoreController.h \ tests/testStoreHashIndex.cc \ tests/testStoreHashIndex.h \ tests/testStoreSupport.cc \ tests/testStoreSupport.h \ tests/TestSwapDir.cc \ tests/TestSwapDir.h \ time.cc \ url.cc \ URLScheme.cc \ @@ -2493,40 +2504,41 @@ ipc/libipc.la \ mgr/libmgr.la \ $(REPL_OBJS) \ $(SQUID_CPPUNIT_LA) tests_testUfs_SOURCES = \ tests/testUfs.cc \ tests/testMain.cc \ tests/testUfs.h \ tests/stub_cache_manager.cc \ tests/stub_client_db.cc \ tests/stub_HelperChildConfig.cc \ tests/stub_icp.cc \ tests/stub_ipc.cc \ tests/stub_MemStore.cc \ tests/stub_pconn.cc \ tests/stub_Port.cc \ tests/stub_UdsOp.cc \ tests/stub_internal.cc \ tests/stub_store_rebuild.cc \ + tests/stub_store_stats.cc \ fd.cc \ disk.cc \ filemap.cc \ HttpBody.cc \ HttpReply.cc \ HttpStatusLine.cc \ int.cc \ list.cc \ MemObject.cc \ StoreSwapLogData.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreFileSystem.cc \ store_io.cc \ @@ -2624,40 +2636,41 @@ $(SQUID_CPPUNIT_LIBS) \ $(SSLLIB) \ $(COMPAT_LIB) \ $(XTRA_LIBS) tests_testUfs_LDFLAGS = $(LIBADD_DL) tests_testUfs_DEPENDENCIES = \ $(SWAP_TEST_DS) tests_testCoss_SOURCES = \ tests/testCoss.cc \ tests/testMain.cc \ tests/testCoss.h \ tests/stub_cache_manager.cc \ tests/stub_client_db.cc \ tests/stub_debug.cc \ tests/stub_HelperChildConfig.cc \ tests/stub_internal.cc \ tests/stub_ipc.cc \ tests/stub_pconn.cc \ tests/stub_store_rebuild.cc \ + tests/stub_store_stats.cc \ fd.cc \ disk.cc \ filemap.cc \ HttpBody.cc \ HttpReply.cc \ HttpStatusLine.cc \ int.cc \ list.cc \ MemObject.cc \ StoreSwapLogData.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreFileSystem.cc \ store_io.cc \ @@ -2693,40 +2706,41 @@ HttpHeader.cc \ mem.cc \ ClientInfo.h \ MemBuf.cc \ HttpHdrContRange.cc \ Packer.cc \ HttpHdrCc.cc \ HttpHdrSc.cc \ HttpHdrScTarget.cc \ url.cc \ StatHist.cc \ HttpHdrRange.cc \ ETag.cc \ tests/stub_errorpage.cc \ tests/stub_HttpRequest.cc \ tests/stub_access_log.cc \ refresh.cc \ tests/stub_MemStore.cc \ tests/stub_Port.cc \ tests/stub_store_client.cc \ + tests/stub_store_stats.cc \ tests/stub_tools.cc \ tests/stub_UdsOp.cc \ tests/testStoreSupport.cc \ tests/testStoreSupport.h \ time.cc \ URLScheme.cc \ wordlist.cc \ $(DISKIO_SOURCE) nodist_tests_testCoss_SOURCES = \ swap_log_op.cc \ SquidMath.cc \ SquidMath.h \ $(TESTSOURCES) \ $(DISKIO_GEN_SOURCE) tests_testCoss_LDADD = \ anyp/libanyp.la \ libsquid.la \ $(REGEXLIB) \ $(AUTH_ACL_LIBS) \ @@ -2751,40 +2765,41 @@ base/libbase.la \ $(top_builddir)/lib/libmisccontainers.la \ $(top_builddir)/lib/libmiscencoding.la \ $(top_builddir)/lib/libmiscutil.la \ $(SQUID_CPPUNIT_LIBS) \ $(REGEXLIB) \ $(SSLLIB) \ $(COMPAT_LIB) \ $(XTRA_LIBS) tests_testCoss_LDFLAGS = $(LIBADD_DL) tests_testCoss_DEPENDENCIES = \ $(SWAP_TEST_DS) tests_testNull_SOURCES = \ tests/testNull.cc \ tests/testMain.cc \ tests/testNull.h \ tests/stub_internal.cc \ tests/stub_CommIO.cc \ tests/stub_store_rebuild.cc \ + tests/stub_store_stats.cc \ fd.cc \ disk.cc \ filemap.cc \ HttpBody.cc \ HttpReply.cc \ HttpStatusLine.cc \ int.cc \ list.cc \ MemObject.cc \ StoreSwapLogData.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreFileSystem.cc \ store_io.cc \ @@ -2820,40 +2835,41 @@ tests/stub_mime.cc \ HttpHeaderTools.cc \ HttpHeader.cc \ mem.cc \ ClientInfo.h \ MemBuf.cc \ HttpHdrContRange.cc \ Packer.cc \ HttpHdrCc.cc \ HttpHdrSc.cc \ HttpHdrScTarget.cc \ url.cc \ StatHist.cc \ HttpHdrRange.cc \ ETag.cc \ tests/stub_errorpage.cc \ tests/stub_HttpRequest.cc \ tests/stub_access_log.cc \ refresh.cc \ tests/stub_store_client.cc \ + tests/stub_store_stats.cc \ tests/stub_tools.cc \ tests/testStoreSupport.cc \ tests/testStoreSupport.h \ time.cc \ URLScheme.cc \ wordlist.cc \ $(DISKIO_SOURCE) nodist_tests_testNull_SOURCES = \ $(TESTSOURCES) \ $(DISKIO_GEN_SOURCE) tests_testNull_LDADD = \ $(REGEXLIB) \ $(AUTH_ACL_LIBS) \ ident/libident.la \ acl/libacls.la \ eui/libeui.la \ acl/libstate.la \ $(AUTH_LIBS) \ acl/libapi.la \ @@ -2987,40 +3003,41 @@ store_swapin.cc \ store_swapmeta.cc \ store_swapout.cc \ StoreFileSystem.cc \ StoreIOState.cc \ StoreMeta.cc \ StoreMetaMD5.cc \ StoreMetaSTD.cc \ StoreMetaSTDLFS.cc \ StoreMetaUnpacker.cc \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ String.cc \ SwapDir.cc \ MemStore.cc \ tests/stub_debug.cc \ tests/stub_DiskIOModule.cc \ tests/stub_main_cc.cc \ tests/stub_ipc_Forwarder.cc \ + tests/stub_store_stats.cc \ tests/testURL.cc \ tests/testURL.h \ tests/testURLScheme.cc \ tests/testURLScheme.h \ tests/testMain.cc \ time.cc \ tools.cc \ tunnel.cc \ url.cc \ URLScheme.cc \ urn.cc \ wccp2.cc \ whois.cc \ FadingCounter.cc \ $(WIN32_SOURCE) \ wordlist.cc nodist_tests_testURL_SOURCES = \ $(BUILT_SOURCES) tests_testURL_LDADD = \ anyp/libanyp.la \ === modified file 'src/MemStore.cc' --- src/MemStore.cc 2011-10-04 17:27:22 +0000 +++ src/MemStore.cc 2011-10-07 22:54:45 +0000 @@ -1,33 +1,34 @@ /* * $Id$ * * DEBUG: section 20 Memory Cache * */ #include "config.h" #include "base/RunnersRegistry.h" #include "ipc/mem/Page.h" #include "ipc/mem/Pages.h" #include "MemObject.h" #include "MemStore.h" +#include "StoreStats.h" #include "HttpReply.h" /// shared memory segment path to use for MemStore maps static const char *ShmLabel = "cache_mem"; // XXX: support storage using more than one page per entry MemStore::MemStore(): map(NULL), theCurrentSize(0) { } MemStore::~MemStore() { delete map; } void MemStore::init() { const int64_t entryLimit = EntryLimit(); @@ -35,40 +36,53 @@ return; // no memory cache configured or a misconfiguration const int64_t diskMaxSize = Store::Root().maxObjectSize(); const int64_t memMaxSize = maxObjectSize(); if (diskMaxSize == -1) { debugs(20, DBG_IMPORTANT, "WARNING: disk-cache maximum object size " "is unlimited but mem-cache maximum object size is " << memMaxSize / 1024.0 << " KB"); } else if (diskMaxSize > memMaxSize) { debugs(20, DBG_IMPORTANT, "WARNING: disk-cache maximum object size " "is too large for mem-cache: " << diskMaxSize / 1024.0 << " KB > " << memMaxSize / 1024.0 << " KB"); } map = new MemStoreMap(ShmLabel); map->cleaner = this; } void +MemStore::getStats(StoreInfoStats &stats) const +{ + const size_t pageSize = Ipc::Mem::PageSize(); + + stats.mem.shared = true; + stats.mem.capacity = + Ipc::Mem::PageLimit(Ipc::Mem::PageId::cachePage) * pageSize; + stats.mem.size = + Ipc::Mem::PageLevel(Ipc::Mem::PageId::cachePage) * pageSize; + stats.mem.count = currentCount(); +} + +void MemStore::stat(StoreEntry &e) const { storeAppendPrintf(&e, "\n\nShared Memory Cache\n"); storeAppendPrintf(&e, "Maximum Size: %.0f KB\n", Config.memMaxSize/1024.0); if (map) { const int limit = map->entryLimit(); storeAppendPrintf(&e, "Maximum entries: %9d\n", limit); if (limit > 0) { storeAppendPrintf(&e, "Current entries: %"PRId64" %.2f%%\n", currentCount(), (100.0 * currentCount() / limit)); if (limit < 100) { // XXX: otherwise too expensive to count Ipc::ReadWriteLockStats stats; map->updateStats(stats); stats.dump(e); } } } === modified file 'src/MemStore.h' --- src/MemStore.h 2011-09-06 22:32:30 +0000 +++ src/MemStore.h 2011-10-07 22:54:45 +0000 @@ -20,40 +20,41 @@ /// Unlike a disk store (SwapDir), operations are synchronous (and fast). class MemStore: public Store, public Ipc::StoreMapCleaner { public: MemStore(); virtual ~MemStore(); /// cache the entry or forget about it until the next considerKeeping call void considerKeeping(StoreEntry &e); /* Store API */ virtual int callback(); virtual StoreEntry * get(const cache_key *); virtual void get(String const key , STOREGETCLIENT callback, void *cbdata); virtual void init(); virtual uint64_t maxSize() const; virtual uint64_t minSize() const; virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual int64_t maxObjectSize() const; + virtual void getStats(StoreInfoStats &stats) const; virtual void stat(StoreEntry &) const; virtual StoreSearch *search(String const url, HttpRequest *); virtual void reference(StoreEntry &); virtual bool dereference(StoreEntry &); virtual void maintain(); static int64_t EntryLimit(); protected: bool willFit(int64_t needed); void keep(StoreEntry &e); bool copyToShm(StoreEntry &e, MemStoreMap::Extras &extras); bool copyFromShm(StoreEntry &e, const MemStoreMap::Extras &extras); // Ipc::StoreMapCleaner API virtual void cleanReadable(const sfileno fileno); private: MemStoreMap *map; ///< index of mem-cached entries === modified file 'src/Store.h' --- src/Store.h 2011-09-11 21:44:54 +0000 +++ src/Store.h 2011-10-07 22:54:45 +0000 @@ -28,66 +28,57 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #ifndef SQUID_STORE_H #define SQUID_STORE_H /** \defgroup StoreAPI Store API \ingroup FileSystems */ #include "squid.h" #include "StoreIOBuffer.h" #include "Range.h" #include "RefCount.h" #include "CommRead.h" #include "comm/forward.h" #include "Packer.h" #include "RemovalPolicy.h" +#include "StoreStats.h" #if USE_SQUID_ESI #include "esi/Element.h" #endif #if HAVE_OSTREAM #include #endif class AsyncCall; class StoreClient; class MemObject; class StoreSearch; class SwapDir; -typedef struct { - - struct { - int calls; - int select_fail; - int create_fail; - int success; - } create; -} StoreIoStats; - extern StoreIoStats store_io_stats; /// maximum number of entries per cache_dir enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen /** \ingroup StoreAPI */ class StoreEntry : public hash_link { public: static DeferredRead::DeferrableRead DeferReader; bool checkDeferRead(int fd) const; virtual const char *getMD5Text() const; StoreEntry(); StoreEntry(const char *url, const char *log_url); virtual ~StoreEntry(); @@ -298,40 +289,43 @@ virtual void init() = 0; /** * The maximum size the store will support in normal use. Inaccuracy is permitted, * but may throw estimates for memory etc out of whack. */ virtual uint64_t maxSize() const = 0; /** The minimum size the store will shrink to via normal housekeeping */ virtual uint64_t minSize() const = 0; /** current store size */ virtual uint64_t currentSize() const = 0; /** the total number of objects stored */ virtual uint64_t currentCount() const = 0; /** the maximum object size that can be stored, -1 if unlimited */ virtual int64_t maxObjectSize() const = 0; + /// collect cache storage-related statistics + virtual void getStats(StoreInfoStats &stats) const = 0; + /** * Output stats to the provided store entry. \todo make these calls asynchronous */ virtual void stat(StoreEntry &) const = 0; /** Sync the store prior to shutdown */ virtual void sync(); /** remove a Store entry from the store */ virtual void unlink (StoreEntry &); /* search in the store */ virtual StoreSearch *search(String const url, HttpRequest *) = 0; /* pulled up from SwapDir for migration.... probably do not belong here */ virtual void reference(StoreEntry &) = 0; /* Reference this object */ /// Undo reference(), returning false iff idle e should be destroyed virtual bool dereference(StoreEntry &e) = 0; === modified file 'src/StoreHashIndex.h' --- src/StoreHashIndex.h 2011-05-12 03:58:16 +0000 +++ src/StoreHashIndex.h 2011-10-07 22:54:45 +0000 @@ -56,40 +56,41 @@ virtual StoreEntry * get (const cache_key *); virtual void get (String const, STOREGETCLIENT, void * cbdata); virtual void init(); virtual void sync(); virtual uint64_t maxSize() const; virtual uint64_t minSize() const; virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual int64_t maxObjectSize() const; + virtual void getStats(StoreInfoStats &stats) const; virtual void stat(StoreEntry&) const; virtual void reference(StoreEntry&); virtual bool dereference(StoreEntry&); virtual void maintain(); virtual StoreSearch *search(String const url, HttpRequest *); private: /* migration logic */ StorePointer store(int const x) const; SwapDir &dir(int const idx) const; }; class StoreHashIndexEntry : public StoreEntry {}; class StoreSearchHashIndex : public StoreSearch === added file 'src/StoreStats.cc' --- src/StoreStats.cc 1970-01-01 00:00:00 +0000 +++ src/StoreStats.cc 2011-10-05 23:11:27 +0000 @@ -0,0 +1,61 @@ +/* + * $Id$ + * + * DEBUG: section 20 Storage Manager Statistics + * + */ + +#include "config.h" +#include "protos.h" /* for xmemset only */ +#include "StoreStats.h" + + +/* StoreInfoStats */ + +StoreInfoStats::StoreInfoStats() +{ + xmemset(this, 0, sizeof(*this)); +} + +StoreInfoStats & +StoreInfoStats::operator +=(const StoreInfoStats &stats) +{ + swap.size += stats.swap.size; + swap.capacity += stats.swap.capacity; + mem.count += stats.mem.count; + swap.count += stats.swap.count; + + // Assume that either all workers use shared memory cache or none do. + // It is possible but difficult to report correct stats for an arbitrary + // mix, and only rather unusual deployments can benefit from mixing. + + // If workers share memory, we will get shared stats from those workers + // and stats from other processes. Do not rely on any specific order and + // just ignore other processes stats because they are zero in most setups. + if (stats.mem.shared) { // workers share memory + // use the latest reported stats, they all should be about the same + mem.shared = true; + mem.capacity = stats.mem.capacity; + mem.size = stats.mem.size; + } else if (!mem.shared) { // do not corrupt shared stats, if any + // workers do not share so we must add everything up + mem.capacity += stats.mem.capacity; + mem.size += stats.mem.size; + } + + open_disk_fd += stats.open_disk_fd; + store_entry_count += stats.store_entry_count; + mem_object_count += stats.mem_object_count; + + return *this; +} + + + +/* StoreIoStats */ + +StoreIoStats::StoreIoStats() +{ + xmemset(this, 0, sizeof(*this)); +} + === added file 'src/StoreStats.h' --- src/StoreStats.h 1970-01-01 00:00:00 +0000 +++ src/StoreStats.h 2011-10-12 20:49:43 +0000 @@ -0,0 +1,57 @@ +#ifndef SQUID_STORE_STATS_H +#define SQUID_STORE_STATS_H + +/// High-level store statistics used by mgr:info action. Used inside PODs! +class StoreInfoStats { +public: + StoreInfoStats(); + + StoreInfoStats &operator +=(const StoreInfoStats &stats); + + /// disk cache (cache_dir) storage stats + struct Swap { + double size; ///< disk cache bytes currently in use + double count; ///< number of disk-cached objects + double capacity; ///< the limit for the disk cache size + + /// mean size of a disk-cached object + double meanObjectSize() const { return count > 0 ? size/count : 0.0; } + + /// number of unused disk cache bytes + double available() const { return capacity - size; } + } swap; + + /// memory cache (cache_mem) storage stats + struct Mem { + bool shared; ///< whether memory cache is shared among workers + double size; ///< memory cache bytes currently in use + double count; ///< number of memory-cached objects + double capacity; ///< the limit for the memory cache size + + /// computes mean size of a memory-cached object + double meanObjectSize() const { return count > 0 ? size/count : 0.0; } + + /// number of unused memory cache bytes + double available() const { return capacity - size; } + } mem; + + double open_disk_fd; ///< number of opened disk files + double store_entry_count; ///< number of StoreEntry objects in existence + double mem_object_count; ///< number of MemObject objects in existence +}; + +// TODO: this should be adjusted for use in StoreIoActionData, DiskdActionData +/// Store statistics related to low-level I/O. +class StoreIoStats { +public: + StoreIoStats(); + + struct { + int calls; + int select_fail; + int create_fail; + int success; + } create; ///< cache_dir selection and disk entry creation stats +}; + +#endif /* SQUID_STORE_STATS_H */ === modified file 'src/SwapDir.cc' --- src/SwapDir.cc 2011-10-04 17:28:03 +0000 +++ src/SwapDir.cc 2011-10-07 22:54:47 +0000 @@ -52,40 +52,51 @@ // TODO: should we delete repl? xfree(path); } void SwapDir::create() {} void SwapDir::dump(StoreEntry &)const {} bool SwapDir::doubleCheck(StoreEntry &) { return false; } void SwapDir::unlink(StoreEntry &) {} void +SwapDir::getStats(StoreInfoStats &stats) const +{ + if (!doReportStat()) + return; + + stats.swap.size = currentSize(); + stats.swap.capacity = maxSize(); + stats.swap.count = currentCount(); +} + +void SwapDir::stat(StoreEntry &output) const { if (!doReportStat()) return; storeAppendPrintf(&output, "Store Directory #%d (%s): %s\n", index, type(), path); storeAppendPrintf(&output, "FS Block Size %d Bytes\n", fs.blksize); statfs(output); if (repl) { storeAppendPrintf(&output, "Removal policy: %s\n", repl->_type); if (repl->Stats) repl->Stats(repl, &output); } } void === modified file 'src/SwapDir.h' --- src/SwapDir.h 2011-10-04 17:28:03 +0000 +++ src/SwapDir.h 2011-10-07 22:54:47 +0000 @@ -58,40 +58,41 @@ virtual void get(String const, STOREGETCLIENT, void * cbdata); /* Store parent API */ virtual void handleIdleEntry(StoreEntry &e); virtual void init(); virtual void maintain(); /* perform regular maintenance should be private and self registered ... */ virtual uint64_t maxSize() const; virtual uint64_t minSize() const; virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual int64_t maxObjectSize() const; + virtual void getStats(StoreInfoStats &stats) const; virtual void stat(StoreEntry &) const; virtual void sync(); /* Sync the store prior to shutdown */ virtual StoreSearch *search(String const url, HttpRequest *); virtual void reference(StoreEntry &); /* Reference this object */ virtual bool dereference(StoreEntry &); /* Unreference this object */ /* the number of store dirs being rebuilt. */ static int store_dirs_rebuilding; private: void createOneStore(Store &aStore); StorePointer swapDir; ///< summary view of all disk caches MemStore *memStore; ///< memory cache }; @@ -129,40 +130,41 @@ char const *type() const; virtual bool needsDiskStrand() const; ///< needs a dedicated kid process virtual bool active() const; ///< may be used in this strand /// whether stat should be reported by this SwapDir virtual bool doReportStat() const { return active(); } /* official Store interface functions */ virtual void diskFull(); virtual StoreEntry * get(const cache_key *); virtual void get(String const, STOREGETCLIENT, void * cbdata); virtual uint64_t maxSize() const { return max_size;} virtual uint64_t minSize() const; virtual int64_t maxObjectSize() const { return max_objsize; } + virtual void getStats(StoreInfoStats &stats) const; virtual void stat (StoreEntry &anEntry) const; virtual StoreSearch *search(String const url, HttpRequest *) = 0; /* migrated from store_dir.cc */ bool objectSizeIsAcceptable(int64_t objsize) const; /// called when the entry is about to forget its association with cache_dir virtual void disconnect(StoreEntry &) {} /// called when entry swap out is complete virtual void swappedOut(const StoreEntry &e) = 0; protected: void parseOptions(int reconfiguring); void dumpOptions(StoreEntry * e) const; virtual ConfigOption *getOptionTree() const; virtual bool allowOptionReconfigure(const char *const) const { return true; } int64_t sizeInBlocks(const int64_t size) const { return (size + fs.blksize - 1) / fs.blksize; } === modified file 'src/mgr/InfoAction.cc' --- src/mgr/InfoAction.cc 2011-08-10 15:54:51 +0000 +++ src/mgr/InfoAction.cc 2011-10-07 22:54:47 +0000 @@ -41,46 +41,43 @@ icp_pkts_recv += stats.icp_pkts_recv; icp_pkts_sent += stats.icp_pkts_sent; icp_replies_queued += stats.icp_replies_queued; #if USE_HTCP htcp_pkts_recv += stats.htcp_pkts_recv; htcp_pkts_sent += stats.htcp_pkts_sent; #endif request_failure_ratio += stats.request_failure_ratio; avg_client_http_requests += stats.avg_client_http_requests; avg_icp_messages += stats.avg_icp_messages; select_loops += stats.select_loops; avg_loop_time += stats.avg_loop_time; request_hit_ratio5 += stats.request_hit_ratio5; request_hit_ratio60 += stats.request_hit_ratio60; byte_hit_ratio5 += stats.byte_hit_ratio5; byte_hit_ratio60 += stats.byte_hit_ratio60; request_hit_mem_ratio5 += stats.request_hit_mem_ratio5; request_hit_mem_ratio60 += stats.request_hit_mem_ratio60; request_hit_disk_ratio5 += stats.request_hit_disk_ratio5; request_hit_disk_ratio60 += stats.request_hit_disk_ratio60; - store_swap_size += stats.store_swap_size; - store_swap_max_size += stats.store_swap_max_size; - store_mem_size += stats.store_mem_size; - store_pages_max += stats.store_pages_max; - store_mem_used += stats.store_mem_used; - objects_size += stats.objects_size; + + store += stats.store; + unlink_requests += stats.unlink_requests; http_requests5 += stats.http_requests5; http_requests60 += stats.http_requests60; cache_misses5 += stats.cache_misses5; cache_misses60 += stats.cache_misses60; cache_hits5 += stats.cache_hits5; cache_hits60 += stats.cache_hits60; near_hits5 += stats.near_hits5; near_hits60 += stats.near_hits60; not_modified_replies5 += stats.not_modified_replies5; not_modified_replies60 += stats.not_modified_replies60; dns_lookups5 += stats.dns_lookups5; dns_lookups60 += stats.dns_lookups60; icp_queries5 += stats.icp_queries5; icp_queries60 += stats.icp_queries60; if (stats.up_time > up_time) up_time = stats.up_time; cpu_time += stats.cpu_time; cpu_usage += stats.cpu_usage; cpu_usage5 += stats.cpu_usage5; @@ -107,45 +104,40 @@ mp_mxfast += stats.mp_mxfast; mp_nlblks += stats.mp_nlblks; mp_grain += stats.mp_grain; mp_uordbytes += stats.mp_uordbytes; mp_allocated += stats.mp_allocated; mp_treeoverhead += stats.mp_treeoverhead; #endif #endif total_accounted += stats.total_accounted; #if !(HAVE_MSTATS && HAVE_GNUMALLOC_H) && HAVE_MALLINFO && HAVE_STRUCT_MALLINFO mem_pool_allocated += stats.mem_pool_allocated; #endif gb_saved_count += stats.gb_saved_count; gb_freed_count += stats.gb_freed_count; max_fd += stats.max_fd; biggest_fd += stats.biggest_fd; number_fd += stats.number_fd; opening_fd += stats.opening_fd; num_fd_free += stats.num_fd_free; reserved_fd += stats.reserved_fd; - store_open_disk_fd += stats.store_open_disk_fd; - store_entries += stats.store_entries; - store_mem_entries += stats.store_mem_entries; - hot_obj_count += stats.hot_obj_count; - n_disk_objects += stats.n_disk_objects; ++count; return *this; } Mgr::InfoAction::Pointer Mgr::InfoAction::Create(const CommandPointer &cmd) { return new InfoAction(cmd); } Mgr::InfoAction::InfoAction(const CommandPointer &cmd): Action(cmd), data() { debugs(16, 5, HERE); } void Mgr::InfoAction::add(const Action& action) { === modified file 'src/mgr/InfoAction.h' --- src/mgr/InfoAction.h 2010-10-28 18:52:59 +0000 +++ src/mgr/InfoAction.h 2011-10-07 22:54:47 +0000 @@ -1,76 +1,74 @@ /* * $Id$ * * DEBUG: section 16 Cache Manager API * */ #ifndef SQUID_MGR_INFO_ACTION_H #define SQUID_MGR_INFO_ACTION_H #include "mgr/Action.h" +#include "StoreStats.h" #include namespace Mgr { /// store general runtime information /// and memory usage class InfoActionData { public: InfoActionData(); InfoActionData& operator += (const InfoActionData& stats); public: struct timeval squid_start; struct timeval current_time; double client_http_clients; double client_http_requests; double icp_pkts_recv; double icp_pkts_sent; double icp_replies_queued; #if USE_HTCP double htcp_pkts_recv; double htcp_pkts_sent; #endif double request_failure_ratio; double avg_client_http_requests; double avg_icp_messages; double select_loops; double avg_loop_time; double request_hit_ratio5; double request_hit_ratio60; double byte_hit_ratio5; double byte_hit_ratio60; double request_hit_mem_ratio5; double request_hit_mem_ratio60; double request_hit_disk_ratio5; double request_hit_disk_ratio60; - double store_swap_size; - double store_swap_max_size; - double store_mem_size; - double store_pages_max; - double store_mem_used; - double objects_size; + + StoreInfoStats store; ///< disk and memory cache statistics + double unlink_requests; double http_requests5; double http_requests60; double cache_misses5; double cache_misses60; double cache_hits5; double cache_hits60; double near_hits5; double near_hits60; double not_modified_replies5; double not_modified_replies60; double dns_lookups5; double dns_lookups60; double icp_queries5; double icp_queries60; double up_time; double cpu_time; double cpu_usage; double cpu_usage5; double cpu_usage60; @@ -96,45 +94,40 @@ double mp_mxfast; double mp_nlblks; double mp_grain; double mp_uordbytes; double mp_allocated; double mp_treeoverhead; #endif #endif double total_accounted; #if !(HAVE_MSTATS && HAVE_GNUMALLOC_H) && HAVE_MALLINFO && HAVE_STRUCT_MALLINFO double mem_pool_allocated; #endif double gb_saved_count; double gb_freed_count; double max_fd; double biggest_fd; double number_fd; double opening_fd; double num_fd_free; double reserved_fd; - double store_open_disk_fd; - double store_entries; - double store_mem_entries; - double hot_obj_count; - double n_disk_objects; unsigned int count; }; /// implement aggregated 'info' action class InfoAction: public Action { protected: InfoAction(const CommandPointer &cmd); public: static Pointer Create(const CommandPointer &cmd); /* Action API */ virtual void add(const Action& action); virtual void respond(const Request& request); virtual void pack(Ipc::TypedMsgHdr& msg) const; virtual void unpack(const Ipc::TypedMsgHdr& msg); protected: /* Action API */ virtual void collect(); === modified file 'src/stat.cc' --- src/stat.cc 2011-09-06 22:32:30 +0000 +++ src/stat.cc 2011-10-12 20:50:19 +0000 @@ -531,50 +531,41 @@ stats.avg_client_http_requests = statCounter.client_http.requests / (runtime / 60.0); stats.avg_icp_messages = (statCounter.icp.pkts_sent + statCounter.icp.pkts_recv) / (runtime / 60.0); stats.select_loops = statCounter.select_loops; stats.avg_loop_time = 1000.0 * runtime / statCounter.select_loops; stats.request_hit_ratio5 = statRequestHitRatio(5); stats.request_hit_ratio60 = statRequestHitRatio(60); stats.byte_hit_ratio5 = statByteHitRatio(5); stats.byte_hit_ratio60 = statByteHitRatio(60); stats.request_hit_mem_ratio5 = statRequestHitMemoryRatio(5); stats.request_hit_mem_ratio60 = statRequestHitMemoryRatio(60); stats.request_hit_disk_ratio5 = statRequestHitDiskRatio(5); stats.request_hit_disk_ratio60 = statRequestHitDiskRatio(60); - stats.store_swap_size = Store::Root().currentSize() / 1024.0; - stats.store_swap_max_size = Store::Root().maxSize(); - - stats.store_mem_size = mem_node::StoreMemSize(); - stats.store_pages_max = store_pages_max; - stats.store_mem_used = mem_node::InUseCount(); - - stats.n_disk_objects = Store::Root().currentCount(); - stats.objects_size = stats.n_disk_objects > 0 ? - stats.store_swap_size / stats.n_disk_objects : 0.0; + Store::Root().getStats(stats.store); stats.unlink_requests = statCounter.unlink.requests; stats.http_requests5 = statPctileSvc(0.5, 5, PCTILE_HTTP); stats.http_requests60 = statPctileSvc(0.5, 60, PCTILE_HTTP); stats.cache_misses5 = statPctileSvc(0.5, 5, PCTILE_MISS); stats.cache_misses60 = statPctileSvc(0.5, 60, PCTILE_MISS); stats.cache_hits5 = statPctileSvc(0.5, 5, PCTILE_HIT); stats.cache_hits60 = statPctileSvc(0.5, 60, PCTILE_HIT); stats.near_hits5 = statPctileSvc(0.5, 5, PCTILE_NH); stats.near_hits60 = statPctileSvc(0.5, 60, PCTILE_NH); stats.not_modified_replies5 = statPctileSvc(0.5, 5, PCTILE_NM); stats.not_modified_replies60 = statPctileSvc(0.5, 60, PCTILE_NM); stats.dns_lookups5 = statPctileSvc(0.5, 5, PCTILE_DNS); stats.dns_lookups60 = statPctileSvc(0.5, 60, PCTILE_DNS); @@ -648,45 +639,40 @@ stats.total_accounted = statMemoryAccounted(); { MemPoolGlobalStats mp_stats; memPoolGetGlobalStats(&mp_stats); #if !(HAVE_MSTATS && HAVE_GNUMALLOC_H) && HAVE_MALLINFO && HAVE_STRUCT_MALLINFO stats.mem_pool_allocated = mp_stats.TheMeter->alloc.level; #endif stats.gb_saved_count = mp_stats.TheMeter->gb_saved.count; stats.gb_freed_count = mp_stats.TheMeter->gb_freed.count; } stats.max_fd = Squid_MaxFD; stats.biggest_fd = Biggest_FD; stats.number_fd = Number_FD; stats.opening_fd = Opening_FD; stats.num_fd_free = fdNFree(); stats.reserved_fd = RESERVED_FD; - stats.store_open_disk_fd = store_open_disk_fd; - - stats.store_entries = StoreEntry::inUseCount(); - stats.store_mem_entries = MemObject::inUseCount(); - stats.hot_obj_count = hot_obj_count; } void DumpInfo(Mgr::InfoActionData& stats, StoreEntry* sentry) { storeAppendPrintf(sentry, "Squid Object Cache: Version %s\n", version_string); #if _SQUID_WINDOWS_ if (WIN32_run_mode == _WIN_SQUID_RUN_MODE_SERVICE) { storeAppendPrintf(sentry,"\nRunning as %s Windows System Service on %s\n", WIN32_Service_name, WIN32_OS_string); storeAppendPrintf(sentry,"Service command line is: %s\n", WIN32_Service_Command_Line); } else storeAppendPrintf(sentry,"Running on %s\n",WIN32_OS_string); #endif storeAppendPrintf(sentry, "Start Time:\t%s\n", mkrfc1123(stats.squid_start.tv_sec)); @@ -735,56 +721,55 @@ storeAppendPrintf(sentry, "Cache information for %s:\n",APP_SHORTNAME); storeAppendPrintf(sentry, "\tHits as %% of all requests:\t5min: %3.1f%%, 60min: %3.1f%%\n", stats.request_hit_ratio5 / fct, stats.request_hit_ratio60 / fct); storeAppendPrintf(sentry, "\tHits as %% of bytes sent:\t5min: %3.1f%%, 60min: %3.1f%%\n", stats.byte_hit_ratio5 / fct, stats.byte_hit_ratio60 / fct); storeAppendPrintf(sentry, "\tMemory hits as %% of hit requests:\t5min: %3.1f%%, 60min: %3.1f%%\n", stats.request_hit_mem_ratio5 / fct, stats.request_hit_mem_ratio60 / fct); storeAppendPrintf(sentry, "\tDisk hits as %% of hit requests:\t5min: %3.1f%%, 60min: %3.1f%%\n", stats.request_hit_disk_ratio5 / fct, stats.request_hit_disk_ratio60 / fct); storeAppendPrintf(sentry, "\tStorage Swap size:\t%.0f KB\n", - stats.store_swap_size); + stats.store.swap.size / 1024); storeAppendPrintf(sentry, "\tStorage Swap capacity:\t%4.1f%% used, %4.1f%% free\n", - Math::doublePercent(stats.store_swap_size, stats.store_swap_max_size), - Math::doublePercent(stats.store_swap_max_size - stats.store_swap_size, stats.store_swap_max_size)); + Math::doublePercent(stats.store.swap.size, stats.store.swap.capacity), + Math::doublePercent(stats.store.swap.available(), stats.store.swap.capacity)); storeAppendPrintf(sentry, "\tStorage Mem size:\t%.0f KB\n", - stats.store_mem_size / 1024); + stats.store.mem.size / 1024); - const double mFree = max(0.0, stats.store_pages_max-stats.store_mem_used); storeAppendPrintf(sentry, "\tStorage Mem capacity:\t%4.1f%% used, %4.1f%% free\n", - Math::doublePercent(stats.store_mem_used, stats.store_pages_max), - Math::doublePercent(mFree, stats.store_pages_max)); + Math::doublePercent(stats.store.mem.size, stats.store.mem.capacity), + Math::doublePercent(stats.store.mem.available(), stats.store.mem.capacity)); storeAppendPrintf(sentry, "\tMean Object Size:\t%0.2f KB\n", - stats.objects_size / fct); + stats.store.swap.meanObjectSize() / 1024); storeAppendPrintf(sentry, "\tRequests given to unlinkd:\t%.0f\n", stats.unlink_requests); storeAppendPrintf(sentry, "Median Service Times (seconds) 5 min 60 min:\n"); fct = stats.count > 1 ? stats.count * 1000.0 : 1000.0; storeAppendPrintf(sentry, "\tHTTP Requests (All): %8.5f %8.5f\n", stats.http_requests5 / fct, stats.http_requests60 / fct); storeAppendPrintf(sentry, "\tCache Misses: %8.5f %8.5f\n", stats.cache_misses5 / fct, stats.cache_misses60 / fct); storeAppendPrintf(sentry, "\tCache Hits: %8.5f %8.5f\n", stats.cache_hits5 / fct, stats.cache_hits60 / fct); storeAppendPrintf(sentry, "\tNear Hits: %8.5f %8.5f\n", @@ -932,51 +917,51 @@ storeAppendPrintf(sentry, "\tmemPoolAlloc calls: %9.0f\n", stats.gb_saved_count); storeAppendPrintf(sentry, "\tmemPoolFree calls: %9.0f\n", stats.gb_freed_count); } storeAppendPrintf(sentry, "File descriptor usage for %s:\n", APP_SHORTNAME); storeAppendPrintf(sentry, "\tMaximum number of file descriptors: %4.0f\n", stats.max_fd); storeAppendPrintf(sentry, "\tLargest file desc currently in use: %4.0f\n", stats.biggest_fd); storeAppendPrintf(sentry, "\tNumber of file desc currently in use: %4.0f\n", stats.number_fd); storeAppendPrintf(sentry, "\tFiles queued for open: %4.0f\n", stats.opening_fd); storeAppendPrintf(sentry, "\tAvailable number of file descriptors: %4.0f\n", stats.num_fd_free); storeAppendPrintf(sentry, "\tReserved number of file descriptors: %4.0f\n", stats.reserved_fd); storeAppendPrintf(sentry, "\tStore Disk files open: %4.0f\n", - stats.store_open_disk_fd); + stats.store.open_disk_fd); storeAppendPrintf(sentry, "Internal Data Structures:\n"); storeAppendPrintf(sentry, "\t%6.0f StoreEntries\n", - stats.store_entries); + stats.store.store_entry_count); storeAppendPrintf(sentry, "\t%6.0f StoreEntries with MemObjects\n", - stats.store_mem_entries); + stats.store.mem_object_count); storeAppendPrintf(sentry, "\t%6.0f Hot Object Cache Items\n", - stats.hot_obj_count); + stats.store.mem.count); storeAppendPrintf(sentry, "\t%6.0f on-disk objects\n", - stats.n_disk_objects); + stats.store.swap.count); } void DumpMallocStatistics(StoreEntry* sentry) { #if XMALLOC_STATISTICS xm_deltat = current_dtime - xm_time; xm_time = current_dtime; storeAppendPrintf(sentry, "\nMemory allocation statistics\n"); storeAppendPrintf(sentry, "%12s %15s %6s %12s\n","Alloc Size","Count","Delta","Alloc/sec"); malloc_statistics(info_get_mallstat, sentry); #endif } void GetServiceTimesStats(Mgr::ServiceTimesActionData& stats) { for (int i = 0; i < Mgr::ServiceTimesActionData::seriesSize; ++i) { double p = (i + 1) * 5 / 100.0; === modified file 'src/store_dir.cc' --- src/store_dir.cc 2011-09-06 22:32:30 +0000 +++ src/store_dir.cc 2011-10-12 21:05:08 +0000 @@ -310,40 +310,60 @@ assert(e->swap_filen >= 0); /* * icons and such; don't write them to the swap log */ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) return; assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX); debugs(20, 3, "storeDirSwapLog: " << swap_log_op_str[op] << " " << e->getMD5Text() << " " << e->swap_dirn << " " << std::hex << std::uppercase << std::setfill('0') << std::setw(8) << e->swap_filen); dynamic_cast(INDEXSD(e->swap_dirn))->logEntry(*e, op); } void +StoreController::getStats(StoreInfoStats &stats) const +{ + if (memStore) + memStore->getStats(stats); + else { + /* move this code to the future exclusive memory cache class */ + stats.mem.shared = false; + stats.mem.capacity = Config.memMaxSize; + stats.mem.size = mem_node::StoreMemSize(); + stats.mem.count = hot_obj_count; + } + + swapDir->getStats(stats); + + // low-level info not specific to memory or disk cache + stats.store_entry_count = StoreEntry::inUseCount(); + stats.mem_object_count = MemObject::inUseCount(); +} + +void StoreController::stat(StoreEntry &output) const { storeAppendPrintf(&output, "Store Directory Statistics:\n"); storeAppendPrintf(&output, "Store Entries : %lu\n", (unsigned long int)StoreEntry::inUseCount()); storeAppendPrintf(&output, "Maximum Swap Size : %"PRIu64" KB\n", maxSize() >> 10); storeAppendPrintf(&output, "Current Store Swap Size: %.2f KB\n", currentSize() / 1024.0); storeAppendPrintf(&output, "Current Capacity : %.2f%% used, %.2f%% free\n", Math::doublePercent(currentSize(), maxSize()), Math::doublePercent((maxSize() - currentSize()), maxSize())); if (memStore) memStore->stat(output); /* now the swapDir */ swapDir->stat(output); } @@ -949,40 +969,56 @@ result += store(i)->currentCount(); } return result; } int64_t StoreHashIndex::maxObjectSize() const { int64_t result = -1; for (int i = 0; i < Config.cacheSwap.n_configured; i++) { if (dir(i).active() && store(i)->maxObjectSize() > result) result = store(i)->maxObjectSize(); } return result; } void +StoreHashIndex::getStats(StoreInfoStats &stats) const +{ + // accumulate per-disk cache stats + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + StoreInfoStats dirStats; + store(i)->getStats(dirStats); + stats += dirStats; + } + + // common to all disks + stats.open_disk_fd = store_open_disk_fd; + + // memory cache stats are collected in StoreController::getStats(), for now +} + +void StoreHashIndex::stat(StoreEntry & output) const { int i; /* Now go through each store, calling its stat routine */ for (i = 0; i < Config.cacheSwap.n_configured; i++) { storeAppendPrintf(&output, "\n"); store(i)->stat(output); } } void StoreHashIndex::reference(StoreEntry &e) { e.store()->reference(e); } bool StoreHashIndex::dereference(StoreEntry &e) === modified file 'src/tests/stub_MemObject.cc' --- src/tests/stub_MemObject.cc 2011-08-10 15:54:51 +0000 +++ src/tests/stub_MemObject.cc 2011-10-07 22:54:47 +0000 @@ -182,20 +182,27 @@ } int64_t MemObject::expectedReplySize() const { fatal ("MemObject.cc required."); return 0; } void MemObject::resetUrls(char const*, char const*) { fatal ("MemObject.cc required."); } void MemObject::markEndOfReplyHeaders() { fatal ("MemObject.cc required."); } + +size_t +MemObject::inUseCount() +{ + fatal ("MemObject.cc required."); + return 0; +} === modified file 'src/tests/stub_MemStore.cc' --- src/tests/stub_MemStore.cc 2011-09-12 00:29:56 +0000 +++ src/tests/stub_MemStore.cc 2011-10-07 22:54:47 +0000 @@ -2,30 +2,31 @@ * $Id$ * * DEBUG: section 84 Helper process maintenance * */ #include "config.h" #include "MemStore.h" #define STUB_API "MemStore.cc" #include "tests/STUB.h" MemStore::MemStore() STUB MemStore::~MemStore() STUB void MemStore::considerKeeping(StoreEntry &) STUB void MemStore::reference(StoreEntry &) STUB void MemStore::maintain() STUB void MemStore::cleanReadable(const sfileno) STUB void MemStore::get(String const, STOREGETCLIENT, void *) STUB void MemStore::init() STUB +void MemStore::getStats(StoreInfoStats&) const STUB void MemStore::stat(StoreEntry &) const STUB int MemStore::callback() STUB_RETVAL(0) StoreEntry *MemStore::get(const cache_key *) STUB_RETVAL(NULL) uint64_t MemStore::maxSize() const STUB_RETVAL(0) uint64_t MemStore::minSize() const STUB_RETVAL(0) uint64_t MemStore::currentSize() const STUB_RETVAL(0) uint64_t MemStore::currentCount() const STUB_RETVAL(0) int64_t MemStore::maxObjectSize() const STUB_RETVAL(0) StoreSearch *MemStore::search(String const, HttpRequest *) STUB_RETVAL(NULL) bool MemStore::dereference(StoreEntry &) STUB_RETVAL(false) === added file 'src/tests/stub_store_stats.cc' --- src/tests/stub_store_stats.cc 1970-01-01 00:00:00 +0000 +++ src/tests/stub_store_stats.cc 2011-10-05 22:36:11 +0000 @@ -0,0 +1,19 @@ +#include "squid.h" + +#define STUB_API "StoreStats.cc" +#include "tests/STUB.h" + +#include "StoreStats.h" +#include + +StoreInfoStats::StoreInfoStats() STUB + +StoreInfoStats & +StoreInfoStats::operator +=(const StoreInfoStats &stats) STUB_RETVAL(*this) + +StoreIoStats::StoreIoStats() +{ + // we have to implement this one because tests/stub_store.cc + // has a StoreIoStats global + memset(this, 0, sizeof(*this)); +} === modified file 'src/tests/testStore.cc' --- src/tests/testStore.cc 2011-08-17 23:59:03 +0000 +++ src/tests/testStore.cc 2011-10-07 22:54:48 +0000 @@ -41,40 +41,45 @@ uint64_t TestStore::currentSize() const { return 2; } uint64_t TestStore::currentCount() const { return 2; } int64_t TestStore::maxObjectSize() const { return 1; } void +TestStore::getStats(StoreInfoStats &) const +{ +} + +void TestStore::stat(StoreEntry &) const { const_cast(this)->statsCalled = true; } StoreSearch * TestStore::search(String const url, HttpRequest *) { return NULL; } void testStore::testSetRoot() { StorePointer aStore(new TestStore); Store::Root(aStore); CPPUNIT_ASSERT(&Store::Root() == aStore.getRaw()); Store::Root(NULL); } === modified file 'src/tests/testStore.h' --- src/tests/testStore.h 2011-06-21 23:44:03 +0000 +++ src/tests/testStore.h 2011-10-07 22:54:48 +0000 @@ -48,34 +48,36 @@ virtual StoreEntry* get (const cache_key*); virtual void get (String, void (*)(StoreEntry*, void*), void*); virtual void init(); virtual void maintain() {}; virtual uint64_t maxSize() const; virtual uint64_t minSize() const; virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual int64_t maxObjectSize() const; + virtual void getStats(StoreInfoStats &) const; + virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */ virtual void reference(StoreEntry &) {} /* Reference this object */ virtual bool dereference(StoreEntry &) { return true; } virtual StoreSearch *search(String const url, HttpRequest *); }; typedef RefCount TestStorePointer; #endif