XrdPfc Namespace Reference

Classes
class	Block
class	BlockResponseHandler
class	Cache
	Attaches/creates and detaches/deletes cache-io objects for disk based cache. More...
struct	ChunkRequest
struct	Configuration
	Contains parameters configurable from the xrootd config file. More...
struct	DataFsPurgeshot
struct	DataFsSnapshot
struct	DataFsState
struct	DataFsStateBase
class	Decision
	Base class for selecting which files should be cached. More...
class	DirectResponseHandler
struct	DirPurgeElement
struct	DirState
struct	DirStateBase
struct	DirStateElement
class	DirStats
struct	DirUsage
class	File
class	FPurgeState
class	FsTraversal
class	Info
	Status of cached file. Can be read from and written into a binary file. More...
class	IO
	Base cache-io class that implements some XrdOucCacheIO abstract methods. More...
class	IOFile
	Downloads original file into a single file on local disk. Handles read requests as they come along. More...
class	IOFileBlock
	Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along. More...
struct	MutexHolder
struct	PathTokenizer
class	Print
class	PurgePin
	Base class for reguesting directory space to obtain. More...
struct	ReadReqRH
struct	ReadRequest
class	ResourceMonitor
struct	SplitParser
class	Stats
	Statistics of cache utilisation by a File object. More...
struct	TmpConfiguration

Typedefs
using	BlockList_i = std::list< Block * >::iterator
using	BlockList_t = std::list< Block * >
using	unlink_func = std::function< int(const std::string &)>
using	vChunkRequest_i = std::vector< ChunkRequest >::iterator
using	vChunkRequest_t = std::vector< ChunkRequest >
typedef std::vector< uint32_t >	vCkSum_t

Enumerations
enum	CkSumCheck_e {   CSChk_Unknown = -1 ,   CSChk_None = 0 ,   CSChk_Cache = 1 ,   CSChk_Net = 2 ,   CSChk_Both = 3 ,   CSChk_TLS = 4 }

Functions
void	OldStylePurgeDriver (DataFsPurgeshot &ps)
m_NDirectories	PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStateElement, m_dir_name, m_stats, m_usage, m_parent, m_daughters_begin, m_daughters_end) PFC_DEFINE_TYPE_NON_INTRUSIVE(DataFsSnapshot
	PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStats, m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_StBlocksAdded, m_NCksumErrors, m_StBlocksRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) PFC_DEFINE_TYPE_NON_INTRUSIVE(DirUsage
long long	UnlinkPurgeStateFilesInMap (FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)

Variables
m_NDirectories	m_disk_total
m_NDirectories	m_disk_used
m_NDirectories	m_file_usage
	m_LastCloseTime
	m_LastOpenTime
m_NDirectories	m_meta_total
m_NDirectories	m_meta_used
	m_NFiles
	m_NFilesOpen
m_NDirectories	m_sshot_stats_reset_time
	m_StBlocks
m_NDirectories	m_usage_update_time
const char *	trace_what_strings [] = {"","error ","warning ","info ","debug ","dump "}

Typedef Documentation

BlockList_i

using XrdPfc::BlockList_i = typedef std::list<Block*>::iterator

Definition at line 167 of file XrdPfcFile.hh.

BlockList_t

using XrdPfc::BlockList_t = typedef std::list<Block*>

Definition at line 166 of file XrdPfcFile.hh.

unlink_func

using XrdPfc::unlink_func = typedef std::function<int(const std::string&)>

Definition at line 38 of file XrdPfcDirState.hh.

vChunkRequest_i

using XrdPfc::vChunkRequest_i = typedef std::vector<ChunkRequest>::iterator

Definition at line 102 of file XrdPfcFile.hh.

vChunkRequest_t

using XrdPfc::vChunkRequest_t = typedef std::vector<ChunkRequest>

Definition at line 101 of file XrdPfcFile.hh.

vCkSum_t

typedef std::vector<uint32_t> XrdPfc::vCkSum_t

Definition at line 31 of file XrdPfcTypes.hh.

Enumeration Type Documentation

CkSumCheck_e

enum XrdPfc::CkSumCheck_e

Enumerator
CSChk_Unknown
CSChk_None
CSChk_Cache
CSChk_Net
CSChk_Both
CSChk_TLS

Definition at line 27 of file XrdPfcTypes.hh.

                  { CSChk_Unknown = -1, CSChk_None = 0, CSChk_Cache = 1, CSChk_Net = 2, CSChk_Both = 3,
                    CSChk_TLS = 4 // Only used during configuration parsing.
};

Function Documentation

OldStylePurgeDriver()

void XrdPfc::OldStylePurgeDriver

(

DataFsPurgeshot &

ps

)

PurgePin

Default purge

Definition at line 99 of file XrdPfcPurge.cc.

{
   static const char *trc_pfx = "OldStylePurgeDriver ";
   const auto &cache = Cache::TheOne();
   const auto &conf  = Cache::Conf();
   auto &oss = *cache.GetOss();
 
   time_t purge_start = time(0);
   
   PurgePin *purge_pin = cache.GetPurgePin();
   long long std_blocks_removed_by_pin = 0;
   if (purge_pin)
   {   
      // set dir stat for each path and calculate nBytes to recover for each path
      // return total bytes to recover within the plugin
      long long clearVal = purge_pin->GetBytesToRecover(ps);
      if (clearVal)
      {
         TRACE(Debug, "PurgePin remove total " << clearVal << " bytes");
         PurgePin::list_t &dpl = purge_pin->refDirInfos();
         // iterate through the plugin paths
         for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit)
         {
            TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes");
 
            FPurgeState fps(ppit->nBytesToRecover, oss);
            bool scan_ok = fps.TraverseNamespace(ppit->path.c_str());
            if ( ! scan_ok) {
               TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path);
               continue;
            } 
            
            fps.MoveListEntriesToMap();
            std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path);
         }
      }
   }
 
 
   // check if the default pargue is still needed after purge pin
   long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll;
   long long default_purge_blocks_removed = 0;
   if (ps.m_bytes_to_remove > pin_removed_bytes)
   {
      // init default purge
      long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes;
      FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required
 
      if (ps.m_age_based_purge)
      {
         purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge);
      }
      if (conf.is_uvkeep_purge_in_effect())
      {
         purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep);
      }
 
      // Make a map of file paths, sorted by access time.
      bool scan_ok = purgeState.TraverseNamespace("/");
      if (!scan_ok)
      {
         TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen.");
         return;
      }
 
      TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes.");
 
      purgeState.MoveListEntriesToMap();
      default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/");
   }
 
   // print the total summary
   int purge_duration = time(0) - purge_start;
   long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll;
   TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration);
}

References XrdPfc::Cache::Conf(), Debug, Error, XrdPfc::PurgePin::GetBytesToRecover(), XrdPfc::FPurgeState::getNBytesTotal(), XrdPfc::DataFsPurgeshot::m_age_based_purge, XrdPfc::DataFsPurgeshot::m_bytes_to_remove, XrdPfc::FPurgeState::MoveListEntriesToMap(), XrdPfc::PurgePin::refDirInfos(), XrdPfc::FPurgeState::setMinTime(), XrdPfc::FPurgeState::setUVKeepMinTime(), XrdPfc::Cache::TheOne(), TRACE, XrdPfc::FPurgeState::TraverseNamespace(), UnlinkPurgeStateFilesInMap(), and TPC::Warning.

Referenced by XrdPfc::ResourceMonitor::perform_purge_task().

Here is the call graph for this function:

Here is the caller graph for this function:

PFC_DEFINE_TYPE_NON_INTRUSIVE() [1/2]

m_NDirectories XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE	(	DirStateElement	,
		m_dir_name	,
		m_stats	,
		m_usage	,
		m_parent	,
		m_daughters_begin	,
		m_daughters_end
	)

PFC_DEFINE_TYPE_NON_INTRUSIVE() [2/2]

XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE	(	DirStats	,
		m_NumIos	,
		m_Duration	,
		m_BytesHit	,
		m_BytesMissed	,
		m_BytesBypassed	,
		m_BytesWritten	,
		m_StBlocksAdded	,
		m_NCksumErrors	,
		m_StBlocksRemoved	,
		m_NFilesOpened	,
		m_NFilesClosed	,
		m_NFilesCreated	,
		m_NFilesRemoved	,
		m_NDirectoriesCreated	,
		m_NDirectoriesRemoved
	)

UnlinkPurgeStateFilesInMap()

long long XrdPfc::UnlinkPurgeStateFilesInMap	(	FPurgeState &	purgeState,
		long long	bytes_to_remove,
		const std::string &	root_path
	)

Definition at line 24 of file XrdPfcPurge.cc.

{
   static const char *trc_pfx = "UnlinkPurgeStateFilesInMap ";
 
   struct stat fstat;
   int         protected_cnt = 0;
   int         deleted_file_count = 0;
   long long   deleted_st_blocks = 0;
   long long   protected_st_blocks = 0;
   long long   st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll;
   
 
   const auto &cache = Cache::TheOne();
   auto &resmon = Cache::ResMon();
   auto &oss = *cache.GetOss();
 
   TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove);
 
   // Loop over map and remove files with oldest values of access time.
   for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it)
   {
      // Finish when enough space has been freed but not while age-based purging is in progress.
      // Those files are marked with time-stamp = 0.
      if (st_blocks_to_remove <= 0 && it->first != 0)
      {
         break;
      }
 
      std::string &infoPath = it->second.path;
      std::string  dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen);
 
      if (cache.IsFileActiveOrPurgeProtected(dataPath))
      {
         ++protected_cnt;
         protected_st_blocks += it->second.nStBlocks;
         TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks);
         continue;
      }
 
      // remove info file
      if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK)
      {
         oss.Unlink(infoPath.c_str());
         TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size);
      }
      else
      {
         TRACE(Error, trc_pfx << "Can't locate file " << dataPath);
      }
 
      // remove data file
      if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK)
      {
         st_blocks_to_remove -= it->second.nStBlocks;
         deleted_st_blocks   += it->second.nStBlocks;
         ++deleted_file_count;
 
         oss.Unlink(dataPath.c_str());
         TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first);
 
         resmon.register_file_purge(dataPath, it->second.nStBlocks);
      }
   }
   if (protected_cnt > 0)
   {
      TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks);
   }
 
   TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks)
 
   return deleted_st_blocks;
}

References Debug, Error, fstat, XrdPfc::FPurgeState::refMap(), XrdPfc::Cache::ResMon(), XrdPfc::Info::s_infoExtensionLen, stat, XrdPfc::Cache::TheOne(), TRACE, and XrdOssOK.

Referenced by OldStylePurgeDriver().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

m_disk_total

m_NDirectories XrdPfc::m_disk_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_disk_used

m_NDirectories XrdPfc::m_disk_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_file_usage

m_NDirectories XrdPfc::m_file_usage

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_LastCloseTime

XrdPfc::m_LastCloseTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

m_LastOpenTime

XrdPfc::m_LastOpenTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

m_meta_total

m_NDirectories XrdPfc::m_meta_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_meta_used

m_NDirectories XrdPfc::m_meta_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_NFiles

XrdPfc::m_NFiles

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

m_NFilesOpen

XrdPfc::m_NFilesOpen

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

m_sshot_stats_reset_time

m_NDirectories XrdPfc::m_sshot_stats_reset_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

m_StBlocks

XrdPfc::m_StBlocks

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

m_usage_update_time

m_NDirectories XrdPfc::m_usage_update_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

trace_what_strings

const char * XrdPfc::trace_what_strings = {"","error ","warning ","info ","debug ","dump "}

Definition at line 26 of file XrdPfcConfiguration.cc.