framework/frog/FROG.cc

0001 #include "FROG.h"
0002
0003 #include <sphenixodbc/ODBCInterface.h>
0004
0005 #include <phool/phool.h>
0006
0007 #include <odbc++/resultset.h>
0008 #include <odbc++/statement.h>  // for Statement
0009
0010 #include <boost/tokenizer.hpp>
0011
0012 #include <fstream>
0013 #include <iostream>
0014 #include <memory>
0015 #include <string>
0016
0017 const char *
0018 FROG::location(const std::string &logical_name)
0019 {
0020   pfn = logical_name;
0021   if (logical_name.empty() || logical_name.find('/') != std::string::npos)
0022   {
0023     if (Verbosity() > 0)
0024     {
0025       if (logical_name.empty())
0026       {
0027         std::cout << "FROG: empty string as filename" << std::endl;
0028       }
0029       else if (logical_name.find('/') != std::string::npos)
0030       {
0031         std::cout << "FROG: found / in filename, assuming it contains a full path" << std::endl;
0032       }
0033     }
0034     return pfn.c_str();
0035   }
0036   try
0037   {
0038     char *gsearchpath_env = getenv("GSEARCHPATH");
0039     if (gsearchpath_env == nullptr)
0040     {
0041       return pfn.c_str();
0042     }
0043     std::string gsearchpath(gsearchpath_env);
0044     if (Verbosity() > 0)
0045     {
0046       std::cout << "FROG: GSEARCHPATH: " << gsearchpath << std::endl;
0047     }
0048     boost::char_separator<char> sep(":");
0049     boost::tokenizer<boost::char_separator<char> > tok(gsearchpath, sep);
0050     for (const auto &iter : tok)
0051     {
0052       if (iter == "PG")
0053       {
0054         if (Verbosity() > 1)
0055         {
0056           std::cout << "Searching FileCatalog for disk resident file "
0057                     << logical_name << std::endl;
0058         }
0059         if (PGSearch(logical_name))
0060         {
0061           if (Verbosity() > 1)
0062           {
0063             std::cout << "Found " << logical_name << " in FileCatalog, returning "
0064                       << pfn << std::endl;
0065           }
0066           break;
0067         }
0068       }
0069       else if (iter == "DCACHE")
0070       {
0071         if (Verbosity() > 1)
0072         {
0073           std::cout << "Searching FileCatalog for dCache file "
0074                     << logical_name << std::endl;
0075         }
0076         if (dCacheSearch(logical_name))
0077         {
0078           if (Verbosity() > 1)
0079           {
0080             std::cout << "Found " << logical_name << " in dCache, returning "
0081                       << pfn << std::endl;
0082           }
0083           break;
0084         }
0085       }
0086       else if (iter == "XROOTD")
0087       {
0088         if (Verbosity() > 1)
0089         {
0090           std::cout << "Searching FileCatalog for XRootD file "
0091                     << logical_name << std::endl;
0092         }
0093         if (XRootDSearch(logical_name))
0094         {
0095           if (Verbosity() > 1)
0096           {
0097             std::cout << "Found " << logical_name << " in XRootD, returning "
0098                       << pfn << std::endl;
0099           }
0100           break;
0101         }
0102       }
0103       else if (iter == "LUSTRE")
0104       {
0105         if (Verbosity() > 1)
0106         {
0107           std::cout << "Searching FileCatalog for Lustre file "
0108                     << logical_name << std::endl;
0109         }
0110         if (LustreSearch(logical_name))
0111         {
0112           if (Verbosity() > 1)
0113           {
0114             std::cout << "Found " << logical_name << " in Lustre, returning "
0115                       << pfn << std::endl;
0116           }
0117           break;
0118         }
0119       }
0120       else if (iter == "RAWDATA")
0121       {
0122         if (Verbosity() > 1)
0123         {
0124           std::cout << "Searching FileCatalog for Raw Data file "
0125                     << logical_name << std::endl;
0126         }
0127         if (RawDataSearch(logical_name))
0128         {
0129           if (Verbosity() > 1)
0130           {
0131             std::cout << "Found raw data file " << logical_name << " in Lustre, returning "
0132                       << pfn << std::endl;
0133           }
0134           break;
0135         }
0136       }
0137       else if (iter == "HPSSRAW")
0138       {
0139         if (Verbosity() > 1)
0140         {
0141           std::cout << "Searching FileCatalog for Hpss Raw Data File "
0142                     << logical_name << std::endl;
0143         }
0144         if (HpssRawDataSearch(logical_name))
0145         {
0146           if (Verbosity() > 1)
0147           {
0148             std::cout << "Found raw data file " << logical_name << " in Hpss, returning "
0149                       << pfn << std::endl;
0150           }
0151           break;
0152         }
0153       }
0154       else if (iter == "MINIO")
0155       {
0156         if (Verbosity() > 1)
0157         {
0158           std::cout << "Searching FileCatalog for Lustre file via MinIO "
0159                     << logical_name << std::endl;
0160         }
0161         if (MinIOSearch(logical_name))
0162         {
0163           if (Verbosity() > 1)
0164           {
0165             std::cout << "Found " << logical_name << " in Lustre, returning MinIO URL "
0166                       << pfn << std::endl;
0167           }
0168           break;
0169         }
0170       }
0171       else  // assuming this is a file path
0172       {
0173         if (Verbosity() > 0)
0174         {
0175           std::cout << "Trying path " << iter << std::endl;
0176         }
0177         std::string fullfile(iter);
0178         fullfile.append("/").append(logical_name);
0179         if (localSearch(fullfile))
0180         {
0181           break;
0182         }
0183       }
0184     }
0185   }
0186   catch (...)
0187   {
0188     if (Verbosity() > 0)
0189     {
0190       std::cout << "FROG: GSEARCHPATH not set " << std::endl;
0191     }
0192   }
0193   if (m_DisconnectFlag)
0194   {
0195     Disconnect();
0196   }
0197   return pfn.c_str();
0198 }
0199
0200 bool FROG::localSearch(const std::string &logical_name)
0201 {
0202   if (std::ifstream(logical_name))
0203   {
0204     pfn = logical_name;
0205     return true;
0206   }
0207   return false;
0208 }
0209
0210 void FROG::Disconnect()
0211 {
0212   ODBCInterface::instance()->Disconnect();
0213 }
0214
0215 bool FROG::PGSearch(const std::string &lname)
0216 {
0217   bool bret = false;
0218   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name <> 'hpss' and full_host_name <> 'dcache' and full_host_name <> 'lustre'";
0219
0220   if (Verbosity() > 1)
0221   {
0222     std::cout << "sql query:" << std::endl
0223               << sqlquery << std::endl;
0224   }
0225   odbc::Statement *statement = ODBCInterface::instance()->getStatement("FileCatalog_read");
0226   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0227
0228   if (resultSet && resultSet->next())
0229   {
0230     pfn = resultSet->getString(1);
0231     bret = true;
0232   }
0233   return bret;
0234 }
0235
0236 bool FROG::dCacheSearch(const std::string &lname)
0237 {
0238   bool bret = false;
0239   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'dcache'";
0240
0241   if (Verbosity() > 1)
0242   {
0243     std::cout << "sql query:" << std::endl
0244               << sqlquery << std::endl;
0245   }
0246   odbc::Statement *statement = ODBCInterface::instance()->getStatement("FileCatalog_read");
0247   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0248
0249   if (resultSet && resultSet->next())
0250   {
0251     std::string dcachefile = resultSet->getString(1);
0252     if (std::ifstream(dcachefile))
0253     {
0254       pfn = "dcache:" + dcachefile;
0255       bret = true;
0256     }
0257   }
0258   return bret;
0259 }
0260
0261 bool FROG::XRootDSearch(const std::string &lname)
0262 {
0263   bool bret = false;
0264   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0265   if (Verbosity() > 1)
0266   {
0267     std::cout << "sql query:" << std::endl
0268               << sqlquery << std::endl;
0269   }
0270   odbc::Statement *statement = ODBCInterface::instance()->getStatement("FileCatalog_read");
0271   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0272
0273   if (resultSet && resultSet->next())
0274   {
0275     std::string xrootdfile = resultSet->getString(1);
0276     pfn = "root://xrdsphenix.rcf.bnl.gov/" + xrootdfile;
0277     bret = true;
0278   }
0279   return bret;
0280 }
0281
0282 bool FROG::LustreSearch(const std::string &lname)
0283 {
0284   bool bret = false;
0285   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0286
0287   odbc::Statement *statement = ODBCInterface::instance()->getStatement("FileCatalog_read");
0288   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0289
0290   if (resultSet && resultSet->next())
0291   {
0292     pfn = resultSet->getString(1);
0293     bret = true;
0294   }
0295   return bret;
0296 }
0297
0298 bool FROG::MinIOSearch(const std::string &lname)
0299 {
0300   bool bret = false;
0301   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0302
0303   if (Verbosity() > 1)
0304   {
0305     std::cout << "sql query:" << std::endl
0306               << sqlquery << std::endl;
0307   }
0308   odbc::Statement *statement = ODBCInterface::instance()->getStatement("FileCatalog_read");
0309   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0310
0311   if (resultSet && resultSet->next())
0312   {
0313     pfn = resultSet->getString(1);
0314     std::string toreplace("/sphenix/lustre01/sphnxpro");
0315     size_t strpos = pfn.find(toreplace);
0316     if (strpos == std::string::npos)
0317     {
0318       std::cout << " could not locate " << toreplace
0319                 << " in full file path " << pfn << std::endl;
0320       exit(1);
0321     }
0322     else if (strpos > 0)
0323     {
0324       std::cout << "full file path " << pfn
0325                 << "does not start with " << toreplace << std::endl;
0326       exit(1);
0327     }
0328     pfn.replace(pfn.begin(), pfn.begin() + toreplace.size(), "s3://sphenixs3.rcf.bnl.gov:9000");
0329     bret = true;
0330   }
0331   return bret;
0332 }
0333
0334 bool FROG::RawDataSearch(const std::string &lname)
0335 {
0336   bool bret = false;
0337   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0338
0339   odbc::Statement *statement = ODBCInterface::instance()->getStatement("RawdataCatalog_read");
0340   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0341
0342   if (resultSet && resultSet->next())
0343   {
0344     pfn = resultSet->getString(1);
0345     bret = true;
0346   }
0347   return bret;
0348 }
0349
0350 bool FROG::HpssRawDataSearch(const std::string &lname)
0351 {
0352   bool bret = false;
0353   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'hpss'";
0354
0355   odbc::Statement *statement = ODBCInterface::instance()->getStatement("RawdataCatalog_read");
0356   std::unique_ptr<odbc::ResultSet> resultSet(statement->executeQuery(sqlquery));
0357
0358   if (resultSet && resultSet->next())
0359   {
0360     pfn = resultSet->getString(1);
0361     bret = true;
0362   }
0363   return bret;
0364 }