framework/frog/FROG.cc

0001 #include "FROG.h"
0002
0003 #include <phool/phool.h>
0004
0005 #include <odbc++/connection.h>
0006 #include <odbc++/drivermanager.h>
0007 #include <odbc++/resultset.h>
0008 #include <odbc++/statement.h>  // for Statement
0009 #include <odbc++/types.h>      // for SQLException
0010
0011 #include <boost/tokenizer.hpp>
0012
0013 #include <chrono>
0014 #include <cstdlib>
0015 #include <fstream>
0016 #include <iostream>
0017 #include <random>  // For retrying connections
0018 #include <string>
0019 #include <thread>
0020
0021 const char *
0022 FROG::location(const std::string &logical_name)
0023 {
0024   pfn = logical_name;
0025   if (logical_name.empty() || logical_name.find('/') != std::string::npos)
0026   {
0027     if (Verbosity() > 0)
0028     {
0029       if (logical_name.empty())
0030       {
0031         std::cout << "FROG: empty string as filename" << std::endl;
0032       }
0033       else if (logical_name.find('/') != std::string::npos)
0034       {
0035         std::cout << "FROG: found / in filename, assuming it contains a full path" << std::endl;
0036       }
0037     }
0038     return pfn.c_str();
0039   }
0040   try
0041   {
0042     char *gsearchpath_env = getenv("GSEARCHPATH");
0043     if (gsearchpath_env == nullptr)
0044     {
0045       return pfn.c_str();
0046     }
0047     std::string gsearchpath(gsearchpath_env);
0048     if (Verbosity() > 0)
0049     {
0050       std::cout << "FROG: GSEARCHPATH: " << gsearchpath << std::endl;
0051     }
0052     boost::char_separator<char> sep(":");
0053     boost::tokenizer<boost::char_separator<char> > tok(gsearchpath, sep);
0054     for (const auto &iter : tok)
0055     {
0056       if (iter == "PG")
0057       {
0058         if (Verbosity() > 1)
0059         {
0060           std::cout << "Searching FileCatalog for disk resident file "
0061                     << logical_name << std::endl;
0062         }
0063         if (PGSearch(logical_name))
0064         {
0065           if (Verbosity() > 1)
0066           {
0067             std::cout << "Found " << logical_name << " in FileCatalog, returning "
0068                       << pfn << std::endl;
0069           }
0070           break;
0071         }
0072       }
0073       else if (iter == "DCACHE")
0074       {
0075         if (Verbosity() > 1)
0076         {
0077           std::cout << "Searching FileCatalog for dCache file "
0078                     << logical_name << std::endl;
0079         }
0080         if (dCacheSearch(logical_name))
0081         {
0082           if (Verbosity() > 1)
0083           {
0084             std::cout << "Found " << logical_name << " in dCache, returning "
0085                       << pfn << std::endl;
0086           }
0087           break;
0088         }
0089       }
0090       else if (iter == "XROOTD")
0091       {
0092         if (Verbosity() > 1)
0093         {
0094           std::cout << "Searching FileCatalog for XRootD file "
0095                     << logical_name << std::endl;
0096         }
0097         if (XRootDSearch(logical_name))
0098         {
0099           if (Verbosity() > 1)
0100           {
0101             std::cout << "Found " << logical_name << " in XRootD, returning "
0102                       << pfn << std::endl;
0103           }
0104           break;
0105         }
0106       }
0107       else if (iter == "LUSTRE")
0108       {
0109         if (Verbosity() > 1)
0110         {
0111           std::cout << "Searching FileCatalog for Lustre file "
0112                     << logical_name << std::endl;
0113         }
0114         if (LustreSearch(logical_name))
0115         {
0116           if (Verbosity() > 1)
0117           {
0118             std::cout << "Found " << logical_name << " in Lustre, returning "
0119                       << pfn << std::endl;
0120           }
0121           break;
0122         }
0123       }
0124       else if (iter == "RAWDATA")
0125       {
0126         if (Verbosity() > 1)
0127         {
0128           std::cout << "Searching FileCatalog for Raw Data file "
0129                     << logical_name << std::endl;
0130         }
0131         if (RawDataSearch(logical_name))
0132         {
0133           if (Verbosity() > 1)
0134           {
0135             std::cout << "Found raw data file " << logical_name << " in Lustre, returning "
0136                       << pfn << std::endl;
0137           }
0138           break;
0139         }
0140       }
0141       else if (iter == "HPSSRAW")
0142       {
0143         if (Verbosity() > 1)
0144         {
0145           std::cout << "Searching FileCatalog for Hpss Raw Data File "
0146                     << logical_name << std::endl;
0147         }
0148         if (HpssRawDataSearch(logical_name))
0149         {
0150           if (Verbosity() > 1)
0151           {
0152             std::cout << "Found raw data file " << logical_name << " in Hpss, returning "
0153                       << pfn << std::endl;
0154           }
0155           break;
0156         }
0157       }
0158       else if (iter == "MINIO")
0159       {
0160         if (Verbosity() > 1)
0161         {
0162           std::cout << "Searching FileCatalog for Lustre file via MinIO "
0163                     << logical_name << std::endl;
0164         }
0165         if (MinIOSearch(logical_name))
0166         {
0167           if (Verbosity() > 1)
0168           {
0169             std::cout << "Found " << logical_name << " in Lustre, returning MinIO URL "
0170                       << pfn << std::endl;
0171           }
0172           break;
0173         }
0174       }
0175       else  // assuming this is a file path
0176       {
0177         if (Verbosity() > 0)
0178         {
0179           std::cout << "Trying path " << iter << std::endl;
0180         }
0181         std::string fullfile(iter);
0182         fullfile.append("/").append(logical_name);
0183         if (localSearch(fullfile))
0184         {
0185           break;
0186         }
0187       }
0188     }
0189   }
0190   catch (...)
0191   {
0192     if (Verbosity() > 0)
0193     {
0194       std::cout << "FROG: GSEARCHPATH not set " << std::endl;
0195     }
0196   }
0197   Disconnect();
0198   return pfn.c_str();
0199 }
0200
0201 bool FROG::localSearch(const std::string &logical_name)
0202 {
0203   if (std::ifstream(logical_name))
0204   {
0205     pfn = logical_name;
0206     return true;
0207   }
0208   return false;
0209 }
0210
0211 odbc::Connection *FROG::GetConnection(const std::string &database)
0212 {
0213   auto iter = m_OdbcConnectionMap.find(database);
0214   if (iter != m_OdbcConnectionMap.end())
0215   {
0216     return iter->second;
0217   }
0218   std::random_device ran_dev;
0219   std::seed_seq seeds{ran_dev(), ran_dev(), ran_dev()};  //...
0220   std::mt19937_64 mersenne_twister(seeds);
0221   std::uniform_int_distribution<> uniform(m_MIN_SLEEP_DUR, m_MAX_SLEEP_DUR);
0222   int icount = 0;
0223   do
0224   {
0225     try
0226     {
0227       odbc::Connection *connection = odbc::DriverManager::getConnection(database, "", "");
0228       m_OdbcConnectionMap[database] = connection;
0229       return connection;
0230     }
0231     catch (odbc::SQLException &e)
0232     {
0233       std::cout << PHWHERE
0234                 << " Exception caught during DriverManager::getConnection" << std::endl;
0235       std::cout << "Message: " << e.getMessage() << std::endl;
0236     }
0237     icount++;
0238     int sleep_time_ms = uniform(mersenne_twister);
0239     std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time_ms));
0240   } while (icount < m_MAX_NUM_RETRIES);
0241   return nullptr;
0242 }
0243
0244 void FROG::Disconnect()
0245 {
0246   for (const auto &iter : m_OdbcConnectionMap)
0247   {
0248     delete iter.second;
0249   }
0250   m_OdbcConnectionMap.clear();
0251 }
0252
0253 bool FROG::PGSearch(const std::string &lname)
0254 {
0255   bool bret = false;
0256   odbc::Connection *odbc_connection = GetConnection("FileCatalog_read");
0257   if (!odbc_connection)
0258   {
0259     return bret;
0260   }
0261   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name <> 'hpss' and full_host_name <> 'dcache' and full_host_name <> 'lustre'";
0262
0263   if (Verbosity() > 1)
0264   {
0265     std::cout << "sql query:" << std::endl
0266               << sqlquery << std::endl;
0267   }
0268   odbc::Statement *stmt = odbc_connection->createStatement();
0269   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0270
0271   if (rs->next())
0272   {
0273     pfn = rs->getString(1);
0274     bret = true;
0275   }
0276   delete rs;
0277   delete stmt;
0278   return bret;
0279 }
0280
0281 bool FROG::dCacheSearch(const std::string &lname)
0282 {
0283   bool bret = false;
0284   odbc::Connection *odbc_connection = GetConnection("FileCatalog_read");
0285   if (!odbc_connection)
0286   {
0287     return bret;
0288   }
0289   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'dcache'";
0290
0291   if (Verbosity() > 1)
0292   {
0293     std::cout << "sql query:" << std::endl
0294               << sqlquery << std::endl;
0295   }
0296   odbc::Statement *stmt = odbc_connection->createStatement();
0297   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0298
0299   if (rs->next())
0300   {
0301     std::string dcachefile = rs->getString(1);
0302     if (std::ifstream(dcachefile))
0303     {
0304       pfn = "dcache:" + dcachefile;
0305       bret = true;
0306     }
0307   }
0308   delete rs;
0309   delete stmt;
0310   return bret;
0311 }
0312
0313 bool FROG::XRootDSearch(const std::string &lname)
0314 {
0315   bool bret = false;
0316   odbc::Connection *odbc_connection = GetConnection("FileCatalog_read");
0317   if (!odbc_connection)
0318   {
0319     return bret;
0320   }
0321   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0322   if (Verbosity() > 1)
0323   {
0324     std::cout << "sql query:" << std::endl
0325               << sqlquery << std::endl;
0326   }
0327   odbc::Statement *stmt = odbc_connection->createStatement();
0328   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0329
0330   if (rs->next())
0331   {
0332     std::string xrootdfile = rs->getString(1);
0333     pfn = "root://xrdsphenix.rcf.bnl.gov/" + xrootdfile;
0334     bret = true;
0335   }
0336   delete rs;
0337   delete stmt;
0338   return bret;
0339 }
0340
0341 bool FROG::LustreSearch(const std::string &lname)
0342 {
0343   bool bret = false;
0344   odbc::Connection *odbc_connection = GetConnection("FileCatalog_read");
0345   if (!odbc_connection)
0346   {
0347     return bret;
0348   }
0349   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0350
0351   odbc::Statement *stmt = odbc_connection->createStatement();
0352   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0353
0354   if (rs->next())
0355   {
0356     pfn = rs->getString(1);
0357     bret = true;
0358   }
0359   delete rs;
0360   delete stmt;
0361   return bret;
0362 }
0363
0364 bool FROG::MinIOSearch(const std::string &lname)
0365 {
0366   bool bret = false;
0367   odbc::Connection *odbc_connection = GetConnection("FileCatalog_read");
0368   if (!odbc_connection)
0369   {
0370     return bret;
0371   }
0372   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0373
0374   if (Verbosity() > 1)
0375   {
0376     std::cout << "sql query:" << std::endl
0377               << sqlquery << std::endl;
0378   }
0379   odbc::Statement *stmt = odbc_connection->createStatement();
0380   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0381
0382   if (rs->next())
0383   {
0384     pfn = rs->getString(1);
0385     std::string toreplace("/sphenix/lustre01/sphnxpro");
0386     size_t strpos = pfn.find(toreplace);
0387     if (strpos == std::string::npos)
0388     {
0389       std::cout << " could not locate " << toreplace
0390                 << " in full file path " << pfn << std::endl;
0391       exit(1);
0392     }
0393     else if (strpos > 0)
0394     {
0395       std::cout << "full file path " << pfn
0396                 << "does not start with " << toreplace << std::endl;
0397       exit(1);
0398     }
0399     pfn.replace(pfn.begin(), pfn.begin() + toreplace.size(), "s3://sphenixs3.rcf.bnl.gov:9000");
0400     bret = true;
0401   }
0402   delete rs;
0403   delete stmt;
0404   return bret;
0405 }
0406
0407 bool FROG::RawDataSearch(const std::string &lname)
0408 {
0409   bool bret = false;
0410   odbc::Connection *odbc_connection = GetConnection("RawdataCatalog_read");
0411   if (!odbc_connection)
0412   {
0413     return bret;
0414   }
0415   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'lustre'";
0416
0417   odbc::Statement *stmt = odbc_connection->createStatement();
0418   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0419
0420   if (rs->next())
0421   {
0422     pfn = rs->getString(1);
0423     bret = true;
0424   }
0425   delete rs;
0426   delete stmt;
0427   return bret;
0428 }
0429
0430 bool FROG::HpssRawDataSearch(const std::string &lname)
0431 {
0432   bool bret = false;
0433   odbc::Connection *odbc_connection = GetConnection("RawdataCatalog_read");
0434   if (!odbc_connection)
0435   {
0436     return bret;
0437   }
0438   std::string sqlquery = "SELECT full_file_path from files where lfn='" + lname + "' and full_host_name = 'hpss'";
0439
0440   odbc::Statement *stmt = odbc_connection->createStatement();
0441   odbc::ResultSet *rs = stmt->executeQuery(sqlquery);
0442
0443   if (rs->next())
0444   {
0445     pfn = rs->getString(1);
0446     bret = true;
0447   }
0448   delete rs;
0449   delete stmt;
0450   return bret;
0451 }