framework/ffarawmodules/ClockDiffCheck.cc

0001 #include "ClockDiffCheck.h"
0002
0003 #include <fun4all/Fun4AllInputManager.h>
0004 #include <fun4all/Fun4AllReturnCodes.h>
0005 #include <fun4all/SubsysReco.h>  // for SubsysReco
0006
0007 #include <ffarawobjects/CaloPacket.h>
0008 #include <ffarawobjects/CaloPacketContainer.h>
0009
0010 #include <phool/PHCompositeNode.h>
0011 #include <phool/PHDataNode.h>
0012 #include <phool/PHNode.h>          // for PHNode
0013 #include <phool/PHNodeIterator.h>  // for PHNodeIterator
0014 #include <phool/PHPointerListIterator.h>
0015 #include <phool/getClass.h>
0016
0017 #include <TH1.h>
0018 #include <TSystem.h>
0019
0020 #include <bitset>
0021 #include <iostream>  // for operator<<, endl, basic_ost...
0022 #include <utility>   // for pair
0023 #include <vector>    // for vector
0024
0025 //____________________________________________________________________________..
0026 ClockDiffCheck::ClockDiffCheck(const std::string &name)
0027   : SubsysReco(name)
0028 {
0029 }
0030
0031 ClockDiffCheck::~ClockDiffCheck()
0032 {
0033   for (auto &[packetid, tup] : m_PacketStuffMap)
0034   {
0035     delete std::get<3>(tup);
0036     std::get<3>(tup) = nullptr;
0037   }
0038 }
0039
0040 //____________________________________________________________________________..
0041 int ClockDiffCheck::InitRun(PHCompositeNode *topNode)
0042 {
0043   PHNodeIterator iter(topNode);
0044   PHCompositeNode *dstNode = dynamic_cast<PHCompositeNode *>(iter.findFirst("PHCompositeNode", "DST"));
0045   if (!dstNode)
0046   {
0047     std::cout << "ClockDiffCheck: " << "Could not find DST Node" << std::endl;
0048     gSystem->Exit(1);
0049     exit(1);
0050   }
0051   PHNodeIterator iterDst(dstNode);
0052   PHCompositeNode *pktNode = dynamic_cast<PHCompositeNode *>(iterDst.findFirst("PHCompositeNode", "Packets"));
0053   PHCompositeNode *pktKeepNode = dynamic_cast<PHCompositeNode *>(iterDst.findFirst("PHCompositeNode", "PacketsKeep"));
0054   if (pktNode)  // old combined packet containers
0055   {
0056     PHNodeIterator iterPkt(pktNode);
0057     PHPointerListIterator<PHNode> nodeIter(iterPkt.ls());
0058     PHNode *thisNode;
0059     while ((thisNode = nodeIter()))
0060     {
0061       m_PacketNodeNames.push_back(thisNode->getName());
0062     }
0063   }
0064   if (pktKeepNode)  // old combined packet containers
0065   {
0066     PHNodeIterator iterPkt(pktKeepNode);
0067     PHPointerListIterator<PHNode> nodeIter(iterPkt.ls());
0068     PHNode *thisNode;
0069     while ((thisNode = nodeIter()))
0070     {
0071       m_PacketNodeNames.push_back(thisNode->getName());
0072     }
0073   }
0074   return Fun4AllReturnCodes::EVENT_OK;
0075 }
0076
0077 //____________________________________________________________________________..
0078 int ClockDiffCheck::process_event(PHCompositeNode *topNode)
0079 {
0080   count++;
0081   //  PHNodeIterator
0082   PHNodeIterator topnodeiter(topNode);
0083   for (auto &iter : m_PacketStuffMap)
0084   {
0085     std::get<0>(iter.second) = std::get<1>(iter.second);
0086     std::get<4>(iter.second) = false;
0087   }
0088   OfflinePacket *pkt = findNode::getClass<OfflinePacket>(topNode, 14001);
0089   if (pkt)
0090   {
0091     FillPacketDiff(pkt);
0092   }
0093   pkt = findNode::getClass<OfflinePacket>(topNode, "GL1Packet");
0094   if (pkt)
0095   {
0096     FillPacketDiff(pkt);
0097   }
0098
0099   std::vector<std::string> nodenames{"CEMCPackets", "HCALPackets", "MBDPackets", "SEPDPackets", "ZDCPackets"};
0100   for (const auto &iter : nodenames)
0101   {
0102     CaloPacketContainer *cemccont = findNode::getClass<CaloPacketContainer>(topNode, iter);
0103     if (!cemccont)
0104     {
0105       //      std::cout << "ClockDiffCheck: " << "could not find " << iter << " node" << std::endl;
0106     }
0107     else
0108     {
0109       FillCaloClockDiff(cemccont);
0110     }
0111   }
0112
0113   for (const auto &iter : m_PacketNodeNames)
0114   {
0115     if (iter == "14001") { continue;
0116 }
0117     CaloPacket *calopacket = findNode::getClass<CaloPacket>(topNode, iter);
0118     if (!calopacket)
0119     {
0120       std::cout << "ClockDiffCheck: " << "could not find " << iter << " node" << std::endl;
0121     }
0122     else
0123     {
0124       if (calopacket->getStatus() != OfflinePacket::PACKET_OK)
0125       {
0126         static int npacketprnt = 0;
0127         if ( npacketprnt < 100)
0128         {
0129           std::cout << "packet " << calopacket->getIdentifier() << " status marked as " << calopacket->getStatus() << ". Skipping diff check " << count << std::endl;
0130           npacketprnt++;
0131         }
0132         std::get<1>(m_PacketStuffMap[calopacket->getIdentifier()]) = std::numeric_limits<uint64_t>::max();
0133         continue;
0134       }
0135       FillCaloClockDiffSngl(calopacket);
0136     }
0137   }
0138
0139   std::vector<unsigned int> badPackets;
0140   uint64_t refdiff = std::numeric_limits<uint64_t>::max();
0141   auto itergl1 = m_PacketStuffMap.find(14001);
0142   if (itergl1 != m_PacketStuffMap.end())
0143   {
0144     refdiff = std::get<2>(itergl1->second);
0145   }
0146   for (auto &iter : m_PacketStuffMap)
0147   {
0148     if (!std::get<4>(iter.second))
0149     {
0150       std::get<0>(iter.second) = std::numeric_limits<uint64_t>::max();
0151     }
0152     else
0153     {
0154       if (Verbosity() > 2)
0155       {
0156         std::cout << "ClockDiffCheck: " << "looking at " << iter.first
0157                   << ", prev bco: " << std::hex << std::get<0>(iter.second)
0158                   << ", curr bco: " << std::get<1>(iter.second)
0159                   << ", clkdiff: " << std::get<2>(iter.second) << std::dec
0160                   << ", valid: " << std::get<4>(iter.second) << std::endl;
0161       }
0162       if (refdiff == std::numeric_limits<uint64_t>::max())
0163       {
0164         refdiff = std::get<2>(iter.second);
0165       }
0166       else
0167       {
0168         if ((refdiff & 0xFFFFFFFFU) != (std::get<2>(iter.second) & 0xFFFFFFFFU))
0169         {
0170           badPackets.push_back(iter.first);
0171           static int nprint = 0;
0172           if (nprint < 1000 || Verbosity() > 1)
0173           {
0174             std::bitset<32> x(refdiff);
0175             std::bitset<32> y0(std::get<0>(iter.second));
0176             std::bitset<32> y1(std::get<1>(iter.second));
0177             std::bitset<32> y2(std::get<2>(iter.second));
0178             std::cout << "ClockDiffCheck: " << "packet " << iter.first << " had different clock diff: 0x" << std::hex
0179                       << std::get<1>(iter.second) << ", ref diff: 0x" << refdiff << std::dec << std::endl;
0180
0181             std::cout << "reff: " << x << std::endl;
0182             std::cout << "this: " << y2 << std::endl;
0183
0184             std::cout << "prev: " << y0 << std::endl;
0185             std::cout << "curr: " << y1 << std::endl;
0186             nprint++;
0187           }
0188         }
0189       }
0190     }
0191   }
0192
0193   for (const auto &nodeiter : nodenames)
0194   {
0195     CaloPacketContainer *container = findNode::getClass<CaloPacketContainer>(topNode, nodeiter);
0196     if (!container)
0197     {
0198       continue;
0199     }
0200     if (delBadPkts)
0201     {
0202       for (unsigned int i = 0; i < container->get_npackets(); i++)
0203       {
0204         unsigned int packetID = container->getPacket(i)->getIdentifier();
0205         for (unsigned int badPacket : badPackets)
0206         {
0207           if (badPacket == packetID)
0208           {
0209             if (Verbosity() > 1)
0210             {
0211               std::cout << "ClockDiffCheck: " << "Dropping packet " << container->getPacket(i)->getIdentifier() << " for XMIT clock mismatch" << std::endl;
0212             }
0213             container->deletePacket(container->getPacket(i));
0214             break;
0215           }
0216         }
0217       }
0218     }
0219
0220     std::vector<std::vector<int>> EvtCounts;
0221     std::vector<int> NrAndCount(2);
0222     NrAndCount[1] = 1;
0223     int counter = 0;
0224     int bestEvt = -1;
0225     int bestEvtCnt = 0;
0226     unsigned int npacket = container->get_npackets();
0227     for (unsigned int i = 0; i < npacket; i++)
0228     {
0229       CaloPacket *packet = container->getPacket(i);
0230       if (packet)
0231       {
0232         int nrModules = packet->iValue(0, "NRMODULES");
0233         for (int j = 0; j < nrModules; j++)
0234         {
0235           int k;
0236           for (k = 0; k < counter; k++)
0237           {
0238             if (EvtCounts[k][0] == packet->iValue(j, "FEMEVTNR"))
0239             {
0240               EvtCounts[k][1]++;
0241               break;
0242             }
0243           }
0244           if (k >= counter)
0245           {
0246             NrAndCount[0] = packet->iValue(j, "FEMEVTNR");
0247             EvtCounts.push_back(NrAndCount);
0248             counter++;
0249           }
0250         }
0251       }
0252     }
0253     if (counter > 1)
0254     {
0255       for (int i = 0; i < counter; i++)
0256       {
0257         if (bestEvtCnt < EvtCounts[i][1])
0258         {
0259           bestEvtCnt = EvtCounts[i][1];
0260           bestEvt = EvtCounts[i][0];
0261         }
0262       }
0263       for (unsigned int i = 0; i < npacket; ++i)
0264       {
0265         CaloPacket *packet = container->getPacket(i);
0266         if (packet)
0267         {
0268           for (int j = 0; j < packet->iValue(0, "NRMODULES"); j++)
0269           {
0270             if (packet->iValue(j, "FEMEVTNR") != bestEvt && bestEvt != -1)
0271             {
0272               static int icnt = 0;
0273               if (icnt < 1000)
0274               {
0275                 std::cout << "ClockDiffCheck: " << "found different FEM clock for packet " << packet->getIdentifier() << std::endl;
0276                 icnt++;
0277               }
0278               if (delBadPkts)
0279               {
0280                 if (Verbosity() > 1)
0281                 {
0282                   std::cout << "ClockDiffCheck: " << "deleting packet " << packet->getIdentifier()
0283                             << " with fem clock mismatch" << std::endl;
0284                 }
0285                 container->deletePacket(packet);
0286               }
0287               break;
0288             }
0289           }
0290         }
0291       }
0292     }
0293   }
0294   for (const auto &iter : m_PacketNodeNames)
0295   {
0296     CaloPacket *calopacket = findNode::getClass<CaloPacket>(topNode, iter);
0297     if (!calopacket)
0298     {
0299       continue;
0300     }
0301     if (delBadPkts)
0302     {
0303       unsigned int packetID = calopacket->getIdentifier();
0304       for (unsigned int badPacket : badPackets)
0305       {
0306         if (badPacket == packetID)
0307         {
0308           static int icnt = 0;
0309           if (icnt < 1000)
0310           {
0311             std::cout << "ClockDiffCheck: " << "Dropping packet " << calopacket->getIdentifier() << " for XMIT clock mismatch" << std::endl;
0312             icnt++;
0313           }
0314           calopacket->Reset();
0315           break;
0316         }
0317       }
0318     }
0319     bool FemClockOk = CheckFemEventNr(calopacket);
0320     if (!FemClockOk)
0321     {
0322       static int icnt = 0;
0323       if (icnt < 100)
0324       {
0325         std::cout << "FemClockOk failed. Resetting packet " << calopacket->getIdentifier()
0326           << " with fem event and clock mismatch" << std::endl;
0327         icnt++;
0328       }
0329       calopacket->Reset();
0330     }
0331   }
0332
0333   return Fun4AllReturnCodes::EVENT_OK;
0334 }
0335
0336 void ClockDiffCheck::FillCaloClockDiff(CaloPacketContainer *pktcont)
0337 {
0338   for (unsigned int i = 0; i < pktcont->get_npackets(); i++)
0339   {
0340     FillCaloClockDiffSngl(pktcont->getPacket(i));
0341   }
0342   return;
0343 }
0344
0345 void ClockDiffCheck::FillCaloClockDiffSngl(CaloPacket *calopkt)
0346 {
0347   unsigned int packetid = calopkt->getIdentifier();
0348   if (!m_PacketStuffMap.contains(packetid))
0349   {
0350     std::string hname = "clkdiff" + std::to_string(packetid);
0351     TH1 *h1 = new TH1F(hname.c_str(), hname.c_str(), 100, 0, 99);
0352     m_PacketStuffMap[packetid] = std::make_tuple(std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), h1, false);
0353     if (Verbosity() > 3)
0354     {
0355       std::cout << "ClockDiffCheck: " << "Add tuple for " << packetid << std::endl;
0356       auto &pktiter = m_PacketStuffMap[packetid];
0357       std::cout << PHWHERE << "packet init " << packetid << std::hex
0358         << ", clk: " << std::get<1>(pktiter)
0359         << ", clkdiff: " << std::get<2>(pktiter) << std::dec << ", valid: " << std::get<4>(pktiter)
0360         << std::endl;
0361     }
0362   }
0363   else
0364   {
0365     auto &pktiter = m_PacketStuffMap[packetid];
0366     uint64_t clk = calopkt->getBCO();
0367     uint64_t clkdiff = std::numeric_limits<uint64_t>::max();
0368     std::get<1>(pktiter) = clk;
0369     // only calculate clk diff and correct clock for rollover if previous clk is set (default is max uint64)
0370     if (std::get<0>(pktiter) < std::numeric_limits<uint64_t>::max())
0371     {
0372       if (clk < std::get<0>(pktiter))
0373       {
0374         clk |= 0x100000000U;
0375       }
0376       clkdiff = clk - std::get<0>(pktiter);
0377       clk &= 0xFFFFFFFF;
0378       std::get<2>(pktiter) = clkdiff;
0379       std::get<4>(pktiter) = true;
0380     }
0381
0382     if (Verbosity() > 2)
0383     {
0384       std::cout << "ClockDiffCheck: " << "packet " << packetid << ", clk: " << std::hex << clk
0385                 << ", clk(tup): " << std::get<1>(pktiter) << ", diff: " << clkdiff
0386                 << ", diff(tup): " << std::get<2>(pktiter) << std::dec << ", valid: " << std::get<4>(pktiter)
0387                 << std::endl;
0388     }
0389   }
0390 }
0391
0392 void ClockDiffCheck::FillPacketDiff(OfflinePacket *pkt)
0393 {
0394   unsigned int packetid = pkt->getIdentifier();
0395   uint64_t clk = (pkt->getBCO() & 0xFFFFFFFF);
0396   if (!m_PacketStuffMap.contains(packetid))
0397   {
0398     std::string hname = "clkdiff" + std::to_string(packetid);
0399     TH1 *h1 = new TH1F(hname.c_str(), hname.c_str(), 100, 0, 99);
0400     m_PacketStuffMap[packetid] = std::make_tuple(std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), h1, false);
0401     if (Verbosity() > 3)
0402     {
0403       std::cout << "ClockDiffCheck: " << "Add tuple for " << packetid << std::endl;
0404     }
0405   }
0406   else
0407   {
0408     auto &pktiter = m_PacketStuffMap[packetid];
0409     uint64_t clkdiff = std::numeric_limits<uint64_t>::max();
0410     if (std::get<0>(pktiter) < std::numeric_limits<uint64_t>::max())
0411     {
0412       if (clk < std::get<0>(pktiter))
0413       {
0414         clk |= 0x100000000U;
0415       }
0416       clkdiff = clk - std::get<0>(pktiter);
0417       clk &= 0xFFFFFFFF;
0418     }
0419     std::get<1>(pktiter) = clk;
0420     std::get<2>(pktiter) = clkdiff;
0421     std::get<4>(pktiter) = true;
0422     if (Verbosity() > 2)
0423     {
0424       std::cout << "ClockDiffCheck: " << "packet " << packetid << ", clk: " << std::hex << clk
0425                 << ", clk(tup): " << std::get<1>(pktiter) << ", diff: " << clkdiff
0426                 << ", diff(tup): " << std::get<2>(pktiter) << std::dec << ", valid: " << std::get<4>(pktiter)
0427                 << std::endl;
0428     }
0429   }
0430 }
0431
0432 bool ClockDiffCheck::CheckFemEventNr(CaloPacket *calopkt)
0433 {
0434   int nrModules = calopkt->iValue(0, "NRMODULES");
0435   std::set<int> EventNoSet;
0436   for (int j = 0; j < nrModules; j++)
0437   {
0438     if (calopkt->getFemStatus(j) == CaloPacket::FEM_OK)
0439     {
0440       EventNoSet.insert(calopkt->iValue(j, "FEMEVTNR"));
0441     }
0442   }
0443   if (EventNoSet.size() > 1)
0444   {
0445     // at least one packet (6024) has a stuck bit in the fem event nr, check fem clock counter in this case
0446     // if they are identical FEM is good (not checked if the FEM clock is stuck though)
0447     std::set<int> FemClockSet;
0448     for (int j = 0; j < nrModules; j++)
0449     {
0450       FemClockSet.insert(calopkt->iValue(j, "FEMCLOCK"));
0451     }
0452     if (FemClockSet.size() == 1)
0453     {
0454       static int icnt = 0;
0455       if (icnt < 100)
0456       {
0457         icnt++;
0458         std::cout << "ClockDiffCheck: " << "clk check Packet " << calopkt->getIdentifier() << " has not unique event numbers"
0459                   << " but FEM Clock counters are identical" << std::endl;
0460       }
0461     }
0462     else  // event nr and fem clock differ
0463     {
0464       // now lets find which one is the outlier
0465       static int icnt = 0;
0466       if (icnt < 1000)
0467       {
0468         icnt++;
0469         std::cout << "ClockDiffCheck: " << "resetting packet " << calopkt->getIdentifier()
0470                   << " with fem event and clock mismatch" << std::endl;
0471         std::map<int, int> EventMap;
0472         std::map<int, int> ClockMap;
0473         for (int j = 0; j < nrModules; j++)
0474         {
0475           EventMap[calopkt->iValue(j, "FEMEVTNR")]++;
0476           ClockMap[calopkt->iValue(j, "FEMCLOCK")]++;
0477         }
0478         for (const auto iterA : EventMap)
0479         {
0480           std::cout << "ClockDiffCheck: " << "Event Nr : " << iterA.first << " shows up " << iterA.second << " times"
0481                     << std::hex << ", Event Nr 0x" << iterA.first << std::dec << std::endl;
0482         }
0483         for (const auto iterA : ClockMap)
0484         {
0485           std::cout << "ClockDiffCheck: " << "Clock : 0x" << std::hex << iterA.first << std::dec
0486                     << " shows up " << iterA.second << " times" << std::endl;
0487         }
0488       }
0489       return false;
0490     }
0491   }
0492   return true;
0493 }