Back to home page

sPhenix code displayed by LXR

 
 

    


Warning, file /macros/calibrations/tpc/gemcurrents/trip_history.py was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).

0001 #!/usr/bin/env python3
0002 
0003 import requests
0004 import statistics
0005 
0006 
0007 class TripHistory:
0008 
0009     # ==============================================================
0010     # Prometheus / timing configuration
0011     # ==============================================================
0012 
0013     PROMETHEUS_URL = (
0014         "http://promspx01.sdcc.bnl.local:9090/api/v1/query_range"
0015     )
0016 
0017     PROMETHEUS_STEP = "10s"
0018     RESTORE_SAMPLES = 6
0019 
0020     mskON = 0b000000000001
0021 
0022     # ==============================================================
0023     # Trip onset
0024     # ==============================================================
0025 
0026     TRIP_ONSET_VMON_MAX = 10.0
0027 
0028     # More than one GEM tripped at the same sample is treated as a
0029     # beam-loss / massive-trip condition.  Such a run is rejected
0030     # from that point to the end rather than allowing a common-mode
0031     # standby state to masquerade as a recovered gain configuration.
0032     MASSIVE_TRIP_GEM_COUNT = 300  # 289 or greater defeats this feature...
0033 
0034     # ==============================================================
0035     # Field-restoration criteria
0036     #
0037     # These retain the existing FieldOK definition:
0038     #
0039     #   1. commanded total stack voltage is correct
0040     #   2. every supply has reached its CURRENT command
0041     #   3. monitored total stack voltage is correct
0042     #
0043     # Balanced recovery is therefore allowed.
0044     # ==============================================================
0045 
0046     DELTA_SET_MAX = 1.0
0047     DELTA_CHAN_MAX = 5.0
0048     DELTA_MON_MAX = 10.0
0049 
0050     MODULE_LAYERS = {
0051         "G1", "G2", "G3", "G4",
0052         "T1", "T2", "T3", "T4"
0053     }
0054 
0055     REFERENCE_CHANNELS = [
0056         "N.00.IFC",
0057         "N.00.OFC",
0058         "S.00.OFC"
0059     ]
0060 
0061     # ==============================================================
0062     # Gain-restoration criteria
0063     #
0064     # GainOK is now a MOVING-TARGET test.
0065     #
0066     # For each GEM channel:
0067     #
0068     #       W_i = N_i * V_i
0069     #
0070     # and W_i is compared with the live median of healthy peers.
0071     #
0072     # G1 : one global peer group, N = 1
0073     # G2 : one global peer group, module-dependent N
0074     # G3 : one global peer group, N = 1
0075     # G4 : separate R1/R2/R3 peer groups, N = 1
0076     #
0077     # T1-T4 do NOT enter GainOK.  They remain fully represented in
0078     # FieldOK, which tests the complete eight-channel stack.
0079     # ==============================================================
0080 
0081     GAIN_V0SET_TOLERANCE = 1.0
0082     GAIN_VMON_TOLERANCE = 5.0
0083 
0084     GEM_LAYERS = {"G1", "G2", "G3", "G4"}
0085 
0086     # Excluding the channel being tested, the maximum peer counts are
0087     # 71 for G1/G2/G3 and 23 for each radial G4 group.
0088     MIN_GAIN_PEERS = {
0089         "G1": 68,
0090         "G2": 68,
0091         "G3": 68,
0092         "G4_R1": 22,
0093         "G4_R2": 22,
0094         "G4_R3": 22,
0095     }
0096 
0097     # ==============================================================
0098     # G2 normalization
0099     #
0100     # Generated directly from the final stable
0101     # ISOBUTANE-OPERATE-GAIN-BALANCED.json configuration:
0102     #
0103     #       N_i = 285.6 V / V_i(reference)
0104     #
0105     # so that a healthy G2 population occupies a common W coordinate.
0106     # ==============================================================
0107 
0108     G2_REFERENCE_VOLTAGE = 285.6
0109 
0110     G2_NORMALIZATION = {
0111         "N.01.R1.G2": 0.950415973,
0112         "N.01.R2.G2": 1.032537961,
0113         "N.01.R3.G2": 0.998252359,
0114         "N.02.R1.G2": 1.019635844,
0115         "N.02.R2.G2": 1.015647226,
0116         "N.02.R3.G2": 1.025862069,
0117         "N.03.R1.G2": 0.992011115,
0118         "N.03.R2.G2": 1.059740260,
0119         "N.03.R3.G2": 1.004219409,
0120         "N.04.R1.G2": 0.968135593,
0121         "N.04.R2.G2": 0.989948007,
0122         "N.04.R3.G2": 1.004219409,
0123         "N.05.R1.G2": 0.966824645,
0124         "N.05.R2.G2": 1.013484741,
0125         "N.05.R3.G2": 1.038923245,
0126         "N.06.R1.G2": 0.979423868,
0127         "N.06.R2.G2": 1.036284470,
0128         "N.06.R3.G2": 0.989948007,
0129         "N.07.R1.G2": 0.957746479,
0130         "N.07.R2.G2": 1.016008538,
0131         "N.07.R3.G2": 1.005633803,
0132         "N.08.R1.G2": 0.974079127,
0133         "N.08.R2.G2": 1.026968716,
0134         "N.08.R3.G2": 1.005633803,
0135         "N.09.R1.G2": 0.997555012,
0136         "N.09.R2.G2": 1.018908313,
0137         "N.09.R3.G2": 0.998252359,
0138         "N.10.R1.G2": 0.956143288,
0139         "N.10.R2.G2": 0.995121951,
0140         "N.10.R3.G2": 1.028818444,
0141         "N.11.R1.G2": 1.013484741,
0142         "N.11.R2.G2": 1.011689692,
0143         "N.11.R3.G2": 1.040816327,
0144         "N.12.R1.G2": 0.983471074,
0145         "N.12.R2.G2": 1.064480060,
0146         "N.12.R3.G2": 1.030674847,
0147         "S.01.R1.G2": 1.008830802,
0148         "S.01.R2.G2": 1.024390244,
0149         "S.01.R3.G2": 1.025125628,
0150         "S.02.R1.G2": 0.981106149,
0151         "S.02.R2.G2": 1.037037037,
0152         "S.02.R3.G2": 0.985167299,
0153         "S.03.R1.G2": 0.994428969,
0154         "S.03.R2.G2": 1.023289144,
0155         "S.03.R3.G2": 1.048073394,
0156         "S.04.R1.G2": 0.990634755,
0157         "S.04.R2.G2": 0.987551867,
0158         "S.04.R3.G2": 1.019635844,
0159         "S.05.R1.G2": 1.000000000,
0160         "S.05.R2.G2": 0.970438328,
0161         "S.05.R3.G2": 0.967807523,
0162         "S.06.R1.G2": 0.989262210,
0163         "S.06.R2.G2": 0.964864865,
0164         "S.06.R3.G2": 0.977078344,
0165         "S.07.R1.G2": 1.012048193,
0166         "S.07.R2.G2": 1.000000000,
0167         "S.07.R3.G2": 0.983471074,
0168         "S.08.R1.G2": 0.976744186,
0169         "S.08.R2.G2": 0.962588473,
0170         "S.08.R3.G2": 0.964864865,
0171         "S.09.R1.G2": 0.949783838,
0172         "S.09.R2.G2": 1.049228508,
0173         "S.09.R3.G2": 1.017094017,
0174         "S.10.R1.G2": 1.012048193,
0175         "S.10.R2.G2": 1.014204545,
0176         "S.10.R3.G2": 1.000700771,
0177         "S.11.R1.G2": 0.961292494,
0178         "S.11.R2.G2": 1.017456359,
0179         "S.11.R3.G2": 0.996858639,
0180         "S.12.R1.G2": 0.974744027,
0181         "S.12.R2.G2": 0.984827586,
0182         "S.12.R3.G2": 0.975743082,
0183     }
0184 
0185     # --------------------------------------------------------------
0186     # One historical exception.
0187     #
0188     # The physics-run scan shows:
0189     #
0190     #   run 73061 : last observed early S.12.R3.G2 setting
0191     #   run 73816 : first observed final S.12.R3.G2 setting
0192     #
0193     # There are no operative physics runs between them in our list.
0194     #
0195     # Before 73816, S.12.R3.G2 used 288.9 V in the archived balanced
0196     # configuration.  From 73816 onward it belongs to the final 292.7 V
0197     # normalization already stored above.
0198     # --------------------------------------------------------------
0199 
0200     S12R3_G2 = "S.12.R3.G2"
0201     S12R3_FINAL_FIRST_RUN = 73816
0202     S12R3_EARLY_REFERENCE_VOLTAGE = 288.9
0203 
0204     # ==============================================================
0205     # Constructor
0206     # ==============================================================
0207 
0208     def __init__(self, begin_time, end_time, runnumber):
0209 
0210         self.begin_time = float(begin_time)
0211         self.end_time = float(end_time)
0212         self.runnumber = int(runnumber)
0213 
0214         self.v0set_metrics = []
0215         self.vmon_metrics = []
0216         self.status_metrics = []
0217 
0218         self.v0set_by_time = {}
0219         self.vmon_by_time = {}
0220         self.status_by_time = {}
0221 
0222         self.field_bad_intervals = []
0223         self.gain_bad_intervals = []
0224 
0225         self.expected_gem_channels = self._build_expected_gem_channels()
0226 
0227         self._load_voltage_history()
0228         self._build_time_tables()
0229         self._fill_voltage_gaps()
0230         self._find_bad_intervals()
0231 
0232     # ==============================================================
0233     # Public interface
0234     # ==============================================================
0235 
0236     def get(self, timestamp):
0237 
0238         timestamp = float(timestamp)
0239 
0240         field_ok = 1
0241         gain_ok = 1
0242 
0243         for begin_bad, end_bad in self.field_bad_intervals:
0244             if begin_bad <= timestamp <= end_bad:
0245                 field_ok = 0
0246                 break
0247 
0248         for begin_bad, end_bad in self.gain_bad_intervals:
0249             if begin_bad <= timestamp <= end_bad:
0250                 gain_ok = 0
0251                 break
0252 
0253         return field_ok, gain_ok
0254 
0255     # ==============================================================
0256     # Prometheus
0257     # ==============================================================
0258 
0259     def _query_prometheus(self, variable_name):
0260 
0261         params = {
0262             "query": variable_name,
0263             "start": self.begin_time,
0264             "end": self.end_time,
0265             "step": self.PROMETHEUS_STEP
0266         }
0267 
0268         response = requests.get(
0269             self.PROMETHEUS_URL,
0270             params=params
0271         )
0272 
0273         if response.status_code != 200:
0274 
0275             print(
0276                 "TPC_PROMETHEUS_ERROR",
0277                 variable_name,
0278                 "HTTP",
0279                 response.status_code
0280             )
0281 
0282             return []
0283 
0284         result_json = response.json()
0285 
0286         if result_json.get("status") != "success":
0287 
0288             print(
0289                 "TPC_PROMETHEUS_ERROR",
0290                 variable_name,
0291                 "query unsuccessful"
0292             )
0293 
0294             return []
0295 
0296         return result_json["data"]["result"]
0297 
0298     def _load_voltage_history(self):
0299 
0300         self.v0set_metrics = self._query_prometheus(
0301             "sphenix_tpc_hv_caen_V0Set"
0302         )
0303 
0304         self.vmon_metrics = self._query_prometheus(
0305             "sphenix_tpc_hv_caen_VMon"
0306         )
0307 
0308         self.status_metrics = self._query_prometheus(
0309             "sphenix_tpc_hv_caen_Status"
0310         )
0311 
0312         print("V0Set series:", len(self.v0set_metrics))
0313         print("VMon series:", len(self.vmon_metrics))
0314         print("Status series:", len(self.status_metrics))
0315 
0316     # ==============================================================
0317     # Convert Prometheus output into time-indexed dictionaries
0318     # ==============================================================
0319 
0320     def _build_time_tables(self):
0321 
0322         for metric in self.v0set_metrics:
0323 
0324             channel = metric["metric"].get("ch_name", "unknown")
0325 
0326             for timestamp, value in metric["values"]:
0327 
0328                 timestamp = float(timestamp)
0329 
0330                 if timestamp not in self.v0set_by_time:
0331                     self.v0set_by_time[timestamp] = {}
0332 
0333                 self.v0set_by_time[timestamp][channel] = float(value)
0334 
0335         for metric in self.vmon_metrics:
0336 
0337             channel = metric["metric"].get("ch_name", "unknown")
0338 
0339             for timestamp, value in metric["values"]:
0340 
0341                 timestamp = float(timestamp)
0342 
0343                 if timestamp not in self.vmon_by_time:
0344                     self.vmon_by_time[timestamp] = {}
0345 
0346                 self.vmon_by_time[timestamp][channel] = float(value)
0347 
0348         for metric in self.status_metrics:
0349 
0350             channel = metric["metric"].get("ch_name", "unknown")
0351 
0352             for timestamp, value in metric["values"]:
0353 
0354                 timestamp = float(timestamp)
0355 
0356                 if timestamp not in self.status_by_time:
0357                     self.status_by_time[timestamp] = {}
0358 
0359                 self.status_by_time[timestamp][channel] = int(float(value))
0360 
0361     # ==============================================================
0362     # Fill internal Prometheus gaps with last observation carried
0363     # forward.  V0Set, VMon and Status move together as one state.
0364     # ==============================================================
0365 
0366     def _fill_voltage_gaps(self):
0367 
0368         timestamps = sorted(
0369             set(self.v0set_by_time.keys()) &
0370             set(self.vmon_by_time.keys()) &
0371             set(self.status_by_time.keys())
0372         )
0373 
0374         if len(timestamps) < 2:
0375             return
0376 
0377         step = 10
0378         gap_number = 0
0379 
0380         for previous_time, next_time in zip(
0381             timestamps[:-1],
0382             timestamps[1:]
0383         ):
0384 
0385             gap_width = (
0386                 int(round((next_time - previous_time) / step)) - 1
0387             )
0388 
0389             if gap_width <= 0:
0390                 continue
0391 
0392             gap_number += 1
0393 
0394             print(
0395                 "TPC_PROM_GAP",
0396                 "gap=",
0397                 gap_number,
0398                 "width=",
0399                 gap_width,
0400                 "begin=",
0401                 previous_time,
0402                 "end=",
0403                 next_time
0404             )
0405 
0406             for i in range(1, gap_width + 1):
0407 
0408                 timestamp = previous_time + i * step
0409 
0410                 self.v0set_by_time[timestamp] = (
0411                     self.v0set_by_time[previous_time].copy()
0412                 )
0413 
0414                 self.vmon_by_time[timestamp] = (
0415                     self.vmon_by_time[previous_time].copy()
0416                 )
0417 
0418                 self.status_by_time[timestamp] = (
0419                     self.status_by_time[previous_time].copy()
0420                 )
0421 
0422     # ==============================================================
0423     # Channel helpers
0424     # ==============================================================
0425 
0426     @staticmethod
0427     def _is_real_tpc_channel(channel):
0428 
0429         return (
0430             channel.startswith("N.") or
0431             channel.startswith("S.")
0432         )
0433 
0434     @classmethod
0435     def _is_gem_channel(cls, channel):
0436 
0437         fields = channel.split(".")
0438 
0439         return (
0440             len(fields) == 4 and
0441             fields[3] in cls.GEM_LAYERS
0442         )
0443 
0444     @staticmethod
0445     def _build_expected_gem_channels():
0446 
0447         channels = []
0448 
0449         for side in ("N", "S"):
0450 
0451             for sector in range(1, 13):
0452 
0453                 for radial in ("R1", "R2", "R3"):
0454 
0455                     for layer in ("G1", "G2", "G3", "G4"):
0456 
0457                         channels.append(
0458                             f"{side}.{sector:02d}.{radial}.{layer}"
0459                         )
0460 
0461         return channels
0462 
0463     def _module_name(self, channel):
0464 
0465         fields = channel.split(".")
0466 
0467         if len(fields) != 4:
0468             return None
0469 
0470         side, sector, radial, layer = fields
0471 
0472         if layer not in self.MODULE_LAYERS:
0473             return None
0474 
0475         return side + "." + sector + "." + radial
0476 
0477     def _gain_group(self, channel):
0478 
0479         fields = channel.split(".")
0480 
0481         if len(fields) != 4:
0482             return None
0483 
0484         side, sector, radial, layer = fields
0485 
0486         if layer in ("G1", "G2", "G3"):
0487             return layer
0488 
0489         if layer == "G4" and radial in ("R1", "R2", "R3"):
0490             return "G4_" + radial
0491 
0492         return None
0493 
0494     # ==============================================================
0495     # Normalization
0496     # ==============================================================
0497 
0498     def _normalization(self, channel):
0499 
0500         fields = channel.split(".")
0501 
0502         if len(fields) != 4:
0503             return None
0504 
0505         layer = fields[3]
0506 
0507         if layer != "G2":
0508             return 1.0
0509 
0510         if channel not in self.G2_NORMALIZATION:
0511             return None
0512 
0513         if (
0514             channel == self.S12R3_G2 and
0515             self.runnumber < self.S12R3_FINAL_FIRST_RUN
0516         ):
0517             return (
0518                 self.G2_REFERENCE_VOLTAGE /
0519                 self.S12R3_EARLY_REFERENCE_VOLTAGE
0520             )
0521 
0522         return self.G2_NORMALIZATION[channel]
0523 
0524     # ==============================================================
0525     # Trip onset
0526     # ==============================================================
0527 
0528     def _tripped_gem_channels(self, timestamp):
0529 
0530         vmon = self.vmon_by_time[timestamp]
0531         status = self.status_by_time[timestamp]
0532 
0533         tripped = set()
0534 
0535         for channel in self.expected_gem_channels:
0536 
0537             if channel in vmon:
0538 
0539                 if abs(vmon[channel]) < self.TRIP_ONSET_VMON_MAX:
0540                     tripped.add(channel)
0541 
0542             if channel in status:
0543 
0544                 if (status[channel] & self.mskON) == 0:
0545                     tripped.add(channel)
0546 
0547         return sorted(tripped)
0548 
0549     def _trip_condition(self, timestamp):
0550 
0551         vmon = self.vmon_by_time[timestamp]
0552         status = self.status_by_time[timestamp]
0553 
0554         trip_found = False
0555 
0556         # Preserve the original broad trip-onset test over all real TPC
0557         # HV channels.  Gain recovery itself is GEM-only.
0558         for channel, value in vmon.items():
0559 
0560             if not self._is_real_tpc_channel(channel):
0561                 continue
0562 
0563             if abs(value) < self.TRIP_ONSET_VMON_MAX:
0564 
0565                 print(
0566                     "TRIP candidate:",
0567                     timestamp,
0568                     channel,
0569                     value
0570                 )
0571 
0572                 trip_found = True
0573 
0574         for channel, value in status.items():
0575 
0576             if not self._is_real_tpc_channel(channel):
0577                 continue
0578 
0579             if (value & self.mskON) == 0:
0580 
0581                 print(
0582                     "TRIP candidate:",
0583                     timestamp,
0584                     channel,
0585                     value
0586                 )
0587 
0588                 trip_found = True
0589 
0590         return trip_found
0591 
0592     # ==============================================================
0593     # Build module collection for FieldOK
0594     # ==============================================================
0595 
0596     def _build_modules(self, timestamp):
0597 
0598         v0set = self.v0set_by_time[timestamp]
0599 
0600         modules = {}
0601 
0602         for channel in v0set.keys():
0603 
0604             if not self._is_real_tpc_channel(channel):
0605                 continue
0606 
0607             module = self._module_name(channel)
0608 
0609             if module is None:
0610                 continue
0611 
0612             layer = channel.split(".")[3]
0613 
0614             if module not in modules:
0615                 modules[module] = {}
0616 
0617             modules[module][layer] = channel
0618 
0619         return modules
0620 
0621     # ==============================================================
0622     # Field restoration -- unchanged physics
0623     # ==============================================================
0624 
0625     def _field_restoration_condition(self, timestamp):
0626 
0627         v0set = self.v0set_by_time[timestamp]
0628         vmon = self.vmon_by_time[timestamp]
0629 
0630         reference_values = []
0631 
0632         for channel in self.REFERENCE_CHANNELS:
0633 
0634             if channel not in v0set:
0635 
0636                 print(
0637                     "FIELD_RESTORE missing reference:",
0638                     timestamp,
0639                     channel
0640                 )
0641 
0642                 return False
0643 
0644             reference_values.append(v0set[channel])
0645 
0646         vtarget = (
0647             sum(reference_values) /
0648             len(reference_values)
0649         )
0650 
0651         modules = self._build_modules(timestamp)
0652 
0653         for module, channels in modules.items():
0654 
0655             if len(channels) != 8:
0656 
0657                 print(
0658                     "FIELD_RESTORE incomplete module:",
0659                     timestamp,
0660                     module,
0661                     len(channels)
0662                 )
0663 
0664                 return False
0665 
0666             sum_v0set = 0.0
0667             sum_vmon = 0.0
0668 
0669             for layer in self.MODULE_LAYERS:
0670 
0671                 if layer not in channels:
0672                     return False
0673 
0674                 channel = channels[layer]
0675 
0676                 if channel not in vmon:
0677                     return False
0678 
0679                 set_voltage = v0set[channel]
0680                 mon_voltage = vmon[channel]
0681 
0682                 sum_v0set += set_voltage
0683                 sum_vmon += mon_voltage
0684 
0685                 if (
0686                     abs(mon_voltage - set_voltage) >=
0687                     self.DELTA_CHAN_MAX
0688                 ):
0689                     return False
0690 
0691             if (
0692                 abs(sum_v0set - vtarget) >=
0693                 self.DELTA_SET_MAX
0694             ):
0695                 return False
0696 
0697             if (
0698                 abs(sum_vmon - vtarget) >=
0699                 self.DELTA_MON_MAX
0700             ):
0701                 return False
0702 
0703         return True
0704 
0705     # ==============================================================
0706     # Live normalized gain targets
0707     # ==============================================================
0708 
0709     def _healthy_gain_population(self, timestamp):
0710 
0711         v0set = self.v0set_by_time[timestamp]
0712         status = self.status_by_time[timestamp]
0713 
0714         groups = {
0715             "G1": [],
0716             "G2": [],
0717             "G3": [],
0718             "G4_R1": [],
0719             "G4_R2": [],
0720             "G4_R3": [],
0721         }
0722 
0723         for channel in self.expected_gem_channels:
0724 
0725             if channel not in v0set:
0726                 continue
0727 
0728             if channel not in status:
0729                 continue
0730 
0731             if (status[channel] & self.mskON) == 0:
0732                 continue
0733 
0734             group = self._gain_group(channel)
0735             normalization = self._normalization(channel)
0736 
0737             if group is None or normalization is None:
0738                 continue
0739 
0740             groups[group].append(
0741                 (
0742                     channel,
0743                     normalization * v0set[channel]
0744                 )
0745             )
0746 
0747         return groups
0748 
0749     def _gain_restoration_condition(self, timestamp):
0750 
0751         v0set = self.v0set_by_time[timestamp]
0752         vmon = self.vmon_by_time[timestamp]
0753         status = self.status_by_time[timestamp]
0754 
0755         groups = self._healthy_gain_population(timestamp)
0756 
0757         # ----------------------------------------------------------
0758         # Every GEM must be present, ON, near its live peer target,
0759         # and actually sitting near its own current command.
0760         # ----------------------------------------------------------
0761 
0762         for channel in self.expected_gem_channels:
0763 
0764             if channel not in v0set:
0765                 return False
0766 
0767             if channel not in vmon:
0768                 return False
0769 
0770             if channel not in status:
0771                 return False
0772 
0773             if (status[channel] & self.mskON) == 0:
0774                 return False
0775 
0776             group = self._gain_group(channel)
0777             normalization = self._normalization(channel)
0778 
0779             if group is None or normalization is None:
0780                 return False
0781 
0782             # Exclusive peer median: do not let the channel being tested
0783             # vote on where its own target should be.
0784             peers = [
0785                 w
0786                 for peer_channel, w in groups[group]
0787                 if peer_channel != channel
0788             ]
0789 
0790             if len(peers) < self.MIN_GAIN_PEERS[group]:
0791                 return False
0792 
0793             target_w = statistics.median(peers)
0794 
0795             set_w = normalization * v0set[channel]
0796             mon_w = normalization * vmon[channel]
0797 
0798             if (
0799                 abs(set_w - target_w) >=
0800                 self.GAIN_V0SET_TOLERANCE
0801             ):
0802                 return False
0803 
0804             if (
0805                 abs(mon_w - set_w) >=
0806                 self.GAIN_VMON_TOLERANCE
0807             ):
0808                 return False
0809 
0810         return True
0811 
0812     # ==============================================================
0813     # Main state machine
0814     # ==============================================================
0815 
0816     def _find_bad_intervals(self):
0817 
0818         timestamps = sorted(
0819             set(self.v0set_by_time.keys()) &
0820             set(self.vmon_by_time.keys()) &
0821             set(self.status_by_time.keys())
0822         )
0823 
0824         if not timestamps:
0825 
0826             print("TPC_TRIP_HISTORY no common timestamps")
0827             return
0828 
0829         bad = False
0830         bad_start = None
0831         massive_trip = False
0832 
0833         field_restored = False
0834         field_restore_count = 0
0835         field_restore_start = None
0836         field_restore_time = None
0837 
0838         gain_restore_count = 0
0839         gain_restore_start = None
0840 
0841         for timestamp in timestamps:
0842 
0843             # ======================================================
0844             # GOOD STATE
0845             # ======================================================
0846 
0847             if not bad:
0848 
0849                 if not self._trip_condition(timestamp):
0850                     continue
0851 
0852                 bad = True
0853                 bad_start = timestamp
0854 
0855                 field_restored = False
0856                 field_restore_count = 0
0857                 field_restore_start = None
0858                 field_restore_time = None
0859 
0860                 gain_restore_count = 0
0861                 gain_restore_start = None
0862 
0863                 tripped_gems = self._tripped_gem_channels(timestamp)
0864 
0865                 massive_trip = (
0866                     len(tripped_gems) >=
0867                     self.MASSIVE_TRIP_GEM_COUNT
0868                 )
0869 
0870                 print(
0871                     "TPC_TRIP_ONSET",
0872                     "time=",
0873                     timestamp,
0874                     "gem_count=",
0875                     len(tripped_gems)
0876                 )
0877 
0878                 if massive_trip:
0879 
0880                     print(
0881                         "TPC_MASSIVE_TRIP",
0882                         "time=",
0883                         timestamp,
0884                         "gem_count=",
0885                         len(tripped_gems)
0886                     )
0887 
0888                 continue
0889 
0890             # ======================================================
0891             # MASSIVE TRIP
0892             #
0893             # Never allow a later common-mode voltage state to be
0894             # mistaken for gain recovery.
0895             # ======================================================
0896 
0897             if massive_trip:
0898                 continue
0899 
0900             # ======================================================
0901             # FIELD RESTORATION
0902             # ======================================================
0903 
0904             field_condition = (
0905                 self._field_restoration_condition(timestamp)
0906             )
0907 
0908             if field_condition:
0909 
0910                 if field_restore_count == 0:
0911                     field_restore_start = timestamp
0912 
0913                 field_restore_count += 1
0914 
0915                 if (
0916                     not field_restored and
0917                     field_restore_count >= self.RESTORE_SAMPLES
0918                 ):
0919 
0920                     field_restored = True
0921                     field_restore_time = field_restore_start
0922 
0923                     self.field_bad_intervals.append(
0924                         (
0925                             bad_start,
0926                             field_restore_time
0927                         )
0928                     )
0929 
0930                     print(
0931                         "TPC_FIELD_RESTORED",
0932                         "time=",
0933                         field_restore_time,
0934                         "dt=",
0935                         field_restore_time - bad_start
0936                     )
0937 
0938             else:
0939 
0940                 if not field_restored:
0941                     field_restore_count = 0
0942                     field_restore_start = None
0943 
0944             # ======================================================
0945             # GAIN RESTORATION
0946             #
0947             # This is independent of any frozen pre-trip snapshot.
0948             # Each sample asks whether all GEMs agree with their live
0949             # healthy peer populations in normalized W coordinates.
0950             # ======================================================
0951 
0952             gain_condition = (
0953                 self._gain_restoration_condition(timestamp)
0954             )
0955 
0956             if gain_condition:
0957 
0958                 if gain_restore_count == 0:
0959                     gain_restore_start = timestamp
0960 
0961                 gain_restore_count += 1
0962 
0963             else:
0964 
0965                 gain_restore_count = 0
0966                 gain_restore_start = None
0967 
0968             # GainOK may not return before FieldOK.
0969             if (
0970                 field_restored and
0971                 gain_restore_count >= self.RESTORE_SAMPLES
0972             ):
0973 
0974                 gain_restore_time = max(
0975                     gain_restore_start,
0976                     field_restore_time
0977                 )
0978 
0979                 self.gain_bad_intervals.append(
0980                     (
0981                         bad_start,
0982                         gain_restore_time
0983                     )
0984                 )
0985 
0986                 print(
0987                     "TPC_GAIN_RESTORED",
0988                     "time=",
0989                     gain_restore_time,
0990                     "dt=",
0991                     gain_restore_time - bad_start
0992                 )
0993 
0994                 bad = False
0995                 bad_start = None
0996                 massive_trip = False
0997 
0998                 field_restored = False
0999                 field_restore_count = 0
1000                 field_restore_start = None
1001                 field_restore_time = None
1002 
1003                 gain_restore_count = 0
1004                 gain_restore_start = None
1005 
1006         # ==========================================================
1007         # Query ended while still bad
1008         # ==========================================================
1009 
1010         if bad:
1011 
1012             if not field_restored:
1013 
1014                 self.field_bad_intervals.append(
1015                     (
1016                         bad_start,
1017                         self.end_time
1018                     )
1019                 )
1020 
1021                 print(
1022                     "TPC_FIELD_NOT_RESTORED",
1023                     "trip_time=",
1024                     bad_start,
1025                     "query_end=",
1026                     self.end_time,
1027                     "dt=",
1028                     self.end_time - bad_start
1029                 )
1030 
1031             self.gain_bad_intervals.append(
1032                 (
1033                     bad_start,
1034                     self.end_time
1035                 )
1036             )
1037 
1038             print(
1039                 "TPC_GAIN_NOT_RESTORED",
1040                 "trip_time=",
1041                 bad_start,
1042                 "query_end=",
1043                 self.end_time,
1044                 "dt=",
1045                 self.end_time - bad_start,
1046                 "massive=",
1047                 int(massive_trip)
1048             )