framework/frog/CreateDstList.pl

0001 #!/usr/bin/env perl
0002
0003 #located in offline/framework/frog/
0004
0005 use DBI;
0006 use strict;
0007 use Getopt::Long;
0008 use Data::Dumper;
0009
0010 sub creatfilelists;
0011 sub printtags;
0012 sub printruns;
0013
0014 my $tag;
0015 my $verbose;
0016 my $runnumber;
0017 my $runlist;
0018 my $printruns;
0019 my $printtags;
0020 my $hpss;
0021 my $noprint;
0022 my $dataset;
0023
0024 GetOptions('dataset:s' => \$dataset, 'hpss' =>\$hpss, 'list:s' => \$runlist, 'printruns' => \$printruns, 'printtags' => \$printtags, 'run:i' => \$runnumber, 'tag:s' => \$tag, "verbose" =>\$verbose);
0025
0026 my %ignore_datasets = (
0027     "mdc2" => 1,
0028     );
0029
0030 my $dbh = DBI->connect("dbi:ODBC:FileCatalog_read") || die $DBI::error;
0031 $dbh->{LongReadLen}=2000; # full file paths need to fit in here
0032
0033 if (defined $printtags)
0034 {
0035     printtags();
0036 }
0037
0038 if (defined $printruns)
0039 {
0040     printruns();
0041 }
0042 if ($#ARGV < 0)
0043 {
0044     if (! defined $noprint)
0045     {
0046     print "usage:\n";
0047     print "CreateDstList.pl <dsttype1> <dsttype2> ...\n";
0048     print "parameters:\n";
0049     print "--hpss <print files in hpss>\n";
0050     print "--list <file with list of runs>\n";
0051     print "--printruns: print existing runs (for piping into a runlist)\n";
0052     print "--printtags: print existing tags\n";
0053     print "--run <run number>\n";
0054     print "--tag <tag (build_cdb_version)> (mandatory)\n";
0055     print "--version <version> (only if we have multiple versions for same build/cdb tag)\n";
0056     print "--verbose print stuff\n";
0057     print "\nfor use with --printruns or --printtags\n";
0058         print "--dataset <dataset (run2pp, run2auau, run3auau, run3cosmics,...)\n";
0059     }
0060     exit(0);
0061 }
0062 my %dsttype = ();
0063 while($#ARGV >= 0)
0064 {
0065     $dsttype{$ARGV[0]} = 1;
0066     shift (@ARGV);
0067 }
0068
0069 if (! defined $tag)
0070 {
0071     print "set tag with --tag <tag>\n";
0072     exit(1);
0073 }
0074 if (! defined $runnumber && ! defined $runlist)
0075 {
0076     print "set run number with --run <run number>\n";
0077     exit(1);
0078 }
0079
0080 my $sqlcmd = sprintf("select filename,dsttype,segment from datasets where runnumber=? and tag = '$tag'");
0081 if (defined $hpss)
0082 {
0083     $sqlcmd = sprintf("%s and status = 0",$sqlcmd)
0084 }
0085 else
0086 {
0087     $sqlcmd = sprintf("%s and status > 0",$sqlcmd)
0088 }
0089 my $getfiles =  $dbh->prepare($sqlcmd);
0090 if (defined $runlist)
0091 {
0092     if (! -f $runlist)
0093     {
0094     print "$runlist does not exist\n";
0095     exit(1);
0096     }
0097     open(F2,"$runlist");
0098     {
0099     while (my $run = <F2>)
0100     {
0101         creatfilelists($run);
0102     }
0103     }
0104 }
0105 else
0106 {
0107     creatfilelists($runnumber);
0108 }
0109
0110 sub creatfilelists
0111 {
0112     my %files = ();
0113     my $run = shift;
0114     $getfiles->execute($run);
0115     while(my @res = $getfiles->fetchrow_array)
0116     {
0117     $files{$res[2]}{$res[1]}{$res[0]} = $res[1];
0118     }
0119     foreach my $dsttyp (keys %dsttype)
0120     {
0121     my $filename = sprintf("%s-%08d.list",lc $dsttyp, $run);
0122     open(F,">$filename");
0123     my $fcnt = 0;
0124     foreach my $segment (sort { $a <=> $b } keys %files)
0125     {
0126         foreach my $file (keys %{$files{$segment}{$dsttyp}})
0127         {
0128         print F "$file\n";
0129         $fcnt++;
0130         }
0131     }
0132     close(F);
0133     if ($fcnt == 0)
0134     {
0135         print "no dsts for type $dsttyp for run $run\n";
0136         unlink $filename;
0137     }
0138     }
0139     #x  print Dumper(\%files);
0140 }
0141
0142 sub printtags
0143 {
0144     my $sqlcmd = sprintf("select distinct(tag) from datasets where ");
0145     my $icnt = 0;
0146     if (defined $dataset)
0147     {
0148     $sqlcmd = sprintf("%s dataset = \'\%s\'",$sqlcmd,$dataset);
0149     }
0150     else
0151     {
0152     foreach my $ignore (sort keys %ignore_datasets)
0153     {
0154         if ($icnt > 0)
0155         {
0156         $sqlcmd = sprintf("%s and ",$sqlcmd);
0157         }
0158         $sqlcmd = sprintf("%s dataset <> \'\%s\'",$sqlcmd,$ignore);
0159         $icnt++;
0160     }
0161     }
0162     $sqlcmd = sprintf("%s and tag is not null order by tag",$sqlcmd);
0163     if (defined $verbose)
0164     {
0165     print "sql cmd: $sqlcmd\n";
0166     }
0167     my $gettags = $dbh->prepare($sqlcmd);
0168     $gettags->execute();
0169     while (my @res = $gettags->fetchrow_array)
0170     {
0171     print "$res[0]\n";
0172     }
0173     exit(0);
0174 }
0175
0176 sub printruns
0177 {
0178     my $sqlcmd = sprintf("select distinct(runnumber) from datasets where ");
0179     my $icnt = 0;
0180     if (defined $dataset)
0181     {
0182     $sqlcmd = sprintf("%s dataset = \'\%s\'",$sqlcmd,$dataset);
0183     }
0184     else
0185     {
0186     foreach my $ignore (sort keys %ignore_datasets)
0187     {
0188         if ($icnt > 0)
0189         {
0190         $sqlcmd = sprintf("%s and ",$sqlcmd);
0191         }
0192         $sqlcmd = sprintf("%s dataset <> \'\%s\'",$sqlcmd,$ignore);
0193         $icnt++;
0194     }
0195     }
0196     $sqlcmd = sprintf("%s and tag = '$tag' and dsttype='$ARGV[0]'order by runnumber",$sqlcmd);
0197     my $getruns =  $dbh->prepare($sqlcmd);
0198     $getruns->execute();
0199     if ($getruns->rows == 0)
0200     {
0201     print "no run found for tag $tag and dst type $ARGV[0]\n";
0202     }
0203     else
0204     {
0205     while (my @res = $getruns->fetchrow_array())
0206     {
0207         print "$res[0]\n";
0208     }
0209     }
0210     exit(0);
0211 }