On Fri, Apr 28, 2006 at 12:12:39PM +0100 or thereabouts, Kostas Georgiou wrote:
> On Fri, Apr 28, 2006 at 11:25:40AM +0100, David McBride wrote:
>
> > I've had exactly this problem. As you've found, the RGMA-GIP code
> > directly calls the information provider scripts fairly continunously,
> > and uses what appears to be a *horribly* inefficient parser to extract
> > the relevant data.
>
> I've rewrote parts of lcg-info-generic last night to speed it up a bit.
> The code is by no means perfect and it really should be using Net::LDAP::Entry
> and Net::LDAP::LDIF which I'll do once I get a few free minutes.
>
> Anyway here is a "alpha" version which is a lot more efficient than the
> original.
Kostas,
I'm convinced:
$ time /opt/lcg/bin/lcg-info-generic /opt/lcg/etc/lcg-info-generic.conf > /tmp/orig.ldif
real 0m50.183s
user 0m15.970s
sys 0m0.020s
$ time /opt/lcg/bin/lcg-info-generic.kostas /opt/lcg/etc/lcg-info-generic.conf > /tmp/kostas.ldif
real 0m5.236s
user 0m0.230s
sys 0m0.210s
which is quite some speedup.
Checking through the output it looks okay other than possibly lowering the case of the attributes which
may in fact be okay as far as ldif is concerned but I expect something will blow up somewhere. But I know
you said it was alpha version. Looking through it it looks like you have lowered the case on purpose?
dn: gluecesebindseuniqueid=dcache-tape.gridpp.rl.ac.uk,gluecesebindgroupceuniqueid=lcgce01.gridpp.rl.ac.uk:2119/jobmanager-lcgpbs-babarl,mds-vo-name=local,o=grid
glueschemaversionmajor: 1
gluecesebindweight: 0
gluecesebindceaccesspoint: /pnfs/gridpp.rl.ac.uk/tape
objectclass: GlueGeneralTop
objectclass: GlueCESEBind
objectclass: GlueSchemaVersion
gluecesebindmountinfo: /pnfs/gridpp.rl.ac.uk/tape
glueschemaversionminor: 2
gluecesebindseuniqueid: dcache-tape.gridpp.rl.ac.uk
gluecesebindceuniqueid: lcgce01.gridpp.rl.ac.uk:2119/jobmanager-lcgpbs-babarL
Steve
>
> Cheers,
> Kostas
> #!/usr/bin/perl -w
> use strict;
> use POSIX;
> use File::stat;
> use FileHandle;
>
> # Set our process group to a distinct value
> setpgrp();
>
> #Read in the configuration file.
> my %config = (
> static_dir => '',
> plugin_dir => '',
> provider_dir => '',
> cache_ttl => '',
> freshness => '',
> timeout => '',
> response => '',
> temp_dir => '',
> );
>
> if ($ARGV[0]){
> my $fh = new FileHandle $ARGV[0]
> or die "Error: Can't open config file: $ARGV[0]\n";
> foreach (<$fh>){
> if ((! m/^\#/) & (m/=/)){
> m/^(.*)=(.*)$/;
> my $key=$1;
> my $value=$2;
> $key=~s/\s+//g;
> $value=~s/\s+//g;
> $config{$key}=$value;
> }
> }
> }else{
> print "Usage: lcg-info-generic <config file>\n";
> exit 1;
> }
>
> while ( my ($key, $value) = each(%config) ) {
> if ( ! $config{$key} ){
> print "Error: configuration parameter $key not set.\n";
> exit 2;
> }
> }
>
> my @files=glob("$config{static_dir}/*.ldif");
> my @static;
>
> foreach(@files){
> #Reads the static ldif.
> open (STATIC, "$_") || die "Couldn't open config file, $_\n";
> while (<STATIC>) {
> push @static, $_;
> }
> close (STATIC);
> }
>
> chomp($config{temp_dir});
> mkdir($config{temp_dir});
>
> my @dynamic;
> my @dynamic_ldif;
> my @provider;
> my $pid;
> my @pid;
> my $file_time;
> my $current_time=time;
>
> #Remove junk files.
> my @junk;
> push @junk, glob("$config{plugin_dir}/*~");
> push @junk, glob("$config{provider_dir}/*~");
> push @junk, glob("$config{plugin_dir}/\#*");
> push @junk, glob("$config{provider_dir}/\#*");
>
> foreach(@junk){
> remove($_);
> }
>
> @dynamic=glob("$config{plugin_dir}/*");
> @provider=glob("$config{provider_dir}/*");
>
> for(@dynamic, @provider){
> my $chksum = unpack("%64C*", $_);
> my @file= split ( );
> $file[0]=~s/.*\///;
>
> if(! -e "$config{temp_dir}/$file[0].ldif.$chksum"){
> system("touch $config{temp_dir}/$file[0].ldif.$chksum");
> $file_time = $current_time - 21;
> }
> else{
> $file_time=stat("$config{temp_dir}/$file[0].ldif.$chksum");
> if($file_time){
> $file_time=$file_time->mtime;
> }else{
> $file_time=0;
> }
> }
>
> if($file_time + $config{freshness} < $current_time ){
>
> # Fork the search.
> unless ($pid=fork){
>
> close STDOUT;
>
> # Set our process group to a distinct value.
> setpgrp();
> my $PGRP=getpgrp();
>
> # Eval will kill the process if it times out.
> eval {
> local $SIG{ALRM} = sub { die "GOT TIRED OF WAITING" };
> alarm ($config{timeout}); #Will call alarm after the timeout.
> system("$_ > $config{temp_dir}/$file[0].ldif.$chksum.$PGRP");
> `mv $config{temp_dir}/$file[0].ldif.$chksum.$PGRP $config{temp_dir}/$file[0].ldif.$chksum`;
> alarm(0); # Cancel the pending alarm if responds.
> };
>
> # This sections is executed if the process times out.
> if ($@ =~ /GOT TIRED OF WAITING/) {
> `rm -f $config{temp_dir}/$file[0].ldif.$chksum.$PGRP`;
> kill (-SIGKILL(), $PGRP);
> exit 1;
> }
> exit 0;
> }
> push @pid, $pid;
> }
> }
>
> # Eval will kill the process if it times out.
> eval {
> local $SIG{ALRM} = sub { die "GOT TIRED OF WAITING" };
> alarm ($config{response}); #Will call alarm after the timeout.
> foreach(@pid){
> waitpid($_,0);
> }
> alarm(0); # Cancel the pending alarm if responds.
> };
>
> for(@provider){
> my $chksum = unpack("%64C*", $_);
> my @file= split ( );
> $file[0]=~s/.*\///;
> $file_time=stat("$config{temp_dir}/$file[0].ldif.$chksum");
> if($file_time){
> $file_time=$file_time->mtime;
> }else{
> $file_time=$current_time;
> }
> if($file_time + $config{cache_ttl} > $current_time ){
> #Reads the output from the provider script.
> open (PROVIDER, "$config{temp_dir}/$file[0].ldif.$chksum") or exit 1;
> while (<PROVIDER>) {
> push @static, $_;
> }
> close PROVIDER;
> }
> }
>
> for(@dynamic){
> my $chksum = unpack("%64C*", $_);
> my @file= split ( );
> $file[0]=~s/.*\///;
> $file_time=stat("$config{temp_dir}/$file[0].ldif.$chksum");
> if($file_time){
> $file_time=$file_time->mtime;
> }else{
> $file_time=$current_time;
> }
> if($file_time + $config{cache_ttl} > $current_time ){
> #Reads the output from the dynamic script.
> open (DYNAMIC, "$config{temp_dir}/$file[0].ldif.$chksum") or exit 1;
> while (<DYNAMIC>) {
> push @dynamic_ldif, $_;
> }
> close DYNAMIC;
> }
> }
>
> my $static_dn;
> my $dynamic_dn;
> my %dynamic_hash;
> my %static_hash;
>
> for(@dynamic_ldif){
> if (/^dn:\s*(.*)/){
> $dynamic_dn = lc($1);
> if( ! defined $dynamic_hash{ $dynamic_dn } ) {
> $dynamic_hash{ $dynamic_dn } = {};
> }
> }elsif(m/^([^:]*:)\s*(.*)/){
> my $attr = lc($1);
> if ( ! defined(${ $dynamic_hash{ $dynamic_dn } }{ $attr }) ) {
> ${ $dynamic_hash{ $dynamic_dn } }{ $attr } = [];
> }
> push @{${ $dynamic_hash{ $dynamic_dn } }{ $attr }}, $2;
> }
> }
>
> for(@static){
> if (/^dn:\s*(.*)/){
> $static_dn = lc($1);
> if( ! defined $static_hash{ $static_dn } ) {
> $static_hash{ $static_dn } = {};
> }
> }elsif(m/^([^:]*:)\s*(.*)/){
> my $attr = lc($1);
> if ( ! defined(${ $static_hash{ $static_dn } }{ $attr }) ) {
> ${ $static_hash{ $static_dn } }{ $attr } = [];
> }
> push @{${ $static_hash{ $static_dn } }{ $attr }}, $2;
> }
> }
>
> print "\n";
> foreach my $dn ( keys %static_hash ) {
> print "dn: $dn\n";
> foreach my $attr ( keys %{$static_hash{ $dn }}) {
> if ( defined( @{${ $dynamic_hash{ $dn } }{ $attr }} ) ) {
> for my $val ( @{${ $dynamic_hash{ $dn } }{ $attr }} ) {
> print "$attr $val\n";
> }
> } else {
> for my $val ( @{${ $static_hash{ $dn } }{ $attr }} ) {
> print "$attr $val\n";
> }
> }
> }
> print "\n";
> }
>
> exit;
>
--
Steve Traylen
[log in to unmask]
http://www.gridpp.ac.uk/
|