#!/usr/bin/perl
#$Id: load_test_data_db.pl,v 1.30 2018/06/14 07:25:21 sqko Exp $
use strict;
use warnings;
use RFC::Rig;
use RFC::Header;
use RFC::Main;
use RFC::RFCCGI;
use SemaforeFile;
use DBI qw (:sql_types);
use AFM qw();
use Unicode::String qw(latin1 utf8);
use Sys::Hostname;

my $db = get_cv('database','db');
my $host = get_cv('database','server');
die "No server or database name found in configuration file!\n" unless ($db && $host);
die "Usage: $0 rig [opt test] [opt temp_2_label]\n" unless @ARGV;
my $rig = RFC::Rig->new(shift);
my $test = shift;
$test = $rig->session unless ($test);
my $tmpf = '/tmp/load_test_data-'.$$.'.spline';
my $tmpfc = '/tmp/load_test_data_i-'.$$.'.spline';
my $temp2label = 'T_center';
$temp2label = shift if (@ARGV);

my $dir = $RFC::Header::publicdir.'rig'.$rig->name.'/'.$rig->name.'test'.$test.'/';
my $url = 'https://'.hostname.'/rig'.$rig->name.'/'.$rig->name.'test'.$test.'/';
my $report_info = SemaforeFile->new($dir.'report_information.txt');
my $info = SemaforeFile->new($dir.'info.txt');
my $proglog = SemaforeFile->new($dir.'proglog');
my $ivdir = $dir.'iv/';
my %repinfo = ();
#die "No report information found for test $test on rig ".$rig->name."\n" unless (-e $report_info->filename);
unless (-e $report_info->filename)
{
    die "I-v curves has not been run for test $test on rig ".$rig->name.", aborting!\n" unless (-d $ivdir);
    warn "No report has been finalised, running minireport...\n";
    ### run mini report ##
    my @infolines = $info->readlines();
    foreach my $str (@infolines)
    {
	if ($str =~ /^(\w+)\s*\#+\s*(.*)/)
	{
	    $repinfo{$1} = $2;
	}
    }
    my $leakdir = $dir.'leak/';
    my $isleak = 0;
    foreach ($proglog->readlines)
    {
	next unless ( /Leak measurement\s+\#/ );
	$isleak = 1;
	last;
    }
    system('/usr/local/bin/plot_leak.pl '.$rig->name.' '.$test) if ($isleak && !(-e $leakdir));
    system('/usr/local/bin/plotall.pl '.$rig->name.' '.$test);
}
else
{
    my @lines = $report_info->readlines;
    foreach my $l (@lines)
    {
	next unless $l;
	my ($key,$val) = split(/[\s\t]*\#[\s\t]*/,$l);
	$repinfo{$key} = to_utf8($val);
    }
}


my $ivnr = 0;
my $impdir = $dir.'impedance/';
if (-d $ivdir)
{
    opendir IND,$ivdir or die "Could not open dir $ivdir $!\n";
    my @ivdata = grep(/ivdatadir/,readdir(IND));
    closedir(IND);
    $ivnr = @ivdata;
}
$info->open_read;
my $state = 0;
my @alt_test_nr = ();
my $cell_nr = undef;
while (my $str = $info->readline)
{
    $str =~ s/[\r\n]+//g;
    $str = to_utf8($str);
    if ($state == 0)
    {
	next unless $str =~ /^Alt/i;
	$state = 1;
    }
    if ($state == 1)
    {
	if ($str =~ /^[\w\s\t]+\:\s*(.*)$/)
	{
	    push @alt_test_nr,$1;
	}
	next unless $str =~ /^Cell|Sample/i;
	pop @alt_test_nr;
    }
    if ($str =~ /^[\w\s\t]+\:\s*(.*)$/)
    {
	$cell_nr = $1;
    }
    last;
}
$info->close;
unless ($cell_nr)
{
    foreach my $str ($info->readlines)
    {
	$str = to_utf8($str);
	if ($str =~ /cell[\s\t]*number[\s\t]*\:[\s\t]*(\w+)/)
	{
	    $cell_nr = $1;
	    last;
	}
	if ($str =~ /Sample[\s\t]*number[\s\t]*\:[\s\t]*(\w+)/)
	{
	    $cell_nr = $1;
	    last;
	}
    }
}
my $jdata = SemaforeFile->new($dir.'jdata');
$jdata->open_read;
my $line = $jdata->readline;
$line = to_utf8($line);
$jdata->close;
my @list = split(/[\s\t]+/,$line);
my $jdataline = $line;
my $starttime = to_timestamp($list[1]);
$jdata->open_readback;
$line = $jdata->readline;
$line = to_utf8($line);
$jdata->close;
@list = split(/[\s\t]+/,$line);
my $endtime = to_timestamp($list[1]);

my @proglines = $proglog->readlines;
my %userlist = ();
foreach my $line (@proglines)
{
    $line = to_utf8($line);
    if ($line =~ /user[\s\'\"]+([\w\/\.]+)[\s\w\"\']*$/)
    {
	my $uname = $1;
	if (length($uname) > 32)
	{
	    ## Use last 32 chars as long usernames will likely be due to a path to a program ###
	    $uname =~ /([\w\.]+)$/;
	    $uname = $1;
	    $uname = substr($uname,-32) if (length($uname) > 32); ## If the username is really a long one... ##
	}
	$userlist{$uname}++;
    }
}
my $vlabel = $rig->get_cv_test('IV_control','voltage_label',$test);
$vlabel = 'cell_voltage' unless $vlabel;
my $clabel = $rig->get_cv_test('IV_control','currnet_label',$test);
$clabel = 'current' unless $clabel;
my $templabel = $rig->get_cv_test('IV_control','temp_label',$test);
$templabel = 'T_center' unless $clabel;
my $area = $rig->get_cv_test('main','cell_area',$test);
$area = AFM::get_config_value_old($rig->name,'area',$test) unless ($area);
$area = 1 unless ($area);
my $epsilon = $rig->get_cv_test('IV_control','epsilon',$test);
$epsilon = 0.05 unless ($epsilon);

my @vlevels = (600,700,1200);
my @ilevels = (-0.5,-0.25,-0.15,0.15,0.25,0.5);
my %gasses = ();
my @gaslist = split(/\s*\,\s*/,$rig->get_cv_test('main','gas_names',$test));
@gaslist = split(/\:/,AFM::get_config_value_old($rig->name,'gas_name',$test)) unless (@gaslist);
my $configfile = '/home/http/html/rig'.$rig->name.'/'.$rig->name.'test'.$test.'/configfile';
@gaslist = ('h2','o2','air','ch4','o2_cathode','n2_cathode','n2') unless (@gaslist && -e $configfile);
foreach (@gaslist)
{
    $gasses{$_} = 0;
}
### Remove duplicate gasses (May happen for some rigs) ###
@gaslist = ();
foreach (keys %gasses)
{
    push @gaslist,$_;
}

#### Find gas use ###
my %gasuse = ();

foreach my $gas (@gaslist)
{
    next unless ($jdataline =~ /\b$gas\b/);
    my $cutoff = $rig->get_cv_test('gas_'.$gas,'cutoff_report',$test);
    $cutoff = AFM::get_config_value_old($rig->name,$gas.'_cutoff',$test) unless (defined($cutoff));
    $cutoff = 0 unless ($cutoff);
    my $cmd = 'awk -f /usr/local/bin/celltest/awk/gasuse.awk name='.$gas.' cutoff='.$cutoff.' '.$jdata->filename;
    my $res = `$cmd`;
    chomp $res;
    $gasuse{$gas} = POSIX::floor($res);
}

my $GNUPLOT = &get_cv('GNUPLOT','location');
### Obtain leak data ###
my %leak = ();
my $leakdir = $dir.'leak/';
if (-d $leakdir)
{
    my %searchstring = ('h2' => 'air', 'air' =>'h2');
    my $gp = SemaforeFile->new('/tmp/leak_'.$$.'.gp');
    foreach my $mode ('air','h2')
    {
	my $file = SemaforeFile->new($leakdir.'const'.$mode);
	next unless (-e $file->filename);
	## Find columns ###
	$file->open_read;
	my $line = $file->readline;
	$file->close;
	$line =~ s/^[\s\t]*//g;
	my @list = split(/[\s\t]+/,$line);
	my $x = 0;
	my $y = 0;
	for (my $i = 0; $i < @list; $i++)
	{
	    $y = $i+2 if ($list[$i] =~ /leak_cell/i);
	    $x = $i+2 if (lc($list[$i]) eq $searchstring{$mode});
	}
	my $cmd = "/usr/bin/awk '{print \$$x,\$$y}' ".$file->filename; 
	my @lines = `$cmd`;
	### Discard 10 % of highest and lowest values ####
	my $cutoff = POSIX::floor(@lines * 0.1);
	my $head = @lines - $cutoff;
	my $tail = $head - $cutoff;
	next unless ($tail> 4);
	$cmd = "/usr/bin/awk '{print \$$y,\$$x}' ".$file->filename.' | /bin/sort -g | /usr/bin/head -'.$head.' | /usr/bin/tail -'.$tail." | /bin/awk '{print \$2,\$1*1000/$area}' > ".$tmpf;
	system($cmd);
	$gp->open_write;
	$gp->writeline('f(x) = a * x + b');
	$gp->writeline("fit f(x) '$tmpf' using 1:2 via a,b");
	$gp->close;
	$cmd = $GNUPLOT. ' '. $gp->filename .' 2>&1';
	my $res = `$cmd`;
	next unless ($res =~ /the fit converged/);
	my @datalines = split("\n",$res);
	$x = 0;
	while ($datalines[$x] !~ /variance/i && $x < @datalines)
	{
	    $x++;
	}
	if ($x < @datalines)
	{
	    my %data = ();
	    $datalines[$x] =~ /\:\s*([\d\.\-\+eE]+)/;
	    $data{'variance'} = $1;
	    my $st = join("\n",splice(@datalines,$x,15));
	    $st =~ /\na[\s\t]+\=\s*([\d\.\-\+eE]+)/;
	    $data{'slope'} = $1;
	    $st =~ /\nb[\s\t]+\=\s*([\d\.\-\+eE]+)/;
	    $data{'intercept'} = $1;
	    $data{'n'} = $tail;
	    $leak{$mode} = \%data;
	}
    }
    unlink $tmpf if (-e $tmpf);
    unlink $gp if (-e $gp);
    
}

my $dbh = DBI->connect(&get_cn_args($db));

my $sth = $dbh->prepare('SELECT 1');
$sth->execute();
my $isallive = $sth->fetchrow_array();
$sth->finish();
my $testid = undef;
unless ($isallive)
{
    warn "Remote database did not return true for 'isalive call'!\n";
    $dbh->disconnect();
    exit;
}
eval {
    $dbh->{AutoCommit} = 0;
    $dbh->{RaiseError} = 1;
    $dbh->{PrintError} = 1;
    my $sth = $dbh->prepare('SELECT id,(date_new = date_modify) FROM test_main WHERE rig = ? AND test = ?');
    $sth->execute($rig->name,$test);
    my $dateok;
    ($testid,$dateok) = $sth->fetchrow_array;
    my %taglist = ();
    my $userid = 0;
    ### Get list of tags which will be deleted ###
    $sth = $dbh->prepare('SELECT name,userid FROM tags_iv_test WHERE rig = ? AND test = ? ORDER BY userid,name');
    $sth->execute($rig->name,$test);
    my @list = ();
    while (my ($n,$u) = $sth->fetchrow_array())
    {
	unless ($u == $userid)
	{
	    my @l = @list;
	    $taglist{$userid}{'iv'} = \@l if (@list && $userid);
	    $userid = $u;
	    @list = ();
	}
	push @list,$n;
    }
    my @l = @list;
    $taglist{$userid}{'iv'} = \@l if (@list && $userid);
    $userid = 0;
    @list = ();
    $sth = $dbh->prepare('SELECT name,userid FROM tags_imp_test WHERE rig = ? AND test = ? ORDER BY userid,name');
    $sth->execute($rig->name,$test);
    while (my ($n,$u) = $sth->fetchrow_array())
    {
	unless ($u == $userid)
	{
	    my @l = @list;
	    $taglist{$userid}{'imp'} = \@l if (@list && $userid);
	    $userid = $u;
	    @list = ();
	}
	push @list,$n;
    }
    my @l2 = @list;
    $taglist{$userid}{'imp'} = \@l2 if (@list && $userid);
    foreach my $userid (keys %taglist)
    {
	my $email = RFC::RFCCGI::passwd_client('email_uid',$userid);
	my $string = 'Due to update of the standard report of '.$rig->name.'test'.$test.",\n";
	$string .= "the folowing tags on the celltest datamining database have\nbeen deleted from one or more i-V curves in that test:\n";
	$string .= join("\n",@{$taglist{$userid}{'iv'}})."\n" if (exists $taglist{$userid}{'iv'} && @{$taglist{$userid}{'iv'}});
	$string .= "\nSimilarly, the following tags have been deleted from one or more impedance spectra:\n";
	$string .= join("\n",@{$taglist{$userid}{'imp'}})."\n" if (exists $taglist{$userid}{'imp'} && @{$taglist{$userid}{'imp'}});
	$string .= "\nPlease log in at the cell test datamining facility and review the data for this test\n";
	$string .= "to reassign any tags as appropriate.\n";
	$string .= "\n\nSorry for the inconveniance this may create.\n";
	my $subj = "Celltest datamining tags deleted due to report update";
	RFC::Main::mail($email,$string,$subj) if ($email);
    }
    if ($testid && $dateok)
    {
	$sth = $dbh->prepare('DELETE FROM test_main WHERE id = ?');
	$sth->execute($testid);
	$testid = undef;
    }
    elsif ($testid)
    {
	$sth = $dbh->prepare('DELETE FROM test_users WHERE testid = ?');
	$sth->execute($testid);
	$sth = $dbh->prepare('DELETE FROM ivcurves WHERE testid = ?');
	$sth->execute($testid);
	$sth = $dbh->prepare('DELETE FROM impedance WHERE testid = ?');
	$sth->execute($testid);
	$sth = $dbh->prepare('DELETE FROM test_gas_use WHERE testid = ?');
	$sth->execute($testid);
    }
    ### Load test info ###
    unless ($testid)
    {
	my @leak_h2 = (undef,undef,undef,undef);
	my @leak_air = (undef,undef,undef,undef);
	my $n = 0;
	foreach ('slope','intercept','variance','n')
	{
	    $leak_h2[$n] = $leak{'h2'}{$_} if (exists($leak{'h2'}) && defined($leak{'h2'}) && ref($leak{'h2'}) eq 'HASH' && exists($leak{'h2'}{$_}));
	    $leak_air[$n] = $leak{'air'}{$_} if (exists($leak{'air'}) && defined($leak{'air'}) && ref($leak{'air'}) eq 'HASH' && exists($leak{'air'}{$_}));
	    $n++;
	}
	$sth = $dbh->prepare('INSERT INTO test_main (rig,test,project,keywords,testplan,testresult,comments,info,proglog,start_date,end_date,alt_test_nr,cell_code,h2_leak_slope,h2_leak_intercept,h2_leak_variance,h2_leak_n,air_leak_slope,air_leak_intercept,air_leak_variance,air_leak_n,area,url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
	$sth->execute($rig->name,$test,$repinfo{'PROJECT'},$repinfo{'KEYWORDS'},$repinfo{'TESTPLAN'},$repinfo{'TESTRESULT'},$repinfo{'COMMENTS'},to_utf8(join("\n",$info->readlines)),to_utf8(join("\n",@proglines)),$starttime,$endtime, join("\t",@alt_test_nr),$cell_nr,@leak_h2,@leak_air,$area,$url);
	$sth = $dbh->prepare('SELECT last_value FROM test_main_id_seq');
	$sth->execute();
	$testid = $sth->fetchrow_array();
    }
    ### Load list of users ###
    $sth = $dbh->prepare('INSERT INTO test_users (testid,username,rank) VALUES (?,?,?)');
    foreach my $user (keys %userlist)
    {
	$sth->execute($testid,$user,$userlist{$user});
    }
    ### Load gas use ###
    my $sthgas = $dbh->prepare('INSERT INTO test_gas_use (testid,gas,use) VALUES (?,?,?)');
    foreach my $gas (@gaslist)
    {
	next unless (exists($gasuse{$gas}));
	$sthgas->execute($testid,$gas,$gasuse{$gas});
    }
    ### Load iv data ###
    my $iv_ins = $dbh->prepare('INSERT INTO ivcurves (testid,ivnr,date_start,date_end,temperature,temperature_2,bottle_temp_anode,bottle_temp_anode_enabled,bottle_temp_cathode,bottle_temp_cathode_enabled,leak_current,OCV,OCV_theory,pO2_in,pO2_out,Imax,comments,url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
    my $get_iv_nr = $dbh->prepare('SELECT last_value FROM ivcurves_id_seq');
    my $iv_gas_ins = $dbh->prepare('INSERT INTO ivgasflow (ivid,gasname,flow) VALUES (?,?,?)');
    my $iv_ins_res = $dbh->prepare('INSERT INTO iv_res (ivid,type,value,secant,fu_corr) VALUES (?,?,?,?,?)');
    foreach my $i (1..$ivnr)
    {
	my @comlines = ();
	foreach (keys %gasses)
	{
	    $gasses{$_} = 0;
	}
	my $idir = $ivdir.'ivdatadir-'.$i.'/';
	my $iurl = 'https://'.hostname.'/rig'.$rig->name.'/'.$rig->name.'test'.$test.'/iv/ivdatadir-'.$i.'/png/ivcurve-resistance.png';
	print "Processing i-V curve nr $i\n";
	my %resvals = ();
	my $f = SemaforeFile->new($idir.'ivdata_up');
	$f->open_read;
	my $line = $f->readline;
	$line = $f->readline unless $line;
	next unless ($line);
	my @list = split(/[\s\t]+/,$line);
	my $vc = 0;
	my $cc = 0;
	my $rc = 0;
	my $ASRc = 0;
	$resvals{'time'} = to_timestamp($list[1]);
	for (my $x = 0; $x < @list; $x+=2)
	{
	    $vc = $x+2 if ($vlabel eq $list[$x]);
	    $cc = $x+2 if ($clabel eq $list[$x]);
	    $rc = $x+2 if ('R_secant' eq $list[$x]);
	    $ASRc = $x+2 if ('ASR_corr' eq $list[$x]);
	    $resvals{'OCV'} = $list[$x+1] if ($list[$x] eq 'OCV_measure');
	    $resvals{'OCV_t'} = $list[$x+1] if ($list[$x] eq 'OCV_theory');
	    $resvals{'O2_in'} = $list[$x+1] if ($list[$x] eq 'O2_in');
	    $resvals{'O2_out'} = $list[$x+1] if ($list[$x] eq 'O2_out');
	    $resvals{'TEMP'} = $list[$x+1] if ($list[$x] eq 'T_OCV_measure');
	    $resvals{'TEMP2'} = $list[$x+1] if ($list[$x] eq $temp2label);
	    $resvals{'WATER'} = $list[$x+1] if ($list[$x] eq 'bottle_temp');
	    $resvals{'WATER_c'} = $list[$x+1] if ($list[$x] eq 'cathode_gas');
	    $resvals{'WATER_ce'} = $list[$x+1] if ($list[$x] eq 'cathode_gas_enabled');
	    $resvals{'LEAK'} = $list[$x+1] if ($list[$x] eq 'leak_current');
	    $resvals{'bottle_temp_enabled'} = $list[$x+1] if ($list[$x] eq 'bottle_temp_enabled');
	    $resvals{'XH2O_theory'} = $list[$x+1] if ($list[$x] eq 'xH2O_in_theory');
	    $resvals{'XH2O_measure'} = $list[$x+1] if ($list[$x] eq 'xH2O_in_measure');
	    $resvals{'XH2O_out'} = $list[$x+1] if ($list[$x] eq 'xH2O_out');
	  INNER:
	    foreach my $k (keys %gasses)
	    {
		next INNER unless ($k eq $list[$x]);
		my $val = $list[$x+1]+0;
		my $cutoff = $rig->get_cv_test('gas_'.$k,'cutoff_report',$test);
		$cutoff = AFM::get_config_value_old($rig->name,$k.'_cutoff',$test) unless (defined($cutoff));
		$cutoff = 0 unless ($cutoff);
		$val = 0 if ($cutoff && $cutoff > $val);
		$gasses{$k} = $val;
		last INNER;
	    }
	}
	unless ($resvals{'LEAK'})
	{
	    ## Fix leak curent ofr old tests which did not include it.... ###
	    foreach ('h2','o2','air','n2_cathode','o2_cathode','n2','co','co2','ch4')
	    {
		$gasses{$_} = 0 unless ($gasses{$_});
	    }
	    my $cmd = '/usr/local/bin/OCV_corr --command leak --H2 '.$gasses{'h2'}.' --O2_add '.$gasses{'o2'}. ' --Air '.$gasses{'air'} .' --O2_cathode '.$gasses{'o2_cathode'} .' --N2_cathode '.$gasses{'n2_cathode'}.' --cell '.$resvals{'TEMP'}.' --emf '.$resvals{'OCV'}.' --I 0 ';
	    my $use_ch4 = 0;
	    $cmd .= ' --water '.$resvals{'WATER'};
	    foreach ('ch4','co','co2','n2')
	    {
		$use_ch4 = 1 if ($gasses{$_});
	    }
	    if ($use_ch4)
	    {
		$cmd .= ' --use_ch4 YES --CH4 '.$gasses{'ch4'}.' --CO '.$gasses{'co'}.' --CO2 '.$gasses{'co2'}.' --N2 '.$gasses{'n2'};
	    }
	    $cmd .= ' --dry 1' if (defined($resvals{'bottle_temp_enabled'}) && $resvals{'bottle_temp_enabled'} == 0);
	    $resvals{'LEAK'} = `$cmd`;
	    chomp $resvals{'LEAK'};
	}
	my $cmd = "/usr/bin/awk '{print \$$vc,\"\\t\",\$$cc}' < ".$f->filename.' | sort -g > '.$tmpfc;
	system($cmd);
	$cmd = "/usr/bin/awk '{if (\$$rc > 0) {print \$$vc,\"\\t\",\$$rc}}' < ".$f->filename.' | /bin/sort -g > '.$tmpf;
	system($cmd);
	open MYIN,'<',$tmpf;
	my @lines = <MYIN>;
	close MYIN;
	$lines[0] =~ /([\d\.\+\-]+)/;
	my $min = $1;
	$lines[$#lines] =~ /([\d\.\+\-]+)/;
	my $max = $1;
	foreach my $level (@vlevels)
	{
	    if ($min < $level && $max > $level)
	    {
		$cmd = $spline.' '.$tmpfc.' '.$level;
		my $i =  `$cmd` + 0;
		next unless ($i > $epsilon || $i < -$epsilon);
		$cmd = $spline.' '.$tmpf.' '.$level;
		$resvals{'r_sec_'.$level} = `$cmd`+0;
	    }
	}
	$cmd = "/usr/bin/awk '{if (\$$ASRc > 0) {print \$$vc,\"\\t\",\$$ASRc}}' < ".$f->filename.' | /bin/sort -g > '.$tmpf;
	system($cmd);
	open MYIN,'<',$tmpf;
	@lines = <MYIN>;
	close MYIN;
	$lines[0] =~ /([\d\.\+\-]+)/;
	$min = $1;
	$lines[$#lines] =~ /([\d\.\+\-]+)/;
	$max = $1;
	foreach my $level (@vlevels)
	{
	    if ($min < $level && $max > $level)
	    {
		$cmd = $spline.' '.$tmpfc.' '.$level;
		my $i =  `$cmd` + 0;
		next unless ($i > $epsilon || $i < -$epsilon);
		$cmd = $spline.' '.$tmpf.' '.$level;
		$resvals{'r_corr_'.$level} = `$cmd`+0;
	    }
	}
	$cmd = "/usr/bin/awk '{if (\$$rc > 0 && (\$$cc > $epsilon || \$$cc < -$epsilon)) {print \$$cc/$area,\"\\t\",\$$rc}}' < ".$f->filename.' | /bin/sort -g > '.$tmpf;
	system($cmd);
	open MYIN,'<',$tmpf;
	@lines = <MYIN>;
	close MYIN;
	$lines[0] =~ /([\d\.\+\-]+)/;
	$min = $1;
	$lines[$#lines] =~ /([\d\.\+\-]+)/;
	$max = $1;
	foreach my $level (@ilevels)
	{
	    if ($min < $level && $max > $level)
	    {
		$cmd = $spline.' '.$tmpf.' '.$level;
		$resvals{'r_sec_I_'.$level} = `$cmd`+0;
	    }
	}
	my $imax = $max * $area;
	my $imin = $min * $area;
	$imax = $imin if (abs($imin) > $imax);
	$cmd = "/usr/bin/awk '{if (\$$ASRc > 0 && (\$$cc > $epsilon || \$$cc < -$epsilon)) {print \$$cc/$area,\"\\t\",\$$ASRc}}' < ".$f->filename.' | /bin/sort -g > '.$tmpf;
	system($cmd);
	open MYIN,'<',$tmpf;
	@lines = <MYIN>;
	close MYIN;
	$lines[0] =~ /([\d\.\+\-]+)/;
	$min = $1;
	$lines[$#lines] =~ /([\d\.\+\-]+)/;
	$max = $1;
	foreach my $level (@ilevels)
	{
	    if ($min < $level && $max > $level)
	    {
		$cmd = $spline.' '.$tmpf.' '.$level;
		$resvals{'r_corr_I_'.$level} = `$cmd`+0;
	    }
	}
	### Get end time of i-V curve
	$cmd = "/usr/bin/awk '{print \$2}' < ".$idir.'ivdata | /bin/sort -g -r | /usr/bin/head -1';
	my $end_date = to_timestamp(`$cmd`);
	### To do, database insert ###
	my $temp2 = undef;
	$temp2 = sprintf("%i",$resvals{'TEMP2'}) if (defined($resvals{'TEMP2'}));
	### Fix for those tests where the water_butler enabled status is not logged, in that case it is assumed to be enabled ###
	if (defined($resvals{'WATER'}) && !defined($resvals{'bottle_temp_enabled'}))
	{
	    $resvals{'bottle_temp_enabled'} = 1;
	    push @comlines,"Assuming water bubler on anode gas is enabled as it is not logged";
	}
	if (defined($resvals{'WATER_c'}) && !defined($resvals{'WATER_ce'}))
	{
	    $resvals{'WATER_ce'} = 1;
	    push @comlines,"Assuming water bubler on cathode gas is enabled as it is not logged";
	}
	foreach ('O2_in','O2_out')
	{
	    next unless (exists($resvals{$_}) && defined($resvals{$_}));
	    my $v = $resvals{$_};
	    $v = undef unless ($v < 100000 && $v > -100000);
	    $resvals{$_} = $v;
	}
	foreach ('OCV','OCV_t')
	{
	    $resvals{$_} = -32768 unless defined($resvals{$_});
            $resvals{$_} = -32768 if ($resvals{$_} =~ /\-\-32/);
	}
        foreach ('LEAK')
        {
            $resvals{$_} = 0 if ($resvals{$_} < 1e-20);
        }
	my $comments = undef;
	$comments = join("\n",@comlines) if (@comlines);
	$iv_ins->execute($testid,$i,$resvals{'time'},$end_date,sprintf("%i",$resvals{'TEMP'}),$temp2,$resvals{'WATER'},$resvals{'bottle_temp_enabled'},$resvals{'WATER_c'},$resvals{'WATER_ce'},$resvals{'LEAK'}*1000/$area,$resvals{'OCV'},$resvals{'OCV_t'},$resvals{'O2_in'},$resvals{'O2_out'},$imax,$comments,$iurl);
	$get_iv_nr->execute();
	my $ivid = $get_iv_nr->fetchrow_array();
	$gasses{'fuel_total'} = 0;
	### Note twice ch4 to account for ch4 + h2o => 2 h2 +co ##
	foreach ('h2','co','co2','n2','ar','ch4','ch4')
	{
	    $gasses{'fuel_total'} += $gasses{$_} if (exists($gasses{$_}) && defined($gasses{$_}));
	}
	$gasses{'oxygen_total'} = 0;
	foreach ('air','o2_cathode','n2_cathode')
	{
	    $gasses{'oxygen_total'} += $gasses{$_} if (exists($gasses{$_}) && defined($gasses{$_}));
	}
	$gasses{'fuel_total'} += $gasses{'h2'} / (1-&pH2O($resvals{'WATER'})) if ($resvals{'bottle_temp_enabled'} && exists($gasses{'h2'}) && defined($gasses{'h2'}));
	foreach my $key (keys %gasses)
	{
	    $gasses{$key} = -32768 if ($gasses{$key} > 1e10);
	    $iv_gas_ins->execute($ivid,$key,$gasses{$key});
	}
	foreach my $level (@ilevels)
	{
	    my $sec = undef;
	    my $fu = undef;
	    $sec = $resvals{'r_sec_I_'.$level} if defined($resvals{'r_sec_I_'.$level});
	    $fu = $resvals{'r_corr_I_'.$level} if defined($resvals{'r_corr_I_'.$level});
	    $iv_ins_res->execute($ivid,'I',$level,$sec,$fu) if (defined($sec) || defined($fu));
	}
	foreach my $level (@vlevels)
	{
	    my $sec = undef;
	    my $fu = undef;
	    $sec = $resvals{'r_sec_'.$level} if defined($resvals{'r_sec_'.$level});
	    $fu = $resvals{'r_corr_'.$level} if defined($resvals{'r_corr_'.$level});
	    $iv_ins_res->execute($ivid,'U',($level/1000),$sec,$fu) if (defined($sec) || defined($fu));
	}
    }
    ### Load impedance data ###
    my $impdata = SemaforeFile->new($impdir.'imp_data');
    if (-e $impdata->filename)
    {
	my %imp = ();
	my $linenr = 1;
	foreach my $line ($impdata->readlines)
	{
	    next unless ($line =~ /\w+/);
	    $line =~ s/^[\s\t]+//;
	    my @list = split(/\s+/,$line);
	    my $ifile = shift(@list);
	    $imp{$ifile} = {};
	    foreach my $item (@list)
	    {
		my ($key,$val) = split(/:/,$item);
		$imp{$ifile}{$key} = $val;
	    }
	    $imp{$ifile}{'linenr'} = $linenr++;
	}

	my $impjdata = SemaforeFile->new($impdir.'imp_jdata');
	if (-e $impjdata->filename)
	{
	    my $impsth = $dbh->prepare('INSERT INTO impedance (testid,impnr,filename,date,temperature,temperature_2,bottle_temp_anode,bottle_temp_anode_enabled,bottle_temp_cathode,bottle_temp_cathode_enabled,leak_current,voltage,current,O2_in,O2_out,OCV_theory,xH2O_theory,xH2O_out,rs,rp,comments,url) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
	    my $getimpnr = $dbh->prepare('SELECT last_value FROM impedance_id_seq');
	    my $impgas = $dbh->prepare('INSERT INTO impgasflow (impid,gasname,flow) VALUES (?,?,?)');
	    my @implines = $impjdata->readlines;
	    my $linenr = 0;
	    foreach my $line (@implines)
	    {
		my @comlines = ();
		$linenr++;
		$line =~ s/^[\s\t]+//g;
		foreach (keys %gasses)
		{
		    $gasses{$_} = 0;
		}
		my @list = split(/[\s\t]+/,$line);
		my $file = shift(@list);
		my $iurl = undef;
		if ($file =~ /(.*)\.\w+$/)
		{
		    $iurl = 'https://'.hostname.'/rig'.$rig->name.'/'.$rig->name.'test'.$test.'/impedance/png/'.$1.'-impedance.png';
		}
		print "Processing file $file\n";
		my %resvals = ();
		$resvals{'time'} = to_timestamp($list[1]);
		for (my $x = 0; $x < @list; $x+=2)
		{
		    $resvals{'voltage'} = $list[$x+1]/1000 if ($list[$x] eq $vlabel);
		    $resvals{'O2_in'} = $list[$x+1]/1000 if ($list[$x] eq 'O2_in');
		    $resvals{'WATER'} = $list[$x+1] if ($list[$x] eq 'bottle_temp');
		    $resvals{'O2_out'} = $list[$x+1]/1000 if ($list[$x] eq 'O2_out');
		    $resvals{'temp'} = $list[$x+1] if ($list[$x] eq $templabel);
		    $resvals{'temp2'} = $list[$x+1] if ($list[$x] eq $temp2label);
		    $resvals{'current'} = $list[$x+1] if ($list[$x] eq $clabel);
		    $resvals{'WATER_c'} = $list[$x+1] if ($list[$x] eq 'cathode_gas');
		    $resvals{'WATER_ce'} = $list[$x+1] if ($list[$x] eq 'cathode_gas_enabled');
		    $resvals{'bottle_temp_enabled'} = $list[$x+1] if ($list[$x] eq 'bottle_temp_enabled');
		  INNER:
		    foreach my $k (keys %gasses)
		    {
			next INNER unless ($k eq $list[$x]);
			my $val = $list[$x+1]+0;
			my $cutoff = $rig->get_cv_test('gas_'.$k,'cutoff_report',$test);
			$cutoff = $rig->get_cv_test('gas_'.$k,'cutoff_set',$test) unless ($cutoff);
			$val = 0 if ($cutoff && $cutoff > $val);
			$gasses{$k} = $val;
			last INNER;
		    }
		}
		$resvals{'current'} = 0 unless ($resvals{'current'} < -$epsilon || $resvals{'current'} > $epsilon);
		### Calcualte XH20 ###
		foreach ('h2','o2','air','n2_cathode','o2_cathode','n2','ch4','co','co2','n2')
		{
		    $gasses{$_} = 0 unless defined($gasses{$_});
		}
		my $emf = -$resvals{'voltage'};
		$emf = 0 if ($resvals{'current'});
		my $cmd = '/usr/local/bin/OCV_corr --command par_all';
		$cmd .= ' --water '.$resvals{'WATER'};
		$cmd .= ' --emf '.$emf if ($emf);
		$cmd .= ' --I '.$resvals{'current'}.' --cell '.$resvals{'temp'};
		$cmd .= ' --H2 '.$gasses{'h2'}. ' --O2_add '.$gasses{'o2'}.' --Air '.$gasses{'air'}.' --O2_cathode '.$gasses{'o2_cathode'}.' --N2_cathode '.$gasses{'n2_cathode'};
		my $use_ch4 = 0;
		foreach ('ch4','co','co2')
		{
		    $use_ch4 = 1 if ($gasses{$_});
		}
		if ($use_ch4)
		{
		    $cmd .= ' --use_ch4 YES --CH4 '.$gasses{'ch4'}.' --CO '.$gasses{'co'}.' --CO2 '.$gasses{'co2'};
		}
		$cmd .= ' --dry 1' if (defined($resvals{'bottle_temp_enabled'}) && $resvals{'bottle_temp_enabled'} == 0);
		my @res = `$cmd`;
		foreach my $l (@res)
		{
		    chomp $l;
		    my ($key,$val) = split(/[\s\t]+/,$l);
		    $resvals{'leak'} = $val if ($key =~ /leak_current/);
		    $resvals{'XH2O_theory'} = $val if ($key =~ /xH2O_in_the/);
		    $resvals{'OCV_theory'} = $val if ($key =~ /OCV_theo/);
		    $resvals{'XH2O_out'} = $val if ($key =~ /xH2O_out/);
		}
		$resvals{'rs'} = $imp{$file}{'Rs'} * $area;
		$resvals{'rp'} = $imp{$file}{'Rp'} * $area;
		$resvals{'leak'} = undef if ($resvals{'current'} < -$epsilon || $resvals{'current'} > $epsilon);
		my $temp2 = undef;
		$temp2 = sprintf("%i",$resvals{'temp2'}) if (defined($resvals{'temp2'}));
		### Fix for those tests where the water_butler enabled status is not logged, in that case it is assumed to be enabled ###
		if (defined($resvals{'WATER'}) && !defined($resvals{'bottle_temp_enabled'}))
		{
		    $resvals{'bottle_temp_enabled'} = 1;
		    push @comlines,"Assuming water bubler on anode gas is enabled as it is not logged";
		}
		if (defined($resvals{'WATER_c'}) && !defined($resvals{'WATER_ce'}))
		{
		    $resvals{'WATER_ce'} = 1;
		    push @comlines,"Assuming water bubler on cathode gas is enabled as it is not logged";
		}
		my $comments = undef;
		$comments = join("\n",@comlines) if (@comlines);
		$impsth->execute($testid,$linenr,$file,$resvals{'time'},sprintf("%i",$resvals{'temp'}),$temp2,$resvals{'WATER'},$resvals{'bottle_temp_enabled'},$resvals{'WATER_c'},$resvals{'WATER_ce'},$resvals{'leak'},$resvals{'voltage'},$resvals{'current'},$resvals{'O2_in'},$resvals{'O2_out'},$resvals{'OCV_theory'},$resvals{'XH2O_theory'},$resvals{'XH2O_out'},$resvals{'rs'},$resvals{'rp'},$comments,$iurl);
		$getimpnr->execute;
		my $impid = $getimpnr->fetchrow_array;
		foreach my $key (keys %gasses)
		{
		    $impgas->execute($impid,$key,$gasses{$key});
		}
	    }
	}
	else
	{
	    my $impsth = $dbh->prepare('INSERT INTO impedance (testid,impnr,filename,date,temperature,current,rs,rp,comments,url) VALUES (?,?,?,?,?,?,?,?,?,?)');
	    my $getimpnr = $dbh->prepare('SELECT last_value FROM impedance_id_seq');
	    my $impgas = $dbh->prepare('INSERT INTO impgasflow (impid,gasname,flow) VALUES (?,?,?)');
	    foreach my $file (keys %imp)
	    {
		my @comlines = ();
		print "Processing file $file\n";
		my $iurl = undef;
		if ($file =~ /(.*)\.\w+$/)
		{
		    $iurl = 'https://'.hostname.'/rig'.$rig->name.'/'.$rig->name.'test'.$test.'/impedance/png/'.$1.'-impedance.png';
		}
		my $time = undef;
		my $f = SemaforeFile->new($impdir.$file);
		my @lines = $f->readlines();
	      INNER:
		foreach (@lines)
		{
		    next unless /^\s*\:\d+/;
		    /^\s*\:([\d\:]+)/;
		    $time = to_timestamp($1);
		    last INNER;
		}
		foreach ('h2','o2','air','ch4','Rs','Rp','current','T_center')
		{
		    ### Make sure the values exists by autovivification ##
		    $imp{$file}{$_} = undef unless (defined($imp{$file}{$_}));
		}
		my $h2 = $imp{$file}{'h2'};
		my $o2 = $imp{$file}{'o2'};
		my $air = $imp{$file}{'air'};
		my $ch4 = $imp{$file}{'ch4'};
		my $cutoff = $rig->get_cv_test('gas_h2','cutoff_report',$test);
		$cutoff = AFM::get_config_value_old($rig->name,'h2_cutoff',$test) unless ($cutoff);
		$cutoff = 0 unless ($cutoff);
		$h2 = 0 if (!defined($h2) || $h2 < $cutoff);
		$cutoff = $rig->get_cv_test('gas_o2','cutoff_report',$test);
		$cutoff = AFM::get_config_value_old($rig->name,'o2_cutoff',$test) unless ($cutoff);
		$cutoff = 0 unless ($cutoff);
		$o2 = 0 if (!defined($o2) || $o2 < $cutoff);
		$cutoff = $rig->get_cv_test('gas_air','cutoff_report',$test);
		$cutoff = AFM::get_config_value_old($rig->name,'air_cutoff',$test) unless ($cutoff);
		$cutoff = 0 unless ($cutoff);
		$air = 0 if (!defined($air) || $air < $cutoff);
		$cutoff = $rig->get_cv_test('gas_ch4','cutoff_report',$test);
		$cutoff = AFM::get_config_value_old($rig->name,'ch4_cutoff',$test) unless ($cutoff);
		$cutoff = 0 unless ($cutoff);
		$ch4 = 0 if (!defined($ch4) || $ch4 < $cutoff);
		my $rs = $imp{$file}{'Rs'} * $area;
		my $rp = $imp{$file}{'Rp'} * $area;
		my $c = $imp{$file}{'Current'};
		my $t = sprintf("%i",$imp{$file}{'T_center'});
		my $comments = undef;
		$time = '1970-01-01' unless ($time);
		$comments = join("\n",@comlines) if (@comlines);
		$impsth->execute($testid,$imp{$file}{'linenr'},$file,$time,$t,$c,$rs,$rp,$comments,$iurl);
		$getimpnr->execute;
		my $impid = $getimpnr->fetchrow_array;
		$impgas->execute($impid,'h2',$h2);
		$impgas->execute($impid,'ch4',$h2);
		$impgas->execute($impid,'o2',$o2);
		$impgas->execute($impid,'air',$air);
	    }
	}
    }
    $dbh->commit();
};
if ($@)
{
    print $@;
    $dbh->rollback();
}
$dbh->disconnect();

unlink $tmpf if (-e $tmpf);
unlink $tmpfc if (-e $tmpfc);



sub to_timestamp
{
    my $string = shift;
    return undef unless ($string);
    my @t = split(/:/,$string);
    return undef unless (@t > 2);
    return join('-',splice(@t,0,3)).' '.join(':',splice(@t,0,3)) if (@t > 5);
    return join('-',splice(@t,0,3));
}


sub get_cn_args()
{
    my $string = 'dbi:Pg:dbname=Labsystem';
    $string = 'dbi:Pg:dbname='.$_[0] if (@_);
    my $host = $host;
    $string .= ';host='.$host if (length($host) > 2);
    my $user = 'sofc';
    my $pass = '';
    return ($string,$user,$pass);
}


sub to_utf8
{
    my $text = shift;
    return $text unless defined($test);
    # if this host was UTF-8 encoded:
    my $text_iso  = (utf8($text))->latin1;
    my $text_utf8 = (latin1($text_iso))->utf8; # reverse check
    if ($text ne $text_utf8) {
        #print "Unequal reverse check! It seems your input data \"$text\" is ISO encoded already, so you don't need the latin1 encoding stuff here!\n";
        # I'm going to fix this
        return (latin1($text))->utf8;
    }
    return $text;
}



sub pH2O($)
{
    my $watertemp = shift;
# Coefficients for the water partial pressure are obtained by fitting a 6-
# power function to data from 'Databog Fysik Kemi', F og K forlaget 10.udgave
    my @a;
    $a[6] = 6.3096E-11;
    $a[5] = 2.5259E-6;
    $a[4] = -3.2193E-3;
    $a[3] = 1.6322;
    $a[2] = -414.9;
    $a[1] = 52920;
    $a[0] = -2.7085E6;
# End ref
    our $ph2o = 0;
    for (my $x=0;$x<7;$x++)
    {
	my $test = $watertemp;
	for (my $y = 1;$y<$x;$y++)
	{
	    $test = $test * $watertemp;
	}
	if ($x==0){$test = 1;};
	$ph2o += $a[$x]*$test;
    }
    $ph2o = $ph2o/100000;
    return $ph2o;
}
