#!/usr/bin/perl -w

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

=pod

=head1 NAME

tv_grab_huro - Grab TV listings for Hungary or Romania.

=head1 SYNOPSIS

tv_grab_huro --help

tv_grab_huro [--config-file FILE] --configure [--gui OPTION]

tv_grab_huro [--config-file FILE] [--output FILE] [--days N]
           [--offset N] [--quiet]
           [--slow] [--get-full-description]

tv_grab_huro --list-channels --loc [hu | ro]

=head1 DESCRIPTION

Output TV listings for several channels available in Hungary or
Romania.  The grabber relies on parsing HTML so it might stop working
at any time.

First run B<tv_grab_huro --configure> to choose, which channels you want
to download. Then running B<tv_grab_huro> with no arguments will output
listings in XML format to standard output.

B<--configure> Prompt for which channels,
and write the configuration file.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_huro.conf>.  This is the file written by
B<--configure> and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar.

B<--output FILE> write to FILE rather than standard output.

B<--days N> grab N days.  The default is eight.

B<--offset N> start N days in the future.  The default is to start
from today.

B<--quiet> suppress the progress messages normally written to standard
error.

B<--slow> enables long strategy run: port.hu publishes only some (vital)
information on the actual listing pages, the rest is shown in a separate
popup window. If you'd like to parse the data from these popups as well,
supply this flag. But consider that the grab process takes much longer when
doing so, since many more web pages have to be retrieved.

B<--get-full-description> is quite like B<--slow> but doesn't always download
data from popup window.  Instead this is only requested if description in
overview is truncated.

B<--list-channels> write output giving <channel> elements for every
channel available (ignoring the config file), but no programmes.

B<--help> print a help message and exit.

=head1 SEE ALSO

L<xmltv(5)>.

=head1 AUTHOR

Attila Szekeres and Zsolt Varga.  Based on tv_grab_fi by Matti Airas.
Heavily patched and now maintained by Stefan siegl <stesie@brokenpipe.de>.

=head1 BUGS

The data source does not include full channels information and the
channels are identified by short names rather than the RFC2838 form
recommended by the XMLTV DTD.

=cut

######################################################################
# initializations

use strict;
use XMLTV::Version '$Id: tv_grab_huro.in,v 1.2 2006/01/08 10:55:02 epaepa Exp $ ';
use Getopt::Long;
use Date::Manip;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
use IO::File;

use XMLTV;
use XMLTV::Memoize;
use XMLTV::ProgressBar;
use XMLTV::Ask;
use XMLTV::DST;
use XMLTV::Get_nice;
use XMLTV::Mode;
use XMLTV::Config_file;
use XMLTV::Date;
# Todo: perhaps we should internationalize messages and docs?
use XMLTV::Usage <<END
$0: get Hungarian or Romanian television listings in XMLTV format
To configure: $0 --configure [--config-file FILE] [--gui OPTION]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--offset N] [--quiet] [--slow] [--get-full-description]
To list channels: $0 --list-channels --loc [hu | ro]
END
  ;

sub domain( $ );
sub xhead( $ );
sub xid( $$ );
sub process_table( $$$$$ );
sub make_programme_hash( $$$$$$ );
sub bump_start_day( $$ );
sub get_program_data( @ );
sub get_time( @ );
sub get_title( @ );
sub get_desc( @ );
sub get_channels( $ );
sub get_infourl_data( $$ );
sub nextday( $ );

# Attributes of the root element in output.
sub domain( $ ) { "port.$_[0]" };
sub xhead( $ ) {
    my $d = &domain;
    return { 'source-info-url'     => "http://www.$d/",
	     'source-data-url'     => "http://www.$d/tv/",
	     'generator-info-name' => 'XMLTV',
	     'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
	   };
}

my %COUNTRIES = (Hungary => [ 'hu', '+0100' ],
		 Romania => [ 'ro', '+0200' ],
		);

# Whether zero-length programmes should be included in the output.
my $WRITE_ZERO_LENGTH = 0;

# Global channel data.
our @ch_all;

######################################################################
# get options

# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
my ($opt_days, $opt_offset, $opt_help, $opt_output, $opt_share,
    $opt_configure, $opt_config_file, $opt_gui, $opt_quiet,
    $opt_list_channels, $opt_loc, $opt_slow, $opt_full_desc);
$opt_slow = 0;
$opt_full_desc = 0;
$opt_days = 8; # default
$opt_offset = 0; # default
$opt_quiet = 0; # default
GetOptions('days=i'        => \$opt_days,
	   'offset=i'      => \$opt_offset,
	   'help'          => \$opt_help,
	   'configure'     => \$opt_configure,
       'gui:s'         => \$opt_gui,
       'config-file=s' => \$opt_config_file,
	   'output=s'      => \$opt_output,
	   'quiet'         => \$opt_quiet,
	   'slow'          => \$opt_slow,
	   'get-full-description' => \$opt_full_desc,
	   'list-channels' => \$opt_list_channels,
	   'loc=s'         => \$opt_loc,
	   'share=s'       => \$opt_share,
	  )
  or usage(0);
die 'number of days must not be negative'
  if (defined $opt_days && $opt_days < 0);
usage(1) if $opt_help;
my $mode = XMLTV::Mode::mode('grab', # default
			     $opt_configure => 'configure',
			     $opt_list_channels => 'list-channels',
			    );

XMLTV::Ask::init($opt_gui);
# File that stores which channels to download.
my $config_file
  = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_huro', $opt_quiet, 'tv_grab_hu');

my @config_lines; # used only in grab mode
if ($mode eq 'configure') {
    XMLTV::Config_file::check_no_overwrite($config_file);
}
elsif ($mode eq 'grab') {
    @config_lines = XMLTV::Config_file::read_lines($config_file);
}
elsif ($mode eq 'list-channels') {
    # Config file not used.
}
else { die }

######################################################################
# write configuration

if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die "cannot write to $config_file: $!";

    my $default_cn = 'Hungary';
    my $cn = ask_choice('Grab listings for which country?',
			 $default_cn,
			 sort keys %COUNTRIES);
    my $c = $COUNTRIES{$cn}[0];
    print CONF "country $c\t# $cn\n";

    # Ask about each channel.
    my %channels = get_channels($c);
    my @chs = sort keys %channels;
    my @names = map { $channels{$_} } @chs;
    my @qs = map { "add channel $_?" } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@chs) {
	my $w = shift @want;
	warn("cannot read input, stopping channel questions"), last
	  if not defined $w;
	# No need to print to user - XMLTV::Ask is verbose enough.

	# Print a config line, but comment it out if channel not wanted.
	print CONF '#' if not $w;
	my $name = shift @names;
	print CONF "channel $_ $name\n";
	# TODO don't store display-name in config file.
    }

    close CONF or warn "cannot close $config_file: $!";
    say("Finished configuration.");

    exit();
}

# Not configuration, we must be writing something, either full
# listings or just channels.
#
die if $mode ne 'grab' and $mode ne 'list-channels';

# Options to be used for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    my $fh = new IO::File(">$opt_output");
    die "cannot write to $opt_output: $!" if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'ISO-8859-2';

if ($mode eq 'list-channels') {
    # Write channels mode.
    if (not defined $opt_loc) {
	my $msg = "--loc option required with --list-channels:\n";
	foreach (sort keys %COUNTRIES) {
	    $msg .= "    --loc $COUNTRIES{$_}[0] for $_\n";
	}
	die $msg;
    }
    my $writer = new XMLTV::Writer(%w_args);
    $writer->start(xhead($opt_loc));
    get_channels($opt_loc); # sets @ch_all
    $writer->write_channel($_) foreach @ch_all;
    $writer->end();
    exit();
}

######################################################################
# We are producing full listings.
die if $mode ne 'grab';

# Read configuration
my $line_num = 0;
my ($country, $tz);
my @channels;
foreach (@config_lines) {
    ++ $line_num;
    next if not defined;
    my $where = "$config_file:$line_num";
    if (/^country:?\s+(\w\w)$/) {
	warn "$where: already seen country\n"
	  if defined $country;
	$country = $1;

	# Lame reverse lookup on %COUNTRIES.
	foreach (values %COUNTRIES) {
	    if ($_->[0] eq $country) {
		$tz = $_->[1];
		last;
	    }
	}
	die "$where: unknown country $country\n" if not defined $tz;
    }
    elsif (/^channel:?\s+(\S+)\s+([^\#]+)/) {
	my $ch_did = $1;
	my $ch_name = $2;
	$ch_name =~ s/\s*$//;
	push @channels, $ch_did;
	# FIXME do not store display-name in the config file - it is
	# ignored here.
	#
    }
    else {
	warn "$config_file:$.: bad line\n";
    }
}
for ($country) {
    if (not defined) {
	$_ = 'hu';
	warn "country not seen in $config_file, assuming '$_'\n";
    }
}

######################################################################
# read jobmap file
# (this is a file, where we store translations of job names from 
#  Hungarian or Romanian language to English.  However we leave some
#  translations blank, namely these that have no field in the credits
#  structure)

# share/ directory for storing channel mapping files.  This next line
# is altered by processing through tv_grab_huro.PL.  But we can
# use the current directory instead of share/tv_grab_huro for
# development.
#
# The 'source' file tv_grab_huro.in has $SHARE_DIR undef, which
# means use the current directory.  In any case the directory can be
# overridden with the --share option (useful for testing).
#
my $SHARE_DIR='/var/tmp/xmltv-0.5.42-4m.mo5-root-builduser/usr/share/xmltv'; # by grab/huro/tv_grab_huro.PL
$SHARE_DIR = $opt_share if defined $opt_share;
my $OUR_SHARE_DIR = (defined $SHARE_DIR) ? "$SHARE_DIR/tv_grab_huro" : '.';

# Read the file with channel mappings.
(my $JOBMAP_FILE = "$OUR_SHARE_DIR/jobmap") =~ tr#/#/#s;
my %jobmap;
$line_num = 0;
foreach (XMLTV::Config_file::read_lines($JOBMAP_FILE, 1)) {
    ++ $line_num;
    next unless defined;
    my $where = "$JOBMAP_FILE:$line_num";
    my @fields = split m/:/;
    die "$where: wrong number of fields"
      if @fields > 2;

    my ($huro_job, $credits_id) = @fields;
    $jobmap{$huro_job} = defined($credits_id) ? $credits_id : "";
}



######################################################################
# begin main program

my $now = DateCalc(parse_date('now'), "$opt_offset days");
my %channels = get_channels($country); # sets @ch_all
my @to_get;
my $writer = new XMLTV::Writer(%w_args);
$writer->start(xhead($country));

# Turn a site channel id into an XMLTV id.
# Parameters: country, site id
sub xid( $$ ) {
    my ($country, $id) = @_;
    return "$id." . domain($country);
}

# Write channel elements
foreach my $ch_did (@channels) {
    my $ch_name=$channels{$ch_did};
    $writer->write_channel({ id => xid($country, $ch_did),
			     'display-name' => [ [ $ch_name ] ] });
}

# The grabber's source allows requests of more than one day per page. This can
# be done by specifying the i_xday argument with the GET request.
#
# To not load their server too much (requesting e.g. 14 channels in one shot
# should 'cause quite some traffic to the SQL server) I think we shouldn't 
# query for more then 5 channels per page. With the default of requesting data
# for 8 days this leads to 2 requests per channel and grab ...
my $daysperpage = int($opt_days / 5) + (($opt_days % 5) ? 1 : 0);
$daysperpage = int($opt_days / $daysperpage);
# print STDERR "requesting $daysperpage days per scraped webpage ...\n"; exit;

# Make list of pages to fetch for each day.
my @days;
my $day=UnixDate($now,'%Q');
for (my $i=1+$opt_offset;$i<$opt_days+$opt_offset+1;$i+=$daysperpage) {
    push @days, [ $day, $i ];
    $day=nextday($day); die if not defined $day;
}

# This progress bar is for both downloading and parsing.  Maybe
# they could be separate stages.
#
my $bar = new XMLTV::ProgressBar('getting listings', @days * @channels)
  if not $opt_quiet;
foreach my $d (@days) {
    my ($day, $i) = @$d;
    my $some_success = 0;
    foreach my $ch_did (@channels) {
	my @ps = process_table($country, $day, xid($country, $ch_did), $ch_did, $i);
	$some_success = 1 if @ps;
	$writer->write_programme($_) foreach @ps;
	update $bar if not $opt_quiet;
    }
    if (@channels and not $some_success) {
	warn "failed to get any listings for day $i, stopping\n";
	last;
    }
}
$bar->finish() if not $opt_quiet;
$writer->end();

######################################################################
# subroutine definitions

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
	*t = sub {};
	*d = sub { '' };
    }
    else {
	*t = \&Log::TraceMessages::t;
	*d = \&Log::TraceMessages::d;
	Log::TraceMessages::check_argv();
    }
}

####
# process_table: fetch a URL and process it
#
# arguments:
#    country id (hu, ro)
#    Date::Manip object giving the day to grab
#    xmltv id of channel
#    site id of channel
#
# returns: list of the programme hashes to write
#
sub process_table( $$$$$ ) {
    my ($country, $date, $ch_xmltv_id, $ch_port_id, $interval) = @_;

    my $d = domain($country);
    my $url = "http://www.$d/pls/tv/tv.channel?i_ch=$ch_port_id&i_days=$interval&i_xday=$daysperpage&i_where=1";
    my $data=get_nice($url);
    if (not defined $data) {
	die "could not fetch $url, aborting\n";
    }
    local $SIG{__WARN__} = sub {
	warn "$url: $_[0]";
    };

    # parse the page to a document object
    my $tree = HTML::TreeBuilder->new();
    $tree->parse($data) or die "cannot parse content of $url\n";
    $tree->eof;

    # the page consists of two major tables, where one holds the data 
    # until 'bout 20 o'clock, the other, i.e. lower, one the evening program
    my $body = $tree->look_down("_tag"=>"body");

    # first scan for all tables in $body->content_list()
    my %tables;
    my $region;
    foreach($body->content_list()) {
	next unless(ref eq "HTML::Element");

        # don't consider anything before the closing </blockquote>
	# parsable information ...
	unless(defined $region) {
	    if($_->tag eq "blockquote"

	       # port.hu unfortunately doesn't show <blockquote> below <body>
	       # but embedded in <font>    :-(
	       || $_->look_down("_tag"=>"blockquote")) {
		$region = "upper";
	    }

	    next;
	}

	# we care for tables only ...
	next unless($_->tag eq "table");

	# filter out tables without strong tags, as those don't hold data.
	next unless($_->look_down("_tag"=>"strong"));

	# input fields are bad, these aren't worth anything for grabbing ...
	next if($_->look_down("_tag"=>"input"));

	# check whether it's one of the lower tables ...
	$region = "lower"
	    if($_->look_down("_tag"=>"img", "src"=>"/tv/kep/vonal.gif"));

	push @{$tables{$region}}, $_;
    }

    # indexes, pointing into table-lists, to continue parsing after break.
    my %indexes = ("upper" => 0, "lower" => 0);

    my @program_data; # the array, we push our programmes into ..
    while(1) {
	my $lasttime = 0;

	foreach my $region ("upper", "lower") {
	    while(1) {
		my $tab = ($tables{$region})->[$indexes{$region}];
		last unless(defined($tab)); # get outta here ...

		# extract the first time specified in this table-piece ...
		$tab->as_text() =~ m/([012][0-9]):([0-5][0-9])/
		    or die "unable to parse returned html page";
		my $time = $1 * 60 + $2;
		$time += 24 * 60
		    if($time < 6 * 60 && ($lasttime >= 24 * 60
					  || $region eq 'lower'));

		last if($time < $lasttime); # must be the next column ...

		# parse this page's data ...
		push @program_data, get_program_data($tab);

		# lookup last time in this minor table ... (as base for comp.)
		$tab->as_text() =~ m/.*([012][0-9]):([0-5][0-9])/
		    or die "unable to parse returned html page";
		$lasttime = $1 * 60 + $2;
		if($lasttime < $time) { $lasttime += 24 * 60; } # bump one day.

		# continue with next table ...
		$indexes{$region} ++;
	    } 
	}

	last unless($lasttime); # we found at least some data => continue.
    }

    $tree->delete; # get rid of HTML::TreeBuilder's in memory representation

    if (not @program_data) {
	warn "no programs found, skipping\n";
	return ();
    }

    my @r;
    my $prev;
    while (@program_data) {
	my $cur = shift @program_data;

	if (defined $prev and bump_start_day($prev, $cur)) {
	    # bump start time, if necessary
	    $date = UnixDate(DateCalc($date, '+ 1 day'), '%Q');
	}

	# Assume country and lang. code the same.
	push @r, make_programme_hash($country, $tz, $date,
				     $ch_xmltv_id, $ch_port_id, $cur);

	if (defined($prev) && $prev->{q(time)} eq $cur->{q(time)}) {
	    # starttime of previous and current programme is equal,
	    # therefore use clumpidx to express relation
	    my $clumps_num;

	    if(defined($r[-2]->{q(clumpidx)})) {
		# previous programme already has a clumpidx arg assigned.
		($clumps_num) = $r[-2]->{q(clumpidx)} =~ m;^\d+/(\d+)$;;
	    } else { $clumps_num = 2; }

	    # okay, assign new clumpidx values ...
	    for(0..($clumps_num-1)) {
		$r[-$clumps_num+$_]->{q(clumpidx)} = "$_/$clumps_num";
	    }
	}

	$prev = $cur;
    }
    return @r;
}

sub make_programme_hash( $$$$$$ ) {
    my ($lang, $tz, $date, $ch_xmltv_id, $ch_port_id, $cur) = @_;

    my %prog;

    $prog{channel}=$ch_xmltv_id;
    $prog{title}=[ [ $cur->{title}, $lang ] ];

    my $start=parse_local_date("$date $cur->{time}", $tz);
    my ($start_base, $start_tz) = @{date_to_local($start, $tz)};
    $prog{start}=UnixDate($start_base, '%q') . " $start_tz";

    # FIXME: parse description field further
    $prog{desc}=[ [ $cur->{desc}{desc}, $lang ] ] if defined $cur->{desc}{desc};

    if($cur->{desc}{infourl}) {
	# always read data from linked page (in --slow mode)
	if($opt_slow) {
	    get_infourl_data(\%prog, $cur->{desc}{infourl});
	}
	
	# in --get-full-description mode read if description ends in '...'
	elsif($opt_full_desc && ($prog{desc})->[0]->[0] =~ m/\.\.\.$/) {
	    get_infourl_data(\%prog, $cur->{desc}{infourl});
	}
    }

    return \%prog;
}
sub bump_start_day( $$ ) {
    my ($cur,$next) = @_;
    if (!defined($next)) {
	return undef;
    }
    my $start = UnixDate($cur->{time},'%H:%M');
    my $stop = UnixDate($next->{time},'%H:%M');
    if (Date_Cmp($start,$stop)>0) {
	return 1;
    } else {
	return 0;
    }
}


#
# program data is split as follows:
# - data is contained within many tag nodes, in a complicated hierarchy that
#   includes several tables...,
#   however each tv program listing has also a fixed head (HORA, TIPO,
#   CANAL...) and the data follows this header in a fixed order, so...
sub get_program_data( @ ) {
    my ($tree) = @_;
    my @data;

    my @html_time = get_time($tree);
    my @html_title = get_title($tree);
    my @html_desc = get_desc($tree);

    my $index = 0;
    while (defined $html_time[$index]) {
	my %h = (time =>$html_time[$index],
		 title=>$html_title[$index],
		);
	for ($html_desc[$index]) {
	    $h{desc} = $_ if defined;
	}
	push @data, \%h;
	$index++;
    }
    return @data;
}

sub get_time( @ ) {

    my ($tree) = @_;

    my @txt_elem;
    my @txt_cont = $tree->look_down("_tag"=>"td", "align"=>"right", "valign"=>"top");
	foreach my $txt (@txt_cont) {
		$_ = $txt->as_text;
		s/^\s+//;s/\s+$//;
		s/^Kb[.]//; # means 'approx' in Magyar

		# enforce <td> to contain timestr, port.hu tends to write out
		# an empty <td> now right before each column.
		m/.*([012][0-9]):([0-5][0-9])/ or next;

		# port.hu tends to write out 24:00 o'clock, replace by 00:00
		# which is expected to mean :)
		s/^24:/00:/;

		push @txt_elem, $_;
	}
    return @txt_elem;
}

sub get_title( @ ) {

    my ($tree) = @_;

    my @txt_elem;
    my @txt_cont = $tree->look_down("_tag"=>"td", "align"=>"left", "valign"=>"top");
	foreach my $txt (@txt_cont) {
		my @fonts = $txt->find_by_tag_name("_tag"=>"font", "size"=>"2");
		my $text = $fonts[0]->as_text;
		for ($text) { s/^\s+//; s/\s+$// }
		push @txt_elem, $text unless $text eq '';
	}
	
    return @txt_elem;
}

sub get_desc( @ ) {

    my ($tree) = @_;
    
    my @txt_elem;
    my @txt_cont = $tree->look_down("_tag"=>"td", "align"=>"left", "valign"=>"top");
    foreach my $txt (@txt_cont) {
	my @alltext;
	my @fonts = $txt->content_list;
	foreach (grep { ref } @fonts) {
	    for ($_->as_text) {
		s/^\s+//;s/\s+$//;
		push @alltext, $_ if length;
	    }
	}
	my $joined;
	if (@alltext) {
	    $joined = join(".  ", @alltext);
	}

	my $infourl;
	if(my $tag = $txt->look_down("_tag"=>"a")) {
	    $infourl = $tag->attr(q(href));
	}
	push @txt_elem, { "desc" => $joined,
			  "infourl" => $infourl
			};
    }
    return @txt_elem;
}


# get channel listing for a country
sub get_channels( $ ) {
    my $country = shift;
    my $d = domain($country);
    my $bar = new XMLTV::ProgressBar('getting list of channels', 1)
	if not $opt_quiet;
    my %channels;
    my $url="http://www.$d/pls/tv/tv.prog";
    my $local_data=get_nice($url);
    die "could not get channel listing $url, aborting\n"
      if not defined $local_data;

    my $tree = HTML::TreeBuilder->new();
    $tree->parse($local_data) or die "cannot parse content of $url\n";
    $tree->eof;

    my @menus = $tree->find_by_tag_name("_tag"=>"select");

    foreach my $elem (@menus) {
	my $cname = $elem->attr('name');
	if ($cname eq "i_ch") {
	    my @ocanals = $elem->find_by_tag_name("_tag"=>"option");
	    @ocanals = sort @ocanals;
	    foreach my $opt (@ocanals) {
		if (not $opt->attr('value') eq "") {
		    my $channel_id = $opt->attr('value');
		    my $channel_name = $opt->as_text;
		    if (length $channel_id eq 1) {
			$channel_id = "00" . $channel_id
		    }
		    if (length $channel_id eq 2) {
			$channel_id = "0" . $channel_id
		    }
		    $channels{$channel_id}=$channel_name;
		    # Assume country code and lang. code the same.
		    push @ch_all, { 'display-name' => [ [ $channel_name,
							  $country ] ],
				    'id'=> "$channel_id.$d" };
		}
	    }
	}
    }
    die "no channels could be found" if not keys %channels;
    update $bar if not $opt_quiet;
    $bar->finish() if not $opt_quiet;
    return %channels;
}


# merge data from linked info page into programme hash
sub get_infourl_data( $$ ) {
    my $prog = shift;
    my $d = domain($country);
    my $url = "http://www.$d" . shift;
    #print STDERR "slow url: $url\n";

    my $voting_regexp;

    if($country eq "hu") {
	$voting_regexp = "Erre a filmre eddig";
    }
    elsif($country eq "ro") {
	$voting_regexp = "prezent acest program";
    }
    else {
	die "line not reached";
    }

    my $tree = HTML::TreeBuilder->new();
    my $data = get_nice($url);
    warn "could not fetch $url (infopage), trying to continue.\n"
	unless(defined $data);
    $tree->parse($data) or warn "cannot parse content of $url\n";
    $tree->eof;

    # get rid of existing description field
    delete($prog->{q(desc)});

    # extract long description text which is located in the first <font>
    # tag, after voting option
    my @fonts = $tree->look_down("_tag"=>"font", size=>"2");
    for(0..(scalar(@fonts)-1)) {
	next unless(ref($fonts[$_]));

	if($fonts[$_]->as_text() =~ m/$voting_regexp/o) {
	    $_ = $fonts[$_+1];

	    # join the presented text, replacing <br>s by spaces.
	    my $desc = "";
	    foreach($_->content_list()) { $desc .= ref($_) ?
					      $_->as_text() . " " : $_; }
	    for($desc) { s/^\s+//; s/\s+$//; }

	    $prog->{q(desc)} = [[ $desc, $country ]]
		if(length($desc));
	    last;
	}
    }

    # now parse credits data block
    my @tds = $tree->look_down("_tag"=>"td", "valign"=>"top", "width"=>"100%");
    my $infoline_match = 0;
    #$tds[1]->dump();
    if(my $root = $tds[1]) {
	#print STDERR "credits data block parsing entered.\n";
	#foreach($tds[1]->look_down("_tag"=>"font")) {
	foreach($tds[1]->content_list()) {
	    # there are many <strong> tags, if not, it is just not important
	    next unless($_->look_down("_tag"=>"strong"));

	    my $text = $_->as_text();
	    my($category, $foo, $episode, $length, $bar, $year, $film_country);

	    next if($text =~ m/^\s*$/); # skip empty lines ...

	    # possibility 1: amerikai filmdrma, 90 perc, 2000
	    # which is: <category>, \d+ minutes, <year>  (hungarian)
	    if(($category, $foo, $episode, $length, $bar, $year) =
	       $text =~ m/^([^0-9,;]+)?(,\s+([0-9\/]+)\. rsz)?,?\s*(\d+) perc(,\s+([12][0-9]{3}))?\s*$/o) {}

            # color means just color in Hungarian and Romanian
            # 'alb-negru' means white/black in Romanian
            #  -> Italia, 1999, serial, color
	    # 'color' and 'alb-negru' may appear for one and the same show!
	    elsif(($film_country, $year, $category, $bar, $length) =
		  $text =~ m/^([^0-9,;\:]+)?,?\s*([12][0-9]{3})?,?\s*(?:([^0-9,]*))(,\s+(\d+) minute)?(?:,\s*(?:color|alb-negru))*$/o) {
		if(defined($category) && $category eq "color") {
		    undef $category
		}
		else {
		    s/\s*$// for($category); # on port.ro there's sometimes some
					     # extra spacing after the category
		}
	    }
	    else {
		goto infoline_store_skip;
	    }

	    goto infoline_store_skip
		if($infoline_match ++); # don't match two times, ...
	    
	    $prog->{q(country)} = [[ $film_country, $country ]]
		if defined $film_country;

            $prog->{q(category)} = [[ $category, $country ]]
		if defined $category and length $category;

	    $prog->{q(length)} = $length * 60
		if defined $length;

	    $prog->{q(date)} = $year
		if defined $year ;

	    if(defined($episode)) {
		if($episode =~ m#(\d+)/(\d+)#) {
		    # episode-num spec with the total number specified.
		    # however XMLTV counts from 0 on ...
		    $prog->{q(episode-num)} = [[
			sprintf("%d/%d", $1 - 1, $2), "xmltv-ns" ]];
		}
		else {
		    $prog->{q(episode-num)} = [[ $episode, "onscreen" ]];
		}
	    }

	    next;

	  infoline_store_skip:
	    # match & warn for everything else containing perc (= minutes)
	    if($text =~ m/ (perc|minute)/ or $text =~ m/ color$/) {
		warn "infourl-reader didn't match '$text'.\n";
	    }

	    # last chance: list of credits ...
	    else {
		my %credits;
		my $job = "foobar";

		foreach($_->content_list()) {
		    next unless ref;

		    if($_->tag eq "strong") {
			($job) = $_->as_text() =~ m/^(.+):/;
		    }
		    elsif($_->tag eq "a") {
			# okay, new person ..
			if(defined($jobmap{$job})) {
			    push @{$credits{$jobmap{$job}}}, $_->as_text()
				if(length $jobmap{$job});
			} else {
			    warn "don't know what job '$job' is ...\n";
			}
		    }
		    elsif($_->tag eq "p") {
			# actors are listed in a separate paragraph 
			push @{$credits{q(actor)}}, $_->as_text()
			    foreach($_->look_down("_tag"=>"a"));
		    }
		}
		$prog->{q(credits)} = \%credits;
	    }
	}
    }

    #exit; # DEBUG
    $tree->delete;
}


# Bump a YYYYMMDD date by one.
sub nextday( $ ) {
    my $d = shift;
    my $p = parse_date($d);
    my $n = DateCalc($p, "+ $daysperpage day");
    return UnixDate($n, '%Q');
}

