use WWW::Mechanize;
use File::Basename;
use utf8;

# Define a new "object" containing your "agent".
# (the autocheck bit tells it to check for errors)
 
my $mech = WWW::Mechanize->new( autocheck => 1 );

# Accept cookies
cookie_jar => {};

# Get the index page for results
$mech->get( "http://www.elections.public.lu/legislatives/2004/resultats/index.html" );

# Find all the links to the commune-level pages and store the urls

my @communepages = $mech->find_all_links(url_regex => qr/communes\//);
my @urls = map{ $_->[0] } @communepages;

# Prep the directory and files
my $directory = "LuxembourgFiles/";
open(PARTYVOTE, ">$directory"."partyvote.txt");
print PARTYVOTE "Commune, ADR, DP, LSAP, DG, CSV, DL, KPL, FPL\n";  

# Loop over each commune url
for my $url (@urls) {
    my $filename = basename($url);      # get the "commune.html" name
    my $file = $directory.$filename;
    $mech->get( $url);                  # go get the page
    my $page = $mech->content();        # store it in $page
    
    open(HTMLFILE, ">$file");           # write a copy to your local drive
    print HTMLFILE $page;
    close(HTMLFILE);
    
    $filename =~ m/^(.+).html/;         # strip down to the commune name
    my $commune = $1;                   # store it in $commune

    my @partyresult = ("",0,0,0,0,0,0,0);   # prep the result line
    $partyresult[0] = $commune;             # store the commune name
    for ($i=1; $i<=8; $i+=1) {              # loop over the parties
       $page =~ m/left">($i - .+?)tr/s;     # find the party's information
       my $partyinfo = $1;
       @partynumbers = ($partyinfo =~ m/>(\d+)</gs);  # find the vote nos.
       $partyresult[$i] = $partynumbers[2];           # keep total vote
    }

    print PARTYVOTE join(", ", @partyresult)."\n";    # write it to file
    
    sleep(1);                               # pause a second to be polite
    
}

close(PARTYVOTE);