Difference between revisions of "Import CSV data into a wiki"
(Column selection from {{{params}}} in template now working) |
(Is this newer, or older?) |
||
Line 7: | Line 7: | ||
# - API: http://en.wikipedia.org/w/api.php | # - API: http://en.wikipedia.org/w/api.php | ||
− | |||
− | |||
− | |||
require('wiki.pl'); | require('wiki.pl'); | ||
Line 17: | Line 14: | ||
# Set a debug conditional | # Set a debug conditional | ||
− | $::debug = | + | $::debug = 0; |
$::log = "$1.log"; | $::log = "$1.log"; | ||
Line 35: | Line 32: | ||
if (/^\*?\s*separator\s*:\s*"(.+?)"\s*$/i) { $::sep = $1 } | if (/^\*?\s*separator\s*:\s*"(.+?)"\s*$/i) { $::sep = $1 } | ||
if (/^\*?\s*title\s*:\s*(.+?)\s*$/i) { $::title = $1 } | if (/^\*?\s*title\s*:\s*(.+?)\s*$/i) { $::title = $1 } | ||
− | if (/^\*?\s*template\s*:\s*(.+?)\s*$/i) { $::template = $1 } | + | if (/^\*?\s*template\s*:\s*(.+?)\s*$/i) { $::template = $1 } |
− | if (/^\*?\s*prefix\s*:\s*(.+?)\s*$/i) | + | if (/^\*?\s*prefix\s*:\s*(.+?)\s*$/i) { $::prefix = $1 } |
} | } | ||
close JOB; | close JOB; | ||
Line 63: | Line 60: | ||
# http://en.wikipedia.org/wiki/Help:Templates#Parameters | # http://en.wikipedia.org/wiki/Help:Templates#Parameters | ||
− | while ($wikitext =~ /\{{ | + | while ($wikitext =~ m/\{{{(.+?)}}}/g ) { |
− | $params{$ | + | $key = $1; |
+ | # Remove default anchors if there are any | ||
+ | $key =~ s/\|.*//; | ||
+ | $params{$key} = undef; | ||
} | } | ||
− | + | ||
− | # | + | # Need to check @headings elements that are identical to keys(%param) |
− | + | for( $i = 0; $i < @headings; $i++ ) { | |
− | if(exists($params{$headings[$i]})) { | + | |
− | + | if(exists($params{$headings[$i]})) { | |
− | + | print "$headings[$i] matches!\n"; | |
+ | push(@cols ,$i); | ||
+ | } | ||
} | } | ||
− | + | ||
if($::debug) { | if($::debug) { | ||
− | print "\@ | + | print "\@A: @headings\n"; |
− | print "% | + | print "%H: @{[%params]}\n"; |
+ | print "\@cols @cols\n"; | ||
+ | die "[\$::debug set exiting]\n" ; | ||
} | } | ||
} | } | ||
Line 90: | Line 94: | ||
@record = split /$::sep/, $1; | @record = split /$::sep/, $1; | ||
$tmpl = "{{$template\n"; | $tmpl = "{{$template\n"; | ||
− | $tmpl .= "|$_ = $record[ | + | # JUST NEED TO FIX UP HERE |
+ | local $/="|"; | ||
+ | $tmpl .= "|$headings[$_] = $record[$_]\n" for 0..$#headings; | ||
$tmpl .= "}}"; | $tmpl .= "}}"; | ||
print "Processing record ".$n++."\n"; | print "Processing record ".$n++."\n"; | ||
− | + | ||
− | |||
− | |||
− | |||
# Update the record | # Update the record | ||
$text = wikiRawPage($::wiki,$record[$::title],0); | $text = wikiRawPage($::wiki,$record[$::title],0); |
Revision as of 23:19, 3 June 2008
- !/usr/bin/perl
- Our Perl scripts.{{#security:*|sysop}}Automated scripts to perform batch automation.
- - Licenced under LGPL (http://www.gnu.org/copyleft/lesser.html)
- - Author: http://www.organicdesign.co.nz/nad
- - Source: http://www.organicdesign.co.nz/scraper.pl
- - Started: 2008-03-21
- - API: http://en.wikipedia.org/w/api.php
require('wiki.pl');
- Job, log and error files
$ARGV[0] or die "No job file specified!"; $ARGV[0] =~ /^(.+?)(\..+?)?$/;
- Set a debug conditional
$::debug = 0;
$::log = "$1.log"; $::err = "$1.err"; $::sep = ','; $::title = 0; $::template = 'Record'; $::prefix = "";
- Parse the job file
if (open JOB,'<',$ARGV[0]) { for (<JOB>) { if (/^\*?\s*csv\s*:\s*(.+?)\s*$/i) { $::csv = $1 } if (/^\*?\s*wiki\s*:\s*(.+?)\s*$/i) { $::wiki = $1 } if (/^\*?\s*user\s*:\s*(.+?)\s*$/i) { $::user = $1 } if (/^\*?\s*pass\s*:\s*(.+?)\s*$/i) { $::pass = $1 } if (/^\*?\s*separator\s*:\s*"(.+?)"\s*$/i) { $::sep = $1 } if (/^\*?\s*title\s*:\s*(.+?)\s*$/i) { $::title = $1 } if (/^\*?\s*template\s*:\s*(.+?)\s*$/i) { $::template = $1 } if (/^\*?\s*prefix\s*:\s*(.+?)\s*$/i) { $::prefix = $1 } } close JOB; } else { die "Couldn't parse job file!" }
- Open CSV file and read in headings line
if (open CSV,'<',$::csv) { $_ = <CSV>; /^\s*(.+?)\s*$/; @headings = split /$::sep/i, $1; } else { die "Could not open CSV file!" }
- Log in to the wiki
wikiLogin($::wiki,$::user,$::pass) or exit;
- fetch the template if it exists
$response = $client->get("$wiki?title=Template:$template&action=raw"); if( $response->is_success ) {
$wikitext = $response->content;
# Remove noinclude areas $wikitext =~ s/.+?<\/noinclude>//gs;
# Find all unique {{{parameters}}} # http://en.wikipedia.org/wiki/Help:Templates#Parameters
while ($wikitext =~ m/\{{{(.+?)}}}/g ) { $key = $1; # Remove default anchors if there are any $key =~ s/\|.*//; $params{$key} = undef; } # Need to check @headings elements that are identical to keys(%param) for( $i = 0; $i < @headings; $i++ ) { if(exists($params{$headings[$i]})) { print "$headings[$i] matches!\n"; push(@cols ,$i); } } if($::debug) { print "\@A: @headings\n"; print "%H: @{[%params]}\n"; print "\@cols @cols\n"; die "[\$::debug set exiting]\n" ; }
}
- Get batch size and current number (also later account for n-bots)
- todo: log batch start
- Process the records
$n = 1; while (<CSV>) { /^\s*(.+?)\s*$/; @record = split /$::sep/, $1; $tmpl = "{{$template\n"; # JUST NEED TO FIX UP HERE
local $/="|";
$tmpl .= "|$headings[$_] = $record[$_]\n" for 0..$#headings; $tmpl .= "}}"; print "Processing record ".$n++."\n";
# Update the record $text = wikiRawPage($::wiki,$record[$::title],0); $text .= "\n$tmpl" unless $text =~ s/\{\{$template.+?\}\}/$tmpl/is; $done = wikiPageEdit($::wiki,$record[$::title],$text,"$template updated by csv2wiki.pl");
# log a row error if any }
close CSV;