#!/usr/bin/ruby

if ARGV.empty?
	File.unlink 'wp.db' rescue nil
	(0..7).each { |step|
		system "ruby #$0 #{step} |sqlite3 wp.db"
	}
	exit
end

require 'strscan'
require 'zlib'

def log(str)
	$stderr.puts "#{Time.now.strftime('%H:%M:%S')} #{str}"
end

def scan_quotedstr(ss)
	str = ''
	ss.scan(/'/)
	loop do
		str << ss.scan(/[^\\']*/).to_s
		break if ss.getch == "'"
		str << ss.getch
	end
	ss.scan(/,/)
	str
end

pagelinks="frwiki-latest-pagelinks.sql.gz"
page="frwiki-latest-page.sql.gz"

case ARGV.shift.to_i
when 0
log 'retrieve archive'
[pagelinks, page].each { |filename|
	system "wget", filename unless File.exist? filename
}

when 1
log 'tables definition'
puts <<EOF
CREATE TABLE links (
	from_id INTEGER,
	to_id INTEGER
);

CREATE TABLE page (
	page_id INTEGER,
	title TEXT
);
EOF

when 2
log 'insert pages'
fd = Zlib::GzipReader.new(File.open(page))
fd.each_line { |l|
	ss = StringScanner.new l
	next unless ss.scan /INSERT INTO `page` VALUES \(/

	puts "BEGIN TRANSACTION;"
	while ss.scan(/(\d+),(\d+),/)
		id = ss[1]
		namespace = ss[2]
		title = scan_quotedstr(ss)
		raise "bad title: #{title}" unless ss.scan(/'[^']*',\d+,\d+,\d+,([\d.e-]*),'[^']*',\d+,\d+,\d+\)/)

		if namespace == '0'
			puts "INSERT INTO `page` VALUES (#{id}, '#{title.gsub("'", "''")}');"
		end
		ss.scan(/,\(/)
	end
	puts "COMMIT TRANSACTION;"
}
fd.close

when 3
log 'index pages'
puts <<EOF
CREATE INDEX page_id ON page (page_id);
CREATE INDEX title ON page (title);
EOF

when 4
log 'insert links'

require 'sqlite3'
system 'cp', 'wp.db', 'wp-tmp.db'
db = SQLite3::Database.open('wp-tmp.db')

ctr = 0
fd = Zlib::GzipReader.new(File.open(pagelinks))
fd.each_line { |l|
	ss = StringScanner.new l
	next unless ss.scan /INSERT INTO `pagelinks` VALUES \(/

ctr += 1
$stderr.print('/-\|'[ctr%4].chr + "\r")
	puts "BEGIN TRANSACTION;"
	cachetitle = cacheid = nil
	while ss.scan(/(\d+),(\d+),/)
		idfrom = ss[1]
		namespace = ss[2]
		titleto = scan_quotedstr(ss)
		raise "bad link: #{idfrom} #{titleto}" unless ss.getch == ')'

		if namespace == '0'
			if cachetitle != titleto
				cachetitle = titleto
				cacheid = db.get_first_value("select page_id from page where title = '#{cachetitle.gsub("'", "''")}'")
			end
			if cacheid
				puts "INSERT INTO `links` VALUES (#{idfrom}, #{cacheid});"	# '#{titleto.gsub("'", "''")}');"
			end
		end
		ss.scan(/,\(/)
	end
	puts "COMMIT TRANSACTION;"
}
fd.close

db.close
system 'rm' 'wp-tmp.db'

when 5
log 'update links'
puts <<EOF
PRAGMA synchronous=OFF;
PRAGMA cache_size=40000;
DELETE FROM links WHERE (SELECT page_id FROM page WHERE page_id = from_id) ISNULL;
-- UPDATE links SET to_id = (SELECT page_id FROM page WHERE title = to_id);

EOF

when 6
log 'index links'
puts <<EOF
PRAGMA synchronous=OFF;
PRAGMA cache_size=40000;
CREATE INDEX from_id ON links (from_id);
CREATE INDEX to_id ON links (to_id);

EOF

when 7
log 'done'

end