#!/usr/bin/ruby

require 'sqlite3'
require 'strscan'
require 'zlib'

pagelinks="frwiki-latest-pagelinks.sql.gz"
page="frwiki-latest-page.sql.gz"
database="wp.db"

[pagelinks, page].each { |filename|
	system "wget", filename unless File.exist? filename
}

def scan_quotedstr(ss)
	str = ''
	ss.scan(/'/)
	loop do
		str << ss.scan(/[^\\']*/).to_s
		break if ss.getch == "'"
		str << ss.getch
	end
	ss.scan(/,/)
	str
end

def time
	start = Time.now
	yield
	dur = (Time.now - start).to_i
	"#{dur/60}mn #{dur%60}"
end

File.unlink database rescue nil

db = SQLite3::Database.new database

db.execute_batch <<EOF
CREATE TABLE links (
	from_id INTEGER,
	to_id INTEGER
);

CREATE TABLE page (
	page_id INTEGER,
	title TEXT
);
EOF

db.cache_size = 40000
db.synchronous = 'off'
db.temp_store = 'memory'

pagecount = 0
puts time {
	fd = Zlib::GzipReader.new(File.open(page))
linecount = 0
	fd.each_line { |l|
print "line #{linecount} \r"
$stdout.flush
linecount += 1
		ss = StringScanner.new l
		next unless ss.scan /INSERT INTO `page` VALUES \(/

		db.transaction {
			id, namespace, title = nil
			while ss.scan(/(\d+),(\d+),/)
				id = ss[1]
				namespace = ss[2]
				title = scan_quotedstr(ss)
				raise "bad title: #{title}" unless ss.scan(/'[^']*',\d+,\d+,\d+,([\d.e-]*),'[^']*',\d+,\d+,\d+\)/)

				if namespace == '0'
					pagecount += 1
					db.execute("INSERT INTO `page` VALUES (#{id}, ?)", title)
				end
				break if ss.scan(/;\s*/)
				ss.scan(/,\(/)
			end
		}
	}
	fd.close
} + " to insert #{pagecount} pages"


puts time {
	db.execute_batch('CREATE INDEX page_id ON page (page_id); CREATE INDEX title ON page (title);')
} + " to create page indexes"

linkcount = 0
puts time {
	fd = Zlib::GzipReader.new(File.open(pagelinks))
linecount = 0
	fd.each_line { |l|
print "line #{linecount} \r"
$stdout.flush
linecount += 1
		ss = StringScanner.new l
		next unless ss.scan /INSERT INTO `pagelinks` VALUES \(/

		db.transaction {
			cachedtitle = nil
			idfrom, titleto, idto = nil
			while ss.scan(/(\d+),(\d+),/)
				idfrom = ss[1]
				namespace = ss[2]
				titleto = scan_quotedstr(ss)
				raise "bad link: #{idfrom} #{titleto}" unless ss.getch == ')'

				if namespace == '0' and db.get_first_value("select page_id from page where page_id = #{idfrom}")
					linkcount += 1
					if titleto != cachedtitle
						idto = db.get_first_value("select page_id from page where title = ?", titleto)
						cachedtitle = titleto
					end
					if idto
						db.execute("INSERT INTO `links` VALUES (#{idfrom}, #{idto})")
					end
				end
				break if ss.scan(/;\s*/)
				ss.scan(/,\(/)
			end
		}
	}
	fd.close
} + " to insert #{linkcount} links"

puts time {
	db.execute_batch('CREATE INDEX from_id ON links (from_id); CREATE INDEX to_id ON links (to_id);')
} + " to index links"

db.close

puts "db size: #{(File.size(database)*10/(1<<20))/10.0} Mo"