#!/usr/bin/perl -w # osvdb-xml-to-postgres - Imports OSVDB XML dump into a postgres database # Copyright (C) 2007 pentestmonkey@pentestmonkey.net # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # You are encouraged to send comments, improvements or suggestions to # me at pentestmonkey@pentestmonkey.net # use strict; use XML::Simple; use Data::Dumper; use DBI; my $verbose = 0; my $dbname = "osvdb"; my $port = 5432; my $host = "localhost"; my $username = "postgres"; my $password = ""; my %f; # TODO pass the above params from command line # Connect to database my $dbh = DBI->connect("dbi:Pg:dbname=$dbname;host=$host;port=$port;", "$username", "$password", {RaiseError => 1, AutoCommit => 0}) or die "Can't connect to DB: $!\n"; # Aghh! Slurping in a 100MB xml file with XML::Simple seems to use 2.4GB ram! my $usage = "Usage: $0 osvdb.xml osvdb-xml-to-postgres.pl v1.0 ( http://pentestmonkey.net/blog/osvdb-import ) Script for importing the OSVDB XML file into a postgres database. "; my $file = shift or die $usage; my $ref = XMLin($file, ForceArray => ['product', 'ext_ref', 'ext_txt', 'credit']); while (my $vuln_href = shift @{$ref->{vuln}}) { my $osvdb_id = $vuln_href->{osvdb_id}; $vuln_href->{osvdb_title} = "" if (ref($vuln_href->{osvdb_title}) eq "HASH"); # Yes, one even has a null title: OSVDB ID 36030 print "Importing OSVDB ID $osvdb_id: " . $vuln_href->{osvdb_title} . "\n"; next if vuln_present($osvdb_id); insert_vuln( $osvdb_id, $vuln_href->{osvdb_title}, $vuln_href->{disclosure_date}, $vuln_href->{discovery_date}, $vuln_href->{osvdb_create_date}, $vuln_href->{osvdb_last_modified_date}, $vuln_href->{exploit_publish_date}, $vuln_href->{location_physical}, $vuln_href->{location_local}, $vuln_href->{location_remote}, $vuln_href->{location_dialup}, $vuln_href->{location_unknown}, # * $vuln_href->{attack_type_auth_manage}, $vuln_href->{attack_type_crypt}, $vuln_href->{attack_type_dos}, $vuln_href->{attack_type_hijack}, # * $vuln_href->{attack_type_info_disclose}, $vuln_href->{attack_type_infrastruct}, # * $vuln_href->{attack_type_input_manip}, $vuln_href->{attack_type_miss_config}, $vuln_href->{attack_type_race}, # * $vuln_href->{attack_type_other}, # * $vuln_href->{attack_type_unknown}, # * $vuln_href->{impact_confidential}, $vuln_href->{impact_integrity}, $vuln_href->{impact_available}, $vuln_href->{impact_unknown}, $vuln_href->{exploit_available}, $vuln_href->{exploit_unavailable}, $vuln_href->{exploit_rumored}, $vuln_href->{exploit_unknown}, $vuln_href->{vuln_verified}, $vuln_href->{vuln_myth_fake}, # * $vuln_href->{vuln_best_prac}, $vuln_href->{vuln_concern}, $vuln_href->{vuln_web_check} # * = I haven't seen these in the XML file ); foreach my $product_href (@{$vuln_href->{products}->{product}}) { insert_object($osvdb_id, $product_href->{affected}, insert_product($product_href)); } foreach my $ext_ref_href (@{$vuln_href->{ext_refs}->{ext_ref}}) { insert_ext_ref($osvdb_id, $ext_ref_href); } foreach my $ext_txt_href (@{$vuln_href->{ext_txts}->{ext_txt}}) { insert_ext_txt($osvdb_id, $ext_txt_href); } foreach my $credit_href (@{$vuln_href->{credits}->{credit}}) { insert_credit($osvdb_id, $credit_href); # TODO } $dbh->commit unless($osvdb_id % 100); # commit periodically to avoid transaction getting too big. } # TODO: score, score_weight aren't imported by this script. How? $dbh->commit; sub vuln_present { my $osvdb_id = shift; my $sth = $dbh->prepare("SELECT osvdb_id FROM vuln WHERE osvdb_id = ?"); $sth->execute($osvdb_id); my ($id) = $sth->fetchrow_array; if (defined($id)) { return 1; } else { return 0; } } # Table: object_base sub insert_object_base { my $base_name = shift; $base_name = "" if (ref($base_name) eq "HASH"); my $id = select_object_base($base_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object_base (base_name) VALUES (?)"); $sth->execute($base_name); $id = select_object_base($base_name); } return $id; } sub select_object_base { my $base_name = shift; my $sth = $dbh->prepare("SELECT base_id FROM object_base WHERE base_name = ?"); $sth->execute($base_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: object_version sub insert_object_version { my $version_name = shift; $version_name = "" if (ref($version_name) eq "HASH"); my $id = select_object_version($version_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object_version (version_name) VALUES (?)"); $sth->execute($version_name); $id = select_object_version($version_name); } return $id; } sub select_object_version { my $version_name = shift; my $sth = $dbh->prepare("SELECT version_id FROM object_version WHERE version_name = ?"); $sth->execute($version_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: object_vendor* sub insert_object_vendor { my $vendor_name = shift; $vendor_name = "" if (ref($vendor_name) eq "HASH"); my $id = select_object_vendor($vendor_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object_vendor (vendor_name) VALUES (?)"); $sth->execute($vendor_name); $id = select_object_vendor($vendor_name); } return $id; } sub select_object_vendor{ my $vendor_name = shift; my $sth = $dbh->prepare("SELECT vendor_id FROM object_vendor WHERE vendor_name = ?"); $sth->execute($vendor_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: object_correlation sub insert_product { my $p_href = shift; my $vendor_id = insert_object_vendor($p_href->{vendor_name}); my $base_id = insert_object_base($p_href->{base_name}); my $version_id = insert_object_version($p_href->{version_name}); my $id = select_object_correlation($vendor_id, $base_id, $version_id); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object_correlation (vendor_id, base_id, version_id) VALUES (?, ?, ?)"); $sth->execute($vendor_id, $base_id, $version_id); $id = select_object_correlation($vendor_id, $base_id, $version_id); } return $id; } sub select_object_correlation { my ($vendor_id, $base_id, $version_id) = @_; my $sth = $dbh->prepare("SELECT corr_id FROM object_correlation WHERE vendor_id = ? AND base_id = ? AND version_id = ?"); $sth->execute($vendor_id, $base_id, $version_id); my ($id) = $sth->fetchrow_array; return $id; } # Table: object_affect_type* sub insert_object_affect_type { my $type_name = shift; $type_name = "" if (ref($type_name) eq "HASH"); my $id = select_object_affect_type($type_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object_affect_type (type_name) VALUES (?)"); $sth->execute($type_name); $id = select_object_affect_type($type_name); } return $id; } sub select_object_affect_type { my $type_name = shift; my $sth = $dbh->prepare("SELECT type_id FROM object_affect_type WHERE type_name = ?"); $sth->execute($type_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: object sub insert_object { my ($osvdb_id, $affected, $corr_id) = @_; $affected = "" if (ref($affected) eq "HASH"); my $type_id = insert_object_affect_type($affected); my $id = select_object($osvdb_id, $corr_id, $type_id); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO object (osvdb_id, corr_id, type_id) VALUES (?, ?, ?)"); $sth->execute($osvdb_id, $corr_id, $type_id); $id = select_object($osvdb_id, $corr_id, $type_id); } return $id; } sub select_object { my ($osvdb_id, $corr_id, $type_id) = @_; my $sth = $dbh->prepare("SELECT object_id FROM object WHERE osvdb_id = ? AND corr_id = ? AND type_id = ?"); $sth->execute($osvdb_id, $corr_id, $type_id); my ($id) = $sth->fetchrow_array; return $id; } # Table: ext_ref_type* sub insert_ext_ref_type { my $type_name = shift; $type_name = "" if (ref($type_name) eq "HASH"); my $id = select_ext_ref_type($type_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO ext_ref_type (type_name) VALUES (?)"); $sth->execute($type_name); $id = select_ext_ref_type($type_name); } return $id; } sub select_ext_ref_type { my $type_name = shift; my $sth = $dbh->prepare("SELECT type_id FROM ext_ref_type WHERE type_name = ?"); $sth->execute($type_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: ext_ref_value sub insert_ext_ref_value { my ($type_id, $ref_value) = @_; $ref_value = "" if (!defined($ref_value) or ref($ref_value) eq "HASH"); my $id = select_ext_ref_value($type_id, $ref_value); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO ext_ref_value (type_id, ref_value) VALUES (?, ?)"); $sth->execute($type_id, $ref_value); $id = select_ext_ref_value($type_id, $ref_value); } return $id; } sub select_ext_ref_value { my ($type_id, $ref_value) = @_; $ref_value = "" if (ref($ref_value) eq "HASH"); my $sth = $dbh->prepare("SELECT value_id FROM ext_ref_value WHERE type_id = ? AND ref_value = ?"); $sth->execute($type_id, $ref_value); my ($id) = $sth->fetchrow_array; return $id; } # Table: ext_ref sub insert_ext_ref { my ($osvdb_id, $href) = @_; my $type_id = insert_ext_ref_type($href->{type_name}); my $value_id = insert_ext_ref_value($type_id, $href->{content}); my $id = select_ext_ref($osvdb_id, $value_id, $href->{indirect}); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO ext_ref (osvdb_id, value_id, indirect) VALUES (?, ?, ?)"); $sth->execute($osvdb_id, $value_id, $href->{indirect}); $id = select_ext_ref($osvdb_id, $value_id, $href->{indirect}); } return $id; } sub select_ext_ref { my ($osvdb_id, $value_id, $indirect) = @_; my $sth = $dbh->prepare("SELECT ref_id FROM ext_ref WHERE osvdb_id = ? AND value_id = ? AND indirect = ?"); $sth->execute($osvdb_id, $value_id, $indirect); my ($id) = $sth->fetchrow_array; return $id; } # Table: ext_txt_type* sub insert_ext_txt_type { my $type_name = shift; $type_name = "" if (ref($type_name) eq "HASH"); my $id = select_ext_txt_type($type_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO ext_txt_type (type_name) VALUES (?)"); $sth->execute($type_name); $id = select_ext_txt_type($type_name); } return $id; } sub select_ext_txt_type { my $type_name = shift; my $sth = $dbh->prepare("SELECT type_id FROM ext_txt_type WHERE type_name = ?"); $sth->execute($type_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: language* sub insert_language { my $lang_name = shift; $lang_name = "" if (ref($lang_name) eq "HASH"); my $id = select_language($lang_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO language (lang_name) VALUES (?)"); $sth->execute($lang_name); $id = select_language($lang_name); } return $id; } sub select_language{ my $lang_name = shift; my $sth = $dbh->prepare("SELECT lang_id FROM language WHERE lang_name = ?"); $sth->execute($lang_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: author sub insert_author { my ($author_name, $author_company, $author_email, $company_url) = @_; $author_name = "" if (ref($author_name) eq "HASH"); $author_company = "" if (ref($author_company) eq "HASH"); $author_email = "" if (ref($author_email) eq "HASH"); $company_url = "" if (ref($company_url) eq "HASH"); my $id = select_author($author_name, $author_company, $author_email, $company_url); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO author (author_name, author_company, author_email, company_url) VALUES (?, ?, ?, ?)"); $sth->execute($author_name, $author_company, $author_email, $company_url); $id = select_author($author_name, $author_company, $author_email, $company_url); } return $id; } sub select_author { my ($author_name, $author_company, $author_email, $company_url) = @_; my $sth = $dbh->prepare("SELECT author_id FROM author WHERE author_name = ? AND author_company = ? AND author_email = ? AND company_url = ?"); $sth->execute($author_name, $author_company, $author_email, $company_url); my ($id) = $sth->fetchrow_array; return $id; } # Table: ext_txt TODO: document the fact that i've removed the author_id field from this table sub insert_ext_txt { my ($osvdb_id, $ext_txt_href) = @_; my $revision = $ext_txt_href->{revision}; my $text = $ext_txt_href->{text}; $text = "" if (ref($text) eq "HASH"); my $type_id = insert_ext_txt_type($ext_txt_href->{type_name}); my $lang_id = insert_language($ext_txt_href->{language}); my $id = select_ext_txt($revision, $text, $osvdb_id, $type_id, $lang_id); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO ext_txt (revision, text, osvdb_id, type_id, lang_id) VALUES (?, ?, ?, ?, ?)"); $sth->execute($revision, $text, $osvdb_id, $type_id, $lang_id); $id = select_ext_txt($revision, $text, $osvdb_id, $type_id, $lang_id); } return $id; } sub select_ext_txt { my ($revision, $text, $osvdb_id, $type_id, $lang_id) = @_; my $sth = $dbh->prepare("SELECT ext_id FROM ext_txt WHERE revision = ? AND text = ? AND osvdb_id = ? AND type_id = ? AND lang_id = ?"); $sth->execute($revision, $text, $osvdb_id, $type_id, $lang_id); my ($id) = $sth->fetchrow_array; return $id; } # Table: credit sub insert_credit { my ($osvdb_id, $credit_href) = @_; # set up author_id my $author_id = insert_author($credit_href->{author_name}, $credit_href->{author_company}, $credit_href->{author_email}, $credit_href->{company_url}); my $id = select_credit($osvdb_id, $author_id); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO credit (osvdb_id, author_id) VALUES (?, ?)"); $sth->execute($osvdb_id, $author_id); $id = select_credit($osvdb_id, $author_id); } return $id; } sub select_credit { my ($osvdb_id, $author_id) = @_; my $sth = $dbh->prepare("SELECT credit_id FROM credit WHERE osvdb_id = ? AND author_id = ?"); $sth->execute($osvdb_id, $author_id); my ($id) = $sth->fetchrow_array; return $id; } # Table: score_weight* sub insert_score_weight { my $weight_name = shift; $weight_name = "" if (ref($weight_name) eq "HASH"); my $id = select_score_weight($weight_name); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO score_weight (weight_name) VALUES (?)"); $sth->execute($weight_name); $id = select_score_weight($weight_name); } return $id; } sub select_score_weight { my $weight_name = shift; my $sth = $dbh->prepare("SELECT weight_id FROM score_weight WHERE weight_name = ?"); $sth->execute($weight_name); my ($id) = $sth->fetchrow_array; return $id; } # Table: score # Table: vuln TODO: add the rest of the vuln data! sub insert_vuln { foreach my $var (@_) { $var = 0 unless defined($var); # undef's not allowed by SQL schema } my ( $osvdb_id, $osvdb_title, $disclosure_date, $discovery_date, $osvdb_create_date, $last_modified_date, $exploit_publish_date, $location_physical, $location_local, $location_remote, $location_dialup, $location_unknown, $attack_type_auth_manage, $attack_type_crypt, $attack_type_dos, $attack_type_hijack, $attack_type_info_disclose, $attack_type_infrastruct, $attack_type_input_manip, $attack_type_miss_config, $attack_type_race, $attack_type_other, $attack_type_unknown, $impact_confidential, $impact_integrity, $impact_available, $impact_unknown, $exploit_available, $exploit_unavailable, $exploit_rumored, $exploit_unknown, $vuln_verified, $vuln_myth_fake, $vuln_best_prac, $vuln_concern, $vuln_web_check ) = @_; my $id = select_vuln( $osvdb_id, $osvdb_title, $disclosure_date, $discovery_date, $osvdb_create_date, $last_modified_date, $exploit_publish_date, $location_physical, $location_local, $location_remote, $location_dialup, $location_unknown, $attack_type_auth_manage, $attack_type_crypt, $attack_type_dos, $attack_type_hijack, $attack_type_info_disclose, $attack_type_infrastruct, $attack_type_input_manip, $attack_type_miss_config, $attack_type_race, $attack_type_other, $attack_type_unknown, $impact_confidential, $impact_integrity, $impact_available, $impact_unknown, $exploit_available, $exploit_unavailable, $exploit_rumored, $exploit_unknown, $vuln_verified, $vuln_myth_fake, $vuln_best_prac, $vuln_concern, $vuln_web_check ); unless (defined($id)) { my $sth = $dbh->prepare("INSERT INTO vuln ( osvdb_id, osvdb_title, disclosure_date, discovery_date, osvdb_create_date, last_modified_date, exploit_publish_date, location_physical, location_local, location_remote, location_dialup, location_unknown, attack_type_auth_manage, attack_type_crypt, attack_type_dos, attack_type_hijack, attack_type_info_disclose, attack_type_infrastruct, attack_type_input_manip, attack_type_miss_config, attack_type_race, attack_type_other, attack_type_unknown, impact_confidential, impact_integrity, impact_available, impact_unknown, exploit_available, exploit_unavailable, exploit_rumored, exploit_unknown, vuln_verified, vuln_myth_fake, vuln_best_prac, vuln_concern, vuln_web_check ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); $sth->execute( $osvdb_id, $osvdb_title, $disclosure_date, $discovery_date, $osvdb_create_date, $last_modified_date, $exploit_publish_date, $location_physical, $location_local, $location_remote, $location_dialup, $location_unknown, $attack_type_auth_manage, $attack_type_crypt, $attack_type_dos, $attack_type_hijack, $attack_type_info_disclose, $attack_type_infrastruct, $attack_type_input_manip, $attack_type_miss_config, $attack_type_race, $attack_type_other, $attack_type_unknown, $impact_confidential, $impact_integrity, $impact_available, $impact_unknown, $exploit_available, $exploit_unavailable, $exploit_rumored, $exploit_unknown, $vuln_verified, $vuln_myth_fake, $vuln_best_prac, $vuln_concern, $vuln_web_check ); $id = select_score_weight($osvdb_id); } return $id; } sub select_vuln { my ( $osvdb_id, $osvdb_title, $disclosure_date, $discovery_date, $osvdb_create_date, $last_modified_date, $exploit_publish_date, $location_physical, $location_local, $location_remote, $location_dialup, $location_unknown, $attack_type_auth_manage, $attack_type_crypt, $attack_type_dos, $attack_type_hijack, $attack_type_info_disclose, $attack_type_infrastruct, $attack_type_input_manip, $attack_type_miss_config, $attack_type_race, $attack_type_other, $attack_type_unknown, $impact_confidential, $impact_integrity, $impact_available, $impact_unknown, $exploit_available, $exploit_unavailable, $exploit_rumored, $exploit_unknown, $vuln_verified, $vuln_myth_fake, $vuln_best_prac, $vuln_concern, $vuln_web_check ) = @_; my $sth = $dbh->prepare("SELECT osvdb_id FROM vuln WHERE osvdb_id = ? AND osvdb_title = ? AND disclosure_date = ? AND discovery_date = ? AND osvdb_create_date = ? AND last_modified_date = ? AND exploit_publish_date = ? AND location_physical = ? AND location_local = ? AND location_remote = ? AND location_dialup = ? AND location_unknown = ? AND attack_type_auth_manage = ? AND attack_type_crypt = ? AND attack_type_dos = ? AND attack_type_hijack = ? AND attack_type_info_disclose = ? AND attack_type_infrastruct = ? AND attack_type_input_manip = ? AND attack_type_miss_config = ? AND attack_type_race = ? AND attack_type_other = ? AND attack_type_unknown = ? AND impact_confidential = ? AND impact_integrity = ? AND impact_available = ? AND impact_unknown = ? AND exploit_available = ? AND exploit_unavailable = ? AND exploit_rumored = ? AND exploit_unknown = ? AND vuln_verified = ? AND vuln_myth_fake = ? AND vuln_best_prac = ? AND vuln_concern = ? AND vuln_web_check = ? "); $sth->execute( $osvdb_id, $osvdb_title, $disclosure_date, $discovery_date, $osvdb_create_date, $last_modified_date, $exploit_publish_date, defined($location_physical) ? $location_physical : 0, defined($location_local) ? $location_local : 0, defined($location_remote) ? $location_remote : 0, defined($location_dialup) ? $location_dialup : 0, defined($location_unknown) ? $location_unknown : 0, defined($attack_type_auth_manage) ? $attack_type_auth_manage : 0, defined($attack_type_crypt) ? $attack_type_crypt : 0, defined($attack_type_dos) ? $attack_type_dos : 0, defined($attack_type_hijack) ? $attack_type_hijack : 0, defined($attack_type_info_disclose) ? $attack_type_info_disclose : 0, defined($attack_type_infrastruct) ? $attack_type_infrastruct : 0, defined($attack_type_input_manip) ? $attack_type_input_manip : 0, defined($attack_type_miss_config) ? $attack_type_miss_config : 0, defined($attack_type_race) ? $attack_type_race : 0, defined($attack_type_other) ? $attack_type_other : 0, defined($attack_type_unknown) ? $attack_type_unknown : 0, defined($impact_confidential) ? $impact_confidential : 0, defined($impact_integrity) ? $impact_integrity : 0, defined($impact_available) ? $impact_available : 0, defined($impact_unknown) ? $impact_unknown : 0, defined($exploit_available) ? $exploit_available : 0, defined($exploit_unavailable) ? $exploit_unavailable : 0, defined($exploit_rumored) ? $exploit_rumored : 0, defined($exploit_unknown) ? $exploit_unknown : 0, defined($vuln_verified) ? $vuln_verified : 0, defined($vuln_myth_fake) ? $vuln_myth_fake : 0, defined($vuln_best_prac) ? $vuln_best_prac : 0, defined($vuln_concern) ? $vuln_concern : 0, defined($vuln_web_check) ? $vuln_web_check : 0 ); my ($id) = $sth->fetchrow_array; return $id; }