Author: vip Date: Sun Dec 30 14:27:11 2007 New Revision: 9169 Modified: packages.php/database.sql packages.php/plough.php Log: - wget files, show more info - current db dump
Modified: packages.php/database.sql ============================================================================== --- packages.php/database.sql (original) +++ packages.php/database.sql Sun Dec 30 14:27:11 2007 @@ -1,52 +1,194 @@ -CREATE TABLE IF NOT EXISTS `arch` ( - `arch_id` smallint(6) NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, +-- MySQL dump 10.13 +-- +-- Host: localhost Database: packages +-- ------------------------------------------------------ +-- Server version 5.1.22-rc + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `arch` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `arch` ( + `arch_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT, + `name` varchar(100) COLLATE utf8_polish_ci NOT NULL, PRIMARY KEY (`arch_id`) -) ENGINE=InnoDB DEFAULT; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; -CREATE TABLE IF NOT EXISTS `description` ( - `name` varchar(100) NOT NULL, - `summary` varchar(255) NOT NULL, - `url` varchar(255) NOT NULL, - `description` text NOT NULL, +-- +-- Dumping data for table `arch` +-- + +INSERT INTO `arch` VALUES (1,'i686'); +INSERT INTO `arch` VALUES (2,'i486'); + +-- +-- Table structure for table `description` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `description` ( + `name` varchar(100) COLLATE utf8_polish_ci NOT NULL, + `summary` varchar(255) COLLATE utf8_polish_ci NOT NULL, + `url` varchar(255) COLLATE utf8_polish_ci NOT NULL, + `description` text COLLATE utf8_polish_ci NOT NULL, PRIMARY KEY (`name`) -) ENGINE=InnoDB DEFAULT; +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; -CREATE TABLE IF NOT EXISTS `dist` ( +-- +-- Dumping data for table `description` +-- + + +-- +-- Table structure for table `dist` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `dist` ( `dist_id` smallint(6) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(20) NOT NULL, + `name` varchar(20) COLLATE utf8_polish_ci NOT NULL, PRIMARY KEY (`dist_id`) -) ENGINE=InnoDB DEFAULT; +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; -CREATE TABLE IF NOT EXISTS `file` ( - `package_id` varchar(50) NOT NULL, - `filepath` text NOT NULL, - KEY `package_id` (`package_id`) -) ENGINE=InnoDB DEFAULT; - -CREATE TABLE IF NOT EXISTS `module` ( +-- +-- Dumping data for table `dist` +-- + +INSERT INTO `dist` VALUES (1,'Th'); +INSERT INTO `dist` VALUES (2,'Ac'); +INSERT INTO `dist` VALUES (3,'Titanium'); + +-- +-- Table structure for table `file` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `file` ( + `package_id` varchar(50) COLLATE utf8_polish_ci NOT NULL, + `filepath` text COLLATE utf8_polish_ci NOT NULL, + KEY `package_id` (`package_id`), + CONSTRAINT `file_ibfk_1` FOREIGN KEY (`package_id`) REFERENCES `package` (`package_id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping data for table `file` +-- + + +-- +-- Table structure for table `module` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `module` ( `module_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT, `dist_id` smallint(5) unsigned NOT NULL, `arch_id` smallint(5) unsigned NOT NULL, - `name` varchar(20) NOT NULL, - `url` text NOT NULL, - PRIMARY KEY (`module_id`) -) ENGINE=InnoDB DEFAULT; - -CREATE TABLE IF NOT EXISTS `package` ( - `package_id` varchar(50) NOT NULL, - `name` varchar(100) NOT NULL, + `name` varchar(20) COLLATE utf8_polish_ci NOT NULL, + `url` text COLLATE utf8_polish_ci NOT NULL, + PRIMARY KEY (`module_id`), + KEY `dist_id` (`dist_id`), + KEY `arch_id` (`arch_id`), + CONSTRAINT `module_ibfk_1` FOREIGN KEY (`dist_id`) REFERENCES `dist` (`dist_id`), + CONSTRAINT `module_ibfk_2` FOREIGN KEY (`arch_id`) REFERENCES `arch` (`arch_id`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping data for table `module` +-- + +INSERT INTO `module` VALUES (1,1,1,'main','http://packages/'); +INSERT INTO `module` VALUES (2,1,2,'main','http://packages/'); + +-- +-- Temporary table structure for view `module_view` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +/*!50001 CREATE TABLE `module_view` ( + `module_id` smallint(5) unsigned, + `name` varchar(20), + `dist` varchar(20), + `arch` varchar(100), + `url` text +) */; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `package` +-- + +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `package` ( + `package_id` varchar(50) COLLATE utf8_polish_ci NOT NULL, + `name` varchar(100) COLLATE utf8_polish_ci NOT NULL, `module_id` smallint(5) unsigned NOT NULL, `epoch` smallint(5) unsigned NOT NULL, - `version` varchar(50) NOT NULL, - `release` varchar(50) NOT NULL, + `version` varchar(50) COLLATE utf8_polish_ci NOT NULL, + `release` varchar(50) COLLATE utf8_polish_ci NOT NULL, `file_time` datetime NOT NULL, `build_time` datetime NOT NULL, `package_size` int(10) unsigned NOT NULL, `installed_size` int(10) unsigned NOT NULL, `archive_size` int(10) unsigned NOT NULL, PRIMARY KEY (`package_id`) -) ENGINE=InnoDB DEFAULT; +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci; +SET character_set_client = @saved_cs_client; + +-- +-- Dumping data for table `package` +-- + + +-- +-- Final view structure for view `module_view` +-- + +/*!50001 DROP TABLE `module_view`*/; +/*!50001 SET @saved_cs_client = @@character_set_client */; +/*!50001 SET @saved_cs_results = @@character_set_results */; +/*!50001 SET @saved_col_connection = @@collation_connection */; +/*!50001 SET character_set_client = utf8 */; +/*!50001 SET character_set_results = utf8 */; +/*!50001 SET collation_connection = utf8_unicode_ci */; +/*!50001 CREATE ALGORITHM=UNDEFINED */ +/*!50013 [EMAIL PROTECTED] SQL SECURITY DEFINER */ +/*!50001 VIEW `module_view` AS select `module`.`module_id` AS `module_id`,`module`.`name` AS `name`,`dist`.`name` AS `dist`,`arch`.`name` AS `arch`,`module`.`url` AS `url` from ((`module` join `arch` on((`module`.`arch_id` = `arch`.`arch_id`))) join `dist` on((`module`.`dist_id` = `dist`.`dist_id`))) */; +/*!50001 SET character_set_client = @saved_cs_client */; +/*!50001 SET character_set_results = @saved_cs_results */; +/*!50001 SET collation_connection = @saved_col_connection */; +/*!40103 SET [EMAIL PROTECTED] */; + +/*!40101 SET [EMAIL PROTECTED] */; +/*!40014 SET [EMAIL PROTECTED] */; +/*!40014 SET [EMAIL PROTECTED] */; +/*!40101 SET [EMAIL PROTECTED] */; +/*!40101 SET [EMAIL PROTECTED] */; +/*!40101 SET [EMAIL PROTECTED] */; +/*!40111 SET [EMAIL PROTECTED] */; -ALTER TABLE `file` - ADD CONSTRAINT `file_ibfk_1` FOREIGN KEY (`package_id`) REFERENCES `package` (`package_id`) ON DELETE CASCADE; +-- Dump completed on 2007-12-30 13:14:08 Modified: packages.php/plough.php ============================================================================== --- packages.php/plough.php (original) +++ packages.php/plough.php Sun Dec 30 14:27:11 2007 @@ -35,10 +35,9 @@ { global $mysqli; -// $module = find_module_id(find_dist_id($dist), find_arch_id($arch), $dir); + echo " p"; $metadata = simplexml_load_file("primary.xml"); - $mysqli->autocommit(false); foreach($metadata->package as $package) @@ -71,7 +70,7 @@ $q .= "')"; $mysqli->query($q); - $q = "INSERT IGNORE INTO description VALUES ('"; + $q = "REPLACE INTO description VALUES ('"; $q .= $package->name; $q .= "','"; $q .= addslashes($package->summary); @@ -80,7 +79,7 @@ $q .= "','"; $q .= addslashes($package->description); $q .= "')"; - $mysqli->query($q); + $mysqli->query($q) or die($mysqli->error); } $mysqli->commit(); } @@ -91,13 +90,9 @@ $i = 0; - $filelists = simplexml_load_file("filelists.xml"); -/* $dir = "ready"; // TODO: gather from ftp dir - $dist = "th"; - $arch = "i686";*/ - -// $module = find_module_id(find_dist_id($dist), find_arch_id($arch), $dir); + echo " p"; + $filelists = simplexml_load_file("filelists.xml"); $mysqli->autocommit(false); foreach($filelists->package as $package) @@ -106,8 +101,6 @@ foreach($package->file as $file) { - //print_r($file); - //die(); $q = "INSERT INTO file VALUES ('"; $q .= $pkgid; $q .= "','"; @@ -115,31 +108,42 @@ $q .= "')"; $mysqli->query($q); - /*if(++$i > 100000) + if(++$i > 100000) { - echo " [.] commit\n"; + echo "."; $mysqli->commit(); $i = 0; - }*/ + } } } $mysqli->commit(); } -echo "[*] gathering sources...\n"; -$sources = fill_source_list(); +function get_files($url, $file) +{ + echo " d"; + $wget = `wget -r -q $url/$file -O $file`; + echo " u"; + $gzip = `gunzip -f $file`; +} -echo "[*] cleaning...\n"; +echo "[*] gathering sources..."; +$sources = fill_source_list(); +echo "\n[*] cleaning..."; clean_database(); -die(); + foreach($sources as $s) { - echo "[*] {$s['dist']}-{$s['name']}, {$s['arch']}; Id: {$s['module_id']} ({$s['url']})\n"; - echo " [+] primary\n"; + echo "\n[*] {$s['dist']}-{$s['name']}, {$s['arch']}; Id: {$s['module_id']} ({$s['url']})"; + echo "\n [+] primary"; + get_files($s['url'], "primary.xml.gz"); parse_primary($s['module_id']); + unlink("primary.xml"); - echo " [+] filelists\n"; + echo "\n [+] filelists"; + get_files($s['url'], "filelists.xml.gz"); parse_filelists(); + unlink("filelists.xml"); } ?> _______________________________________________ pld-cvs-commit mailing list [email protected] http://lists.pld-linux.org/mailman/listinfo/pld-cvs-commit
