SVN: packages.php: database.sql plough.php
vip
vip at pld-linux.org
Sun Dec 30 14:27:11 CET 2007
Author: vip
Date: Sun Dec 30 14:27:11 2007
New Revision: 9169
Modified:
packages.php/database.sql
packages.php/plough.php
Log:
- wget files, show more info
- current db dump
Modified: packages.php/database.sql
==============================================================================
--- packages.php/database.sql (original)
+++ packages.php/database.sql Sun Dec 30 14:27:11 2007
@@ -1,52 +1,194 @@
-CREATE TABLE IF NOT EXISTS `arch` (
- `arch_id` smallint(6) NOT NULL AUTO_INCREMENT,
- `name` varchar(100) NOT NULL,
+-- MySQL dump 10.13
+--
+-- Host: localhost Database: packages
+-- ------------------------------------------------------
+-- Server version 5.1.22-rc
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `arch`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `arch` (
+ `arch_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT,
+ `name` varchar(100) COLLATE utf8_polish_ci NOT NULL,
PRIMARY KEY (`arch_id`)
-) ENGINE=InnoDB DEFAULT;
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
-CREATE TABLE IF NOT EXISTS `description` (
- `name` varchar(100) NOT NULL,
- `summary` varchar(255) NOT NULL,
- `url` varchar(255) NOT NULL,
- `description` text NOT NULL,
+--
+-- Dumping data for table `arch`
+--
+
+INSERT INTO `arch` VALUES (1,'i686');
+INSERT INTO `arch` VALUES (2,'i486');
+
+--
+-- Table structure for table `description`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `description` (
+ `name` varchar(100) COLLATE utf8_polish_ci NOT NULL,
+ `summary` varchar(255) COLLATE utf8_polish_ci NOT NULL,
+ `url` varchar(255) COLLATE utf8_polish_ci NOT NULL,
+ `description` text COLLATE utf8_polish_ci NOT NULL,
PRIMARY KEY (`name`)
-) ENGINE=InnoDB DEFAULT;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
-CREATE TABLE IF NOT EXISTS `dist` (
+--
+-- Dumping data for table `description`
+--
+
+
+--
+-- Table structure for table `dist`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `dist` (
`dist_id` smallint(6) unsigned NOT NULL AUTO_INCREMENT,
- `name` varchar(20) NOT NULL,
+ `name` varchar(20) COLLATE utf8_polish_ci NOT NULL,
PRIMARY KEY (`dist_id`)
-) ENGINE=InnoDB DEFAULT;
+) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
-CREATE TABLE IF NOT EXISTS `file` (
- `package_id` varchar(50) NOT NULL,
- `filepath` text NOT NULL,
- KEY `package_id` (`package_id`)
-) ENGINE=InnoDB DEFAULT;
-
-CREATE TABLE IF NOT EXISTS `module` (
+--
+-- Dumping data for table `dist`
+--
+
+INSERT INTO `dist` VALUES (1,'Th');
+INSERT INTO `dist` VALUES (2,'Ac');
+INSERT INTO `dist` VALUES (3,'Titanium');
+
+--
+-- Table structure for table `file`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `file` (
+ `package_id` varchar(50) COLLATE utf8_polish_ci NOT NULL,
+ `filepath` text COLLATE utf8_polish_ci NOT NULL,
+ KEY `package_id` (`package_id`),
+ CONSTRAINT `file_ibfk_1` FOREIGN KEY (`package_id`) REFERENCES `package` (`package_id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Dumping data for table `file`
+--
+
+
+--
+-- Table structure for table `module`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `module` (
`module_id` smallint(5) unsigned NOT NULL AUTO_INCREMENT,
`dist_id` smallint(5) unsigned NOT NULL,
`arch_id` smallint(5) unsigned NOT NULL,
- `name` varchar(20) NOT NULL,
- `url` text NOT NULL,
- PRIMARY KEY (`module_id`)
-) ENGINE=InnoDB DEFAULT;
-
-CREATE TABLE IF NOT EXISTS `package` (
- `package_id` varchar(50) NOT NULL,
- `name` varchar(100) NOT NULL,
+ `name` varchar(20) COLLATE utf8_polish_ci NOT NULL,
+ `url` text COLLATE utf8_polish_ci NOT NULL,
+ PRIMARY KEY (`module_id`),
+ KEY `dist_id` (`dist_id`),
+ KEY `arch_id` (`arch_id`),
+ CONSTRAINT `module_ibfk_1` FOREIGN KEY (`dist_id`) REFERENCES `dist` (`dist_id`),
+ CONSTRAINT `module_ibfk_2` FOREIGN KEY (`arch_id`) REFERENCES `arch` (`arch_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Dumping data for table `module`
+--
+
+INSERT INTO `module` VALUES (1,1,1,'main','http://packages/');
+INSERT INTO `module` VALUES (2,1,2,'main','http://packages/');
+
+--
+-- Temporary table structure for view `module_view`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+/*!50001 CREATE TABLE `module_view` (
+ `module_id` smallint(5) unsigned,
+ `name` varchar(20),
+ `dist` varchar(20),
+ `arch` varchar(100),
+ `url` text
+) */;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Table structure for table `package`
+--
+
+SET @saved_cs_client = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `package` (
+ `package_id` varchar(50) COLLATE utf8_polish_ci NOT NULL,
+ `name` varchar(100) COLLATE utf8_polish_ci NOT NULL,
`module_id` smallint(5) unsigned NOT NULL,
`epoch` smallint(5) unsigned NOT NULL,
- `version` varchar(50) NOT NULL,
- `release` varchar(50) NOT NULL,
+ `version` varchar(50) COLLATE utf8_polish_ci NOT NULL,
+ `release` varchar(50) COLLATE utf8_polish_ci NOT NULL,
`file_time` datetime NOT NULL,
`build_time` datetime NOT NULL,
`package_size` int(10) unsigned NOT NULL,
`installed_size` int(10) unsigned NOT NULL,
`archive_size` int(10) unsigned NOT NULL,
PRIMARY KEY (`package_id`)
-) ENGINE=InnoDB DEFAULT;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_polish_ci;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Dumping data for table `package`
+--
+
+
+--
+-- Final view structure for view `module_view`
+--
+
+/*!50001 DROP TABLE `module_view`*/;
+/*!50001 SET @saved_cs_client = @@character_set_client */;
+/*!50001 SET @saved_cs_results = @@character_set_results */;
+/*!50001 SET @saved_col_connection = @@collation_connection */;
+/*!50001 SET character_set_client = utf8 */;
+/*!50001 SET character_set_results = utf8 */;
+/*!50001 SET collation_connection = utf8_unicode_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`mysql`@`localhost` SQL SECURITY DEFINER */
+/*!50001 VIEW `module_view` AS select `module`.`module_id` AS `module_id`,`module`.`name` AS `name`,`dist`.`name` AS `dist`,`arch`.`name` AS `arch`,`module`.`url` AS `url` from ((`module` join `arch` on((`module`.`arch_id` = `arch`.`arch_id`))) join `dist` on((`module`.`dist_id` = `dist`.`dist_id`))) */;
+/*!50001 SET character_set_client = @saved_cs_client */;
+/*!50001 SET character_set_results = @saved_cs_results */;
+/*!50001 SET collation_connection = @saved_col_connection */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-ALTER TABLE `file`
- ADD CONSTRAINT `file_ibfk_1` FOREIGN KEY (`package_id`) REFERENCES `package` (`package_id`) ON DELETE CASCADE;
+-- Dump completed on 2007-12-30 13:14:08
Modified: packages.php/plough.php
==============================================================================
--- packages.php/plough.php (original)
+++ packages.php/plough.php Sun Dec 30 14:27:11 2007
@@ -35,10 +35,9 @@
{
global $mysqli;
-// $module = find_module_id(find_dist_id($dist), find_arch_id($arch), $dir);
+ echo " p";
$metadata = simplexml_load_file("primary.xml");
-
$mysqli->autocommit(false);
foreach($metadata->package as $package)
@@ -71,7 +70,7 @@
$q .= "')";
$mysqli->query($q);
- $q = "INSERT IGNORE INTO description VALUES ('";
+ $q = "REPLACE INTO description VALUES ('";
$q .= $package->name;
$q .= "','";
$q .= addslashes($package->summary);
@@ -80,7 +79,7 @@
$q .= "','";
$q .= addslashes($package->description);
$q .= "')";
- $mysqli->query($q);
+ $mysqli->query($q) or die($mysqli->error);
}
$mysqli->commit();
}
@@ -91,13 +90,9 @@
$i = 0;
- $filelists = simplexml_load_file("filelists.xml");
-/* $dir = "ready"; // TODO: gather from ftp dir
- $dist = "th";
- $arch = "i686";*/
-
-// $module = find_module_id(find_dist_id($dist), find_arch_id($arch), $dir);
+ echo " p";
+ $filelists = simplexml_load_file("filelists.xml");
$mysqli->autocommit(false);
foreach($filelists->package as $package)
@@ -106,8 +101,6 @@
foreach($package->file as $file)
{
- //print_r($file);
- //die();
$q = "INSERT INTO file VALUES ('";
$q .= $pkgid;
$q .= "','";
@@ -115,31 +108,42 @@
$q .= "')";
$mysqli->query($q);
- /*if(++$i > 100000)
+ if(++$i > 100000)
{
- echo " [.] commit\n";
+ echo ".";
$mysqli->commit();
$i = 0;
- }*/
+ }
}
}
$mysqli->commit();
}
-echo "[*] gathering sources...\n";
-$sources = fill_source_list();
+function get_files($url, $file)
+{
+ echo " d";
+ $wget = `wget -r -q $url/$file -O $file`;
+ echo " u";
+ $gzip = `gunzip -f $file`;
+}
-echo "[*] cleaning...\n";
+echo "[*] gathering sources...";
+$sources = fill_source_list();
+echo "\n[*] cleaning...";
clean_database();
-die();
+
foreach($sources as $s)
{
- echo "[*] {$s['dist']}-{$s['name']}, {$s['arch']}; Id: {$s['module_id']} ({$s['url']})\n";
- echo " [+] primary\n";
+ echo "\n[*] {$s['dist']}-{$s['name']}, {$s['arch']}; Id: {$s['module_id']} ({$s['url']})";
+ echo "\n [+] primary";
+ get_files($s['url'], "primary.xml.gz");
parse_primary($s['module_id']);
+ unlink("primary.xml");
- echo " [+] filelists\n";
+ echo "\n [+] filelists";
+ get_files($s['url'], "filelists.xml.gz");
parse_filelists();
+ unlink("filelists.xml");
}
?>
More information about the pld-cvs-commit
mailing list