Performance improvements for the poller

diff --git a/boot.php b/boot.php
index 6791162..208293a 100644 (file)
--- a/boot.php
+++ b/boot.php
@@ -1266,8 +1266,20 @@ class App {
        function proc_run($args) {
 
                // Add the php path if it is a php call
-               if (count($args) && ($args[0] === 'php' OR is_int($args[0])))
+               if (count($args) && ($args[0] === 'php' OR is_int($args[0]))) {
+
+                       // If the last worker fork was less than 10 seconds before then don't fork another one.
+                       // This should prevent the forking of masses of workers.
+                       if (get_config("system", "worker")) {
+                               if ((time() - get_config("system", "proc_run_started")) < 10)
+                                       return;
+
+                               // Set the timestamp of the last proc_run
+                               set_config("system", "proc_run_started", time());
+                       }
+
                        $args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php');
+               }
 
                // add baseurl to args. cli scripts can't construct it
                $args[] = $this->get_baseurl();

diff --git a/include/poller.php b/include/poller.php
index ecdb9eb..1f4d257 100644 (file)
--- a/include/poller.php
+++ b/include/poller.php
@@ -239,10 +239,13 @@ function poller_kill_stale_workers() {
                        $max_duration_defaults = array(PRIORITY_SYSTEM => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180);
                        $max_duration = $max_duration_defaults[$pid["priority"]];
 
+                       $argv = json_decode($pid["parameter"]);
+                       $argv[0] = basename($argv[0]);
+
                        // How long is the process already running?
                        $duration = (time() - strtotime($pid["executed"])) / 60;
                        if ($duration > $max_duration) {
-                               logger("Worker process ".$pid["pid"]." (".$pid["parameter"].") took more than ".$max_duration." minutes. It will be killed now.");
+                               logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
                                posix_kill($pid["pid"], SIGTERM);
 
                                // We killed the stale process.
@@ -254,7 +257,7 @@ function poller_kill_stale_workers() {
                                        intval(PRIORITY_LOW),
                                        intval($pid["pid"]));
                        } else
-                               logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
+                               logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
                }
 }
 

diff --git a/include/post_update.php b/include/post_update.php
index 6918bf0..d45a51b 100644 (file)
--- a/include/post_update.php
+++ b/include/post_update.php
@@ -159,7 +159,7 @@ function post_update_1198() {
        logger("Start", LOGGER_DEBUG);
 
        // Check if the first step is done (Setting "author-id" and "owner-id" in the item table)
-       $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 1000");
+       $r = q("SELECT `author-link`, `owner-link`, `uid` FROM `item` WHERE `author-id` = 0 AND `owner-id` = 0 LIMIT 100");
        if (!$r) {
                // Are there unfinished entries in the thread table?
                $r = q("SELECT COUNT(*) AS `total` FROM `thread`

diff --git a/include/queue.php b/include/queue.php
index 9779a07..47a3596 100644 (file)
--- a/include/queue.php
+++ b/include/queue.php
@@ -45,55 +45,57 @@ function queue_run(&$argv, &$argc){
        $deadservers = array();
        $serverlist = array();
 
-       logger('queue: start');
+       if (!$queue_id) {
 
-       // Handling the pubsubhubbub requests
-       proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
+               logger('queue: start');
 
-       $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
+               // Handling the pubsubhubbub requests
+               proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
 
-       // If we are using the worker we don't need a delivery interval
-       if (get_config("system", "worker"))
-               $interval = false;
+               $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
 
-       $r = q("select * from deliverq where 1");
-       if($r) {
-               foreach($r as $rr) {
-                       logger('queue: deliverq');
-                       proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
-                       if($interval)
+               // If we are using the worker we don't need a delivery interval
+               if (get_config("system", "worker"))
+                       $interval = false;
+
+               $r = q("select * from deliverq where 1");
+               if($r) {
+                       foreach($r as $rr) {
+                               logger('queue: deliverq');
+                               proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
+                               if($interval)
                                @time_sleep_until(microtime(true) + (float) $interval);
+                       }
                }
-       }
 
-       $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
-               INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
-               WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
-       if($r) {
-               foreach($r as $rr) {
-                       logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
-                       logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
+               $r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
+                       INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
+                       WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
+               if($r) {
+                       foreach($r as $rr) {
+                               logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
+                               logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
+                       }
+                       q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
                }
-               q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
-       }
-
-       if($queue_id) {
-               $r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
-                       intval($queue_id)
-               );
-       }
-       else {
 
                // For the first 12 hours we'll try to deliver every 15 minutes
                // After that, we'll only attempt delivery once per hour.
 
                $r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
+       } else {
+               logger('queue: start for id '.$queue_id);
+
+               $r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
+                       intval($queue_id)
+               );
        }
-       if(! $r){
+
+       if (!$r){
                return;
        }
 
-       if(! $queue_id)
+       if (!$queue_id)
                call_hooks('queue_predeliver', $a, $r);
 
 
@@ -107,16 +109,17 @@ function queue_run(&$argv, &$argc){
                // queue_predeliver hooks may have changed the queue db details,
                // so check again if this entry still needs processing
 
-               if($queue_id) {
+               if($queue_id)
                        $qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
-                               intval($queue_id)
-                       );
-               }
-               else {
+                               intval($queue_id));
+               elseif (get_config("system", "worker")) {
+                       logger('Call queue for id '.$q_item['id']);
+                       proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
+                       continue;
+               } else
                        $qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
-                               intval($q_item['id'])
-                       );
-               }
+                               intval($q_item['id']));
+
                if(! count($qi))
                        continue;