Merge pull request #9032 from annando/local-access

Some more local network access are removed

diff --git a/bin/daemon.php b/bin/daemon.php
index c2ce05c..596f4de 100755 (executable)
--- a/bin/daemon.php
+++ b/bin/daemon.php
@@ -185,7 +185,12 @@ while (true) {
                $do_cron = true;
        }
 
-       Worker::spawnWorker($do_cron);
+       if ($do_cron || (!DI::process()->isMaxLoadReached() && Worker::entriesExists() && Worker::isReady())) {
+               Worker::spawnWorker($do_cron);
+       } else {
+               Logger::info('Cool down', ['pid' => $pid]);
+               sleep(10);
+       }
 
        if ($do_cron) {
                // We force a reconnect of the database connection.

diff --git a/database.sql b/database.sql
index e1a8f7f..f544d31 100644 (file)
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2020.09-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1361
+-- DB_UPDATE_VERSION 1362
 -- ------------------------------------------
 
 
@@ -1411,11 +1411,12 @@ CREATE VIEW `network-item-view` AS SELECT
        `item`.`contact-id` AS `contact-id`
        FROM `item`
                        INNER JOIN `thread` ON `thread`.`iid` = `item`.`parent`
-                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`)
+                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
                        LEFT JOIN `user-item` ON `user-item`.`iid` = `item`.`id` AND `user-item`.`uid` = `thread`.`uid`
                        LEFT JOIN `user-contact` AS `author` ON `author`.`uid` = `thread`.`uid` AND `author`.`cid` = `thread`.`author-id`
                        LEFT JOIN `user-contact` AS `owner` ON `owner`.`uid` = `thread`.`uid` AND `owner`.`cid` = `thread`.`owner-id`
                        WHERE `thread`.`visible` AND NOT `thread`.`deleted` AND NOT `thread`.`moderated`
+                       AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
                        AND (`user-item`.`hidden` IS NULL OR NOT `user-item`.`hidden`)
                        AND (`author`.`blocked` IS NULL OR NOT `author`.`blocked`)
                        AND (`owner`.`blocked` IS NULL OR NOT `owner`.`blocked`);
@@ -1438,12 +1439,13 @@ CREATE VIEW `network-thread-view` AS SELECT
        `thread`.`network` AS `network`,
        `thread`.`contact-id` AS `contact-id`
        FROM `thread`
-                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`)
+                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
                        STRAIGHT_JOIN `item` ON `item`.`id` = `thread`.`iid`
                        LEFT JOIN `user-item` ON `user-item`.`iid` = `item`.`id` AND `user-item`.`uid` = `thread`.`uid`
                        LEFT JOIN `user-contact` AS `author` ON `author`.`uid` = `thread`.`uid` AND `author`.`cid` = `thread`.`author-id`
                        LEFT JOIN `user-contact` AS `owner` ON `owner`.`uid` = `thread`.`uid` AND `owner`.`cid` = `thread`.`owner-id`
                        WHERE `thread`.`visible` AND NOT `thread`.`deleted` AND NOT `thread`.`moderated`
+                       AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
                        AND (`user-item`.`hidden` IS NULL OR NOT `user-item`.`hidden`)
                        AND (`author`.`blocked` IS NULL OR NOT `author`.`blocked`)
                        AND (`owner`.`blocked` IS NULL OR NOT `owner`.`blocked`);

diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index c9a8bcb..1cde613 100644 (file)
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -81,27 +81,8 @@ class Worker
                        self::killStaleWorkers();
                }
 
-               // Count active workers and compare them with a maximum value that depends on the load
-               if (self::tooMuchWorkers()) {
-                       Logger::info('Pre check: Active worker limit reached, quitting.');
-                       return;
-               }
-
-               // Do we have too few memory?
-               if (DI::process()->isMinMemoryReached()) {
-                       Logger::info('Pre check: Memory limit reached, quitting.');
-                       return;
-               }
-
-               // Possibly there are too much database connections
-               if (self::maxConnectionsReached()) {
-                       Logger::info('Pre check: maximum connections reached, quitting.');
-                       return;
-               }
-
-               // Possibly there are too much database processes that block the system
-               if (DI::process()->isMaxProcessesReached()) {
-                       Logger::info('Pre check: maximum processes reached, quitting.');
+               // Check if the system is ready
+               if (!self::isReady()) {
                        return;
                }
 
@@ -174,13 +155,49 @@ class Worker
                Logger::info("Couldn't select a workerqueue entry, quitting process", ['pid' => getmypid()]);
        }
 
+       /**
+        * Checks if the system is ready.
+        *
+        * Several system parameters like memory, connections and processes are checked.
+        *
+        * @return boolean
+        */
+       public static function isReady()
+       {
+               // Count active workers and compare them with a maximum value that depends on the load
+               if (self::tooMuchWorkers()) {
+                       Logger::info('Active worker limit reached, quitting.');
+                       return false;
+               }
+
+               // Do we have too few memory?
+               if (DI::process()->isMinMemoryReached()) {
+                       Logger::info('Memory limit reached, quitting.');
+                       return false;
+               }
+
+               // Possibly there are too much database connections
+               if (self::maxConnectionsReached()) {
+                       Logger::info('Maximum connections reached, quitting.');
+                       return false;
+               }
+
+               // Possibly there are too much database processes that block the system
+               if (DI::process()->isMaxProcessesReached()) {
+                       Logger::info('Maximum processes reached, quitting.');
+                       return false;
+               }
+               
+               return true;
+       }
+
        /**
         * Check if non executed tasks do exist in the worker queue
         *
         * @return boolean Returns "true" if tasks are existing
         * @throws \Exception
         */
-       private static function entriesExists()
+       public static function entriesExists()
        {
                $stamp = (float)microtime(true);
                $exists = DBA::exists('workerqueue', ["NOT `done` AND `pid` = 0 AND `next_try` < ?", DateTimeFormat::utcNow()]);

diff --git a/src/Worker/ClearCache.php b/src/Worker/ClearCache.php
index d87fd46..a1e7843 100644 (file)
--- a/src/Worker/ClearCache.php
+++ b/src/Worker/ClearCache.php
@@ -21,7 +21,6 @@
 
 namespace Friendica\Worker;
 
-use Friendica\Core\Logger;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Photo;
@@ -35,21 +34,12 @@ class ClearCache
        public static function execute()
        {
                $a = DI::app();
-               $last = DI::config()->get('system', 'cache_last_cleared');
-
-               if ($last) {
-                       $next = $last + (3600); // Once per hour
-                       $clear_cache = ($next <= time());
-               } else {
-                       $clear_cache = true;
-               }
-
-               if (!$clear_cache) {
-                       return;
-               }
 
                // clear old cache
                DI::cache()->clear();
+               if (DI::config()->get('system', 'optimize_tables')) {
+                       DBA::e("OPTIMIZE TABLE `cache`");
+               }
 
                // clear old item cache files
                clear_cache();
@@ -76,25 +66,14 @@ class ClearCache
 
                // Delete the cached OEmbed entries that are older than three month
                DBA::delete('oembed', ["`created` < NOW() - INTERVAL 3 MONTH"]);
+               if (DI::config()->get('system', 'optimize_tables')) {
+                       DBA::e("OPTIMIZE TABLE `oembed`");
+               }
 
                // Delete the cached "parse_url" entries that are older than three month
                DBA::delete('parsed_url', ["`created` < NOW() - INTERVAL 3 MONTH"]);
-
                if (DI::config()->get('system', 'optimize_tables')) {
-                       Logger::info('Optimize start');
-                       DBA::e("OPTIMIZE TABLE `auth_codes`");
-                       DBA::e("OPTIMIZE TABLE `cache`");
-                       DBA::e("OPTIMIZE TABLE `challenge`");
-                       DBA::e("OPTIMIZE TABLE `locks`");
-                       DBA::e("OPTIMIZE TABLE `oembed`");
                        DBA::e("OPTIMIZE TABLE `parsed_url`");
-                       DBA::e("OPTIMIZE TABLE `profile_check`");
-                       DBA::e("OPTIMIZE TABLE `session`");
-                       DBA::e("OPTIMIZE TABLE `tokens`");
-                       DBA::e("OPTIMIZE TABLE `process`");
-                       Logger::info('Optimize finished');
                }
-
-               DI::config()->set('system', 'cache_last_cleared', time());
        }
 }

diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php
index d7cd7c0..762dead 100644 (file)
--- a/src/Worker/Cron.php
+++ b/src/Worker/Cron.php
@@ -63,9 +63,6 @@ class Cron
                // Call possible post update functions
                Worker::add(PRIORITY_LOW, 'PostUpdate');
 
-               // Clear cache entries
-               Worker::add(PRIORITY_LOW, 'ClearCache');
-
                // Repair entries in the database
                Worker::add(PRIORITY_LOW, 'RepairDatabase');
 
@@ -94,6 +91,10 @@ class Cron
 
                        self::checkdeletedContacts();
 
+                       if (DI::config()->get('system', 'optimize_tables')) {
+                               self::optimizeTables();
+                       }
+       
                        DI::config()->set('system', 'last_expire_day', $d2);
                }
 
@@ -110,9 +111,20 @@ class Cron
 
                        // Optimizing this table only last seconds
                        if (DI::config()->get('system', 'optimize_tables')) {
-                               DBA::e("OPTIMIZE TABLE `workerqueue`");
+                               // We are acquiring the two locks from the worker to avoid locking problems
+                               if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
+                                       if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
+                                               DBA::e("OPTIMIZE TABLE `workerqueue`");
+                                               DBA::e("OPTIMIZE TABLE `process`");                     
+                                               DI::lock()->release(Worker::LOCK_WORKER);
+                                       }
+                                       DI::lock()->release(Worker::LOCK_PROCESS);
+                               }
                        }
 
+                       // Clear cache entries
+                       Worker::add(PRIORITY_LOW, 'ClearCache');
+
                        DI::config()->set('system', 'last_cron_hourly', time());
                }
 
@@ -136,6 +148,25 @@ class Cron
                return;
        }
 
+       /**
+        * Optimize tables that are known to grow and shrink all the time
+        *
+        * @return void
+        */
+       private static function optimizeTables()
+       {
+               Logger::info('Optimize start');
+
+               DBA::e("OPTIMIZE TABLE `auth_codes`");
+               DBA::e("OPTIMIZE TABLE `challenge`");
+               DBA::e("OPTIMIZE TABLE `locks`");
+               DBA::e("OPTIMIZE TABLE `profile_check`");
+               DBA::e("OPTIMIZE TABLE `session`");
+               DBA::e("OPTIMIZE TABLE `tokens`");
+
+               DI::lock()->release('optimize_tables');
+       }
+
        /**
         * Checks for contacts that are about to be deleted and ensures that they are removed.
         * This should be done automatically in the "remove" function. This here is a cleanup job.

diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php
index b5849fb..045d0be 100755 (executable)
--- a/static/dbstructure.config.php
+++ b/static/dbstructure.config.php
@@ -54,7 +54,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-       define('DB_UPDATE_VERSION', 1361);
+       define('DB_UPDATE_VERSION', 1362);
 }
 
 return [

diff --git a/static/dbview.config.php b/static/dbview.config.php
index b27e3a7..a508936 100755 (executable)
--- a/static/dbview.config.php
+++ b/static/dbview.config.php
@@ -86,11 +86,12 @@ return [
                ],
                "query" => "FROM `item`
                        INNER JOIN `thread` ON `thread`.`iid` = `item`.`parent`
-                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`)
+                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
                        LEFT JOIN `user-item` ON `user-item`.`iid` = `item`.`id` AND `user-item`.`uid` = `thread`.`uid`
                        LEFT JOIN `user-contact` AS `author` ON `author`.`uid` = `thread`.`uid` AND `author`.`cid` = `thread`.`author-id`
                        LEFT JOIN `user-contact` AS `owner` ON `owner`.`uid` = `thread`.`uid` AND `owner`.`cid` = `thread`.`owner-id`
                        WHERE `thread`.`visible` AND NOT `thread`.`deleted` AND NOT `thread`.`moderated`
+                       AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
                        AND (`user-item`.`hidden` IS NULL OR NOT `user-item`.`hidden`)
                        AND (`author`.`blocked` IS NULL OR NOT `author`.`blocked`)
                        AND (`owner`.`blocked` IS NULL OR NOT `owner`.`blocked`)"
@@ -111,12 +112,13 @@ return [
                        "contact-id" => ["thread", "contact-id"],
                ],
                "query" => "FROM `thread`
-                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id` AND (NOT `contact`.`blocked` OR `contact`.`pending`)
+                       STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
                        STRAIGHT_JOIN `item` ON `item`.`id` = `thread`.`iid`
                        LEFT JOIN `user-item` ON `user-item`.`iid` = `item`.`id` AND `user-item`.`uid` = `thread`.`uid`
                        LEFT JOIN `user-contact` AS `author` ON `author`.`uid` = `thread`.`uid` AND `author`.`cid` = `thread`.`author-id`
                        LEFT JOIN `user-contact` AS `owner` ON `owner`.`uid` = `thread`.`uid` AND `owner`.`cid` = `thread`.`owner-id`
                        WHERE `thread`.`visible` AND NOT `thread`.`deleted` AND NOT `thread`.`moderated`
+                       AND (NOT `contact`.`readonly` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`)
                        AND (`user-item`.`hidden` IS NULL OR NOT `user-item`.`hidden`)
                        AND (`author`.`blocked` IS NULL OR NOT `author`.`blocked`)
                        AND (`owner`.`blocked` IS NULL OR NOT `owner`.`blocked`)"