searxng · return42 · Nov 1, 2023 · Oct 2, 2023 · Oct 2, 2023
diff --git a/docs/admin/index.rst b/docs/admin/index.rst
@@ -15,7 +15,7 @@ Administrator documentation
    installation-apache
    update-searxng
    answer-captcha
-   searx.botdetection
+   searx.limiter
    api
    architecture
    plugins

diff --git a/docs/admin/searx.limiter.rst b/docs/admin/searx.limiter.rst
@@ -0,0 +1,17 @@
+.. _limiter:
+
+=======
+Limiter
+=======
+
+.. sidebar:: info
+
+   The limiter requires a :ref:`Redis <settings redis>` database.
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.limiter
+   :members:
diff --git a/docs/admin/settings/settings_server.rst b/docs/admin/settings/settings_server.rst
@@ -12,6 +12,7 @@
        bind_address: "127.0.0.1"
        secret_key: "ultrasecretkey"           # change this!
        limiter: false
+       public_instance: false
        image_proxy: false
        default_http_headers:
          X-Content-Type-Options : nosniff
@@ -20,7 +21,6 @@
          X-Robots-Tag : noindex, nofollow
          Referrer-Policy : no-referrer
 
-
 ``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv <make buildenv>`
   The base URL where SearXNG is deployed.  Used to create correct inbound links.
   If you change the value, don't forget to rebuild instance's environment
@@ -36,11 +36,19 @@
 ``secret_key`` : ``$SEARXNG_SECRET``
   Used for cryptography purpose.
 
-.. _limiter:
-
 ``limiter`` :
   Rate limit the number of request on the instance, block some bots.  The
-  :ref:`limiter src` requires a :ref:`settings redis` database.
+  :ref:`limiter` requires a :ref:`settings redis` database.
+
+.. _public_instance:
+
+``public_instance`` :
+
+  Setting that allows to enable features specifically for public instances (not
+  needed for local usage).  By set to ``true`` the following features are
+  activated:
+
+  - :py:obj:`searx.botdetection.link_token` in the :ref:`limiter`
 
 .. _image_proxy:
 

diff --git a/docs/conf.py b/docs/conf.py
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import  sys, os
+from pathlib import Path
 from pallets_sphinx_themes import ProjectLink
 
 from searx import get_setting
@@ -13,7 +14,6 @@
 copyright = 'SearXNG team'
 author = 'SearXNG team'
 release, version = VERSION_STRING, VERSION_STRING
-
 SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng'
 ISSUE_URL = get_setting('brand.issue_url')
 DOCS_URL = get_setting('brand.docs_url')
@@ -22,6 +22,9 @@
 CONTACT_URL = get_setting('general.contact_url')
 WIKI_URL = get_setting('brand.wiki_url')
 
+SOURCEDIR = Path(__file__).parent.parent / "searx"
+os.environ['SOURCEDIR'] = str(SOURCEDIR)
+
 # hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set
 #       to string 'none' [2]
 #

diff --git a/docs/admin/searx.botdetection.rst → docs/src/searx.botdetection.rst b/docs/admin/searx.botdetection.rst → docs/src/searx.botdetection.rst
@@ -12,8 +12,10 @@ Bot Detection
 .. automodule:: searx.botdetection
   :members:
 
-.. automodule:: searx.botdetection.limiter
-  :members:
+.. _botdetection ip_lists:
+
+IP lists
+========
 
 .. automodule:: searx.botdetection.ip_lists
   :members:
@@ -50,3 +52,11 @@ Probe HTTP headers
 
 .. automodule:: searx.botdetection.http_user_agent
   :members:
+
+.. _botdetection config:
+
+Config
+======
+
+.. automodule:: searx.botdetection.config
+  :members:
diff --git a/searx/__init__.py b/searx/__init__.py
@@ -108,5 +108,6 @@ def logging_config_debug():
 if settings['server']['public_instance']:
     logger.warning(
         "Be aware you have activated features intended only for public instances. "
-        + "This force the usage of the bot limiter and link_token plugins."
+        "This force the usage of the limiter and link_token / "
+        "see https://docs.searxng.org/admin/searx.limiter.html"
     )
diff --git a/searx/botdetection/__init__.py b/searx/botdetection/__init__.py
@@ -2,43 +2,22 @@
 # lint: pylint
 """.. _botdetection src:
 
-The :ref:`limiter <limiter src>` implements several methods to block bots:
-
-a. Analysis of the HTTP header in the request / can be easily bypassed.
-
-b. Block and pass lists in which IPs are listed / difficult to maintain, since
-   the IPs of bots are not all known and change over the time.
-
-c. Detection of bots based on the behavior of the requests and blocking and, if
-   necessary, unblocking of the IPs via a dynamically changeable IP block list.
-
-For dynamically changeable IP lists a Redis database is needed and for any kind
-of IP list the determination of the IP of the client is essential.  The IP of
-the client is determined via the X-Forwarded-For_ HTTP header
-
-.. _X-Forwarded-For:
-   https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
-
-X-Forwarded-For
-===============
-
-.. attention::
-
-   A correct setup of the HTTP request headers ``X-Forwarded-For`` and
-   ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
-
-   - `NGINX RequestHeader`_
-   - `Apache RequestHeader`_
-
-.. _NGINX RequestHeader:
-    https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
-.. _Apache RequestHeader:
-    https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
-
-.. autofunction:: searx.botdetection.get_real_ip
+Implementations used for bot detection.
 
 """
 
 from ._helpers import dump_request
 from ._helpers import get_real_ip
+from ._helpers import get_network
 from ._helpers import too_many_requests
+
+__all__ = ['dump_request', 'get_network', 'get_real_ip', 'too_many_requests']
+
+redis_client = None
+cfg = None
+
+
+def init(_cfg, _redis_client):
+    global redis_client, cfg  # pylint: disable=global-statement
+    redis_client = _redis_client
+    cfg = _cfg
diff --git a/searx/botdetection/_helpers.py b/searx/botdetection/_helpers.py
@@ -13,8 +13,8 @@
 import flask
 import werkzeug
 
-from searx.tools import config
 from searx import logger
+from . import config
 
 logger = logger.getChild('botdetection')
 
@@ -104,10 +104,10 @@ def get_real_ip(request: flask.Request) -> str:
     if not forwarded_for:
         _log_error_only_once("X-Forwarded-For header is not set!")
     else:
-        from .limiter import get_cfg  # pylint: disable=import-outside-toplevel, cyclic-import
+        from . import cfg  # pylint: disable=import-outside-toplevel, cyclic-import
 
         forwarded_for = [x.strip() for x in forwarded_for.split(',')]
-        x_for: int = get_cfg()['real_ip.x_for']  # type: ignore
+        x_for: int = cfg['real_ip.x_for']  # type: ignore
         forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
 
     if not real_ip:

diff --git a/searx/tools/config.py → searx/botdetection/config.py b/searx/tools/config.py → searx/botdetection/config.py
diff --git a/searx/botdetection/http_accept.py b/searx/botdetection/http_accept.py
@@ -24,7 +24,7 @@
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

diff --git a/searx/botdetection/http_accept_encoding.py b/searx/botdetection/http_accept_encoding.py
@@ -25,7 +25,7 @@
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

diff --git a/searx/botdetection/http_accept_language.py b/searx/botdetection/http_accept_language.py
@@ -21,7 +21,7 @@
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

diff --git a/searx/botdetection/http_connection.py b/searx/botdetection/http_connection.py
@@ -22,7 +22,7 @@
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

diff --git a/searx/botdetection/http_user_agent.py b/searx/botdetection/http_user_agent.py
@@ -24,7 +24,7 @@
 import flask
 import werkzeug
 
-from searx.tools import config
+from . import config
 from ._helpers import too_many_requests
 
 

diff --git a/searx/botdetection/ip_limit.py b/searx/botdetection/ip_limit.py
@@ -13,8 +13,7 @@
 
 The :py:obj:`.link_token` method can be used to investigate whether a request is
 *suspicious*.  To activate the :py:obj:`.link_token` method in the
-:py:obj:`.ip_limit` method add the following to your
-``/etc/searxng/limiter.toml``:
+:py:obj:`.ip_limit` method add the following configuration:
 
 .. code:: toml
 
@@ -46,13 +45,12 @@
 
 import flask
 import werkzeug
-from searx.tools import config
-from searx import settings
 
 from searx import redisdb
 from searx.redislib import incr_sliding_window, drop_counter
 
 from . import link_token
+from . import config
 from ._helpers import (
     too_many_requests,
     logger,
@@ -110,7 +108,7 @@ def filter_request(
         if c > API_MAX:
             return too_many_requests(network, "too many request in API_WINDOW")
 
-    if settings['server']['public_instance'] or cfg['botdetection.ip_limit.link_token']:
+    if cfg['botdetection.ip_limit.link_token']:
 
         suspicious = link_token.is_suspicious(network, request, True)
 

diff --git a/searx/botdetection/ip_lists.py b/searx/botdetection/ip_lists.py
@@ -33,7 +33,7 @@
     IPv6Address,
 )
 
-from searx.tools import config
+from . import config
 from ._helpers import logger
 
 logger = logger.getChild('ip_limit')

diff --git a/searx/botdetection/link_token.py b/searx/botdetection/link_token.py
@@ -99,15 +99,13 @@ def ping(request: flask.Request, token: str):
     The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
 
     """
-    from . import limiter  # pylint: disable=import-outside-toplevel, cyclic-import
+    from . import redis_client, cfg  # pylint: disable=import-outside-toplevel, cyclic-import
 
-    redis_client = redisdb.client()
     if not redis_client:
         return
     if not token_is_valid(token):
         return
 
-    cfg = limiter.get_cfg()
     real_ip = ip_address(get_real_ip(request))
     network = get_network(real_ip, cfg)