[change] Updated modules, fixed build and tests

Updated openwisp modules, base images,
django (4.2) and postgres (12).
Fixed selenium tests.

Author: nemesifier

.github/workflows/branch.yml

@@ -15,7 +15,9 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install Testing Requirements
-        run: sudo pip install -r requirements-test.txt
+        run: |
+          sudo pip install -U pip setuptools wheel
+          sudo pip install -r requirements-test.txt
 
       - name: Lint
         run: openwisp-qa-check --skip-checkmigrations

docker-compose.yml

@@ -174,7 +174,7 @@ services:
       - NET_ADMIN
 
   postgres:
-    image: mdillon/postgis:11-alpine
+    image: postgis/postgis:12-3.4
     restart: always
     environment:
       - POSTGRES_DB=$DB_NAME

images/common/openwisp/settings.py

@@ -265,7 +265,7 @@
 EMAIL_HOST_USER = os.environ['EMAIL_HOST_USER']
 EMAIL_HOST_PASSWORD = os.environ['EMAIL_HOST_PASSWORD']
 EMAIL_USE_TLS = env_bool(os.environ['EMAIL_HOST_TLS'])
-EMAIL_TIMEOUT = os.environ['EMAIL_TIMEOUT']
+EMAIL_TIMEOUT = int(os.environ['EMAIL_TIMEOUT'])
 
 # Logging
 # http://docs.djangoproject.com/en/dev/topics/logging

images/openwisp_base/Dockerfile

@@ -25,18 +25,18 @@ FROM system AS openwisp_python
 ENV PATH="${PATH}:/home/openwisp/.local/bin"
 ENV PYTHONPATH=/home/openwisp/.local/lib/python3.10/site-packages
 
-RUN pip install --no-cache-dir  --user --upgrade pip~=23.0.1
+RUN pip install --no-cache-dir --user --upgrade pip~=24.0.0
 # TODO: Remove when next version of openwisp-monitoring is released
-ARG OPENWISP_MONITORING_SOURCE=https://github.com/openwisp/openwisp-monitoring/tarball/15977c74b48d846ac469316abc7f13ccbef405f0
+ARG OPENWISP_MONITORING_SOURCE=https://github.com/openwisp/openwisp-monitoring/tarball/18cc5249de1f057554826f9d6e6ed8683e3f24bc
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_MONITORING_SOURCE}
-ARG OPENWISP_FIRMWARE_SOURCE=https://github.com/openwisp/openwisp-firmware-upgrader/tarball/9c7840c7436eb8b8cd5a63e83194041ccf887cd2
+ARG OPENWISP_FIRMWARE_SOURCE=https://github.com/openwisp/openwisp-firmware-upgrader/tarball/5a104a8f06fd566b9c09dddb93005f223df0bb47
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_FIRMWARE_SOURCE}
-ARG OPENWISP_TOPOLOGY_SOURCE=https://github.com/openwisp/openwisp-network-topology/tarball/ade4d5f844753879ccb54cce1d3528f17c55a490
+ARG OPENWISP_TOPOLOGY_SOURCE=https://github.com/openwisp/openwisp-network-topology/tarball/d150127e6ee244b2c25aa2d35f804d3d1124bde4
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_TOPOLOGY_SOURCE}
-ARG OPENWISP_RADIUS_SOURCE=https://github.com/openwisp/openwisp-radius/tarball/1a748466c269df3548557d9d42df6bb886da4f57
+ARG OPENWISP_RADIUS_SOURCE=https://github.com/openwisp/openwisp-radius/tarball/6a2d98861a8d43e7622d87f7e1af97503c2677f9
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${OPENWISP_RADIUS_SOURCE}
 
@@ -49,7 +49,7 @@ RUN if [ "$OPENWISP_IPAM_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_IPAM_SOURCE}; \
     fi
 # TODO: Remove when next version of openwisp-controller is released
-ARG OPENWISP_CONTROLLER_SOURCE=https://github.com/openwisp/openwisp-controller/tarball/3383bc11e5f8e9c89ebd9ac8aefa0066576d3133
+ARG OPENWISP_CONTROLLER_SOURCE=https://github.com/openwisp/openwisp-controller/tarball/1e641fe094838178556b4db09317065bd28c625c
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_CONTROLLER_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_CONTROLLER_SOURCE}; \
@@ -59,13 +59,13 @@ ARG OPENWISP_NOTIFICATION_SOURCE=default
 RUN if [ "$OPENWISP_NOTIFICATION_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade ${OPENWISP_NOTIFICATION_SOURCE}; \
     fi
-ARG OPENWISP_USERS_SOURCE=https://github.com/openwisp/openwisp-users/tarball/55023698cf39711585480be90c98d64ce5a04b8d
+ARG OPENWISP_USERS_SOURCE=https://github.com/openwisp/openwisp-users/tarball/8a39e488b0b955c7322f6df7927d2adb71ce7caf
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_USERS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall ${OPENWISP_USERS_SOURCE}; \
     fi
 # TODO: Remove when next version of openwisp-utils is released
-ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/9fd347fb861af674674a4bcc59c54bcdeef18b8c "
+ARG OPENWISP_UTILS_SOURCE="openwisp-utils[celery,rest] @ https://github.com/openwisp/openwisp-utils/tarball/1d3b00cbd8b36686a39b3f24c42667e6482197b2"
 # hadolint ignore=DL3013
 RUN if [ "$OPENWISP_UTILS_SOURCE" != "default" ] ; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall "${OPENWISP_UTILS_SOURCE}"; \
@@ -76,7 +76,7 @@ RUN if [ "$DJANGO_X509_SOURCE" != "default" ]; then \
         pip install --no-cache-dir --user --upgrade --force-reinstall ${DJANGO_X509_SOURCE}; \
     fi
 
-ARG DJANGO_SOURCE=django~=4.0.10
+ARG DJANGO_SOURCE=django~=4.2.0
 # hadolint ignore=DL3013
 RUN pip install --no-cache-dir --user --upgrade ${DJANGO_SOURCE}
 

images/openwisp_freeradius/Dockerfile

@@ -1,8 +1,8 @@
 FROM freeradius/freeradius-server:3.0.26-alpine
 
 # hadolint ignore=DL3018
-RUN apk add --no-cache --update tzdata~=2022f-r1 postgresql-dev~=13.8-r0 \
-                                postgresql-client~=13.8-r0 && \
+RUN apk add --no-cache --update tzdata~=2022f-r1 \
+                                postgresql-client~=13.12-r0 && \
     rm -rf /var/cache/apk/* /tmp/*
 
 RUN addgroup -S freerad && \

images/openwisp_nfs/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.16
 
 # hadolint ignore=DL3018
 RUN apk add --no-cache --update --verbose \
-            tzdata~=2022f-r1 \
+            tzdata~=2024a-r0 \
             nfs-utils~=2.6.1-r1 && \
     rm -rf /var/cache/apk/* /tmp/*
 

images/openwisp_nginx/Dockerfile

@@ -1,7 +1,7 @@
 FROM nginx:1.23.3-alpine
 
 RUN apk add --update --no-cache \
-            openssl~=3.0.8-r0 \
+            openssl~=3.0.12-r5 \
             certbot~=1.32.0-r0 \
             certbot-nginx~=1.32.0-r0 && \
     rm -rf /var/cache/apk/* /tmp/*

images/openwisp_postfix/Dockerfile

@@ -2,13 +2,13 @@ FROM alpine:3.16
 
 WORKDIR /opt/openwisp/
 RUN apk add --no-cache --upgrade \
-            openssl~=1.1.1t-r0 \
+            openssl~=1.1.1w-r1 \
             cyrus-sasl~=2.1.28-r1	\
             cyrus-sasl-login~=2.1.28-r1 && \
     apk add --no-cache \
-            postfix~=3.7.3-r0 \
+            postfix~=3.7.11-r0 \
             rsyslog~=8.2204.1-r0	 \
-            tzdata~=2022f-r1 && \
+            tzdata~=2024a-r0 && \
     rm -rf /tmp/* /var/cache/apk/*
 
 CMD ["sh", "init_command.sh"]

requirements-test.txt

@@ -1,3 +1,2 @@
-docker~=5.0.3
-selenium~=4.1.0
-openwisp-utils[qa] @ https://github.com/openwisp/openwisp-utils/tarball/master
+docker~=7.0.0
+openwisp-utils[qa,selenium] @ https://github.com/openwisp/openwisp-utils/tarball/master

tests/runtests.py

@@ -9,6 +9,7 @@
 from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.chrome.options import Options as ChromiumOptions
 from selenium.webdriver.common.by import By
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from utils import TestUtilities
 
 
@@ -120,13 +121,17 @@ def setUpClass(cls):
             )
         # Create base drivers (Chromium)
         if cls.config['driver'] == 'chromium':
-            chrome_options = ChromiumOptions()
-            chrome_options.set_capability("goog:loggingPrefs", {'browser': 'ALL'})
-            chrome_options.add_argument('--ignore-certificate-errors')
+            options = ChromiumOptions()
+            options.add_argument('--headless')
+            options.add_argument('--ignore-certificate-errors')
             if cls.config['headless']:
-                chrome_options.add_argument('--headless')
-            cls.base_driver = webdriver.Chrome(options=chrome_options)
-            cls.second_driver = webdriver.Chrome(options=chrome_options)
+                options.add_argument('--headless')
+            options.add_argument(f'--remote-debugging-port={5003 + 100}')
+            capabilities = DesiredCapabilities.CHROME
+            capabilities['goog:loggingPrefs'] = {'browser': 'ALL'}
+            options.set_capability('cloud:options', capabilities)
+            cls.base_driver = webdriver.Chrome(options=options)
+            cls.second_driver = webdriver.Chrome(options=options)
         cls.base_driver.set_window_size(1366, 768)
         cls.second_driver.set_window_size(1366, 768)
 
@@ -137,8 +142,8 @@ def tearDownClass(cls):
                 cls._delete_object(resource_link)
             except NoSuchElementException:
                 print(f'Unable to delete resource at: {resource_link}')
-        cls.second_driver.close()
-        cls.base_driver.close()
+        cls.second_driver.quit()
+        cls.base_driver.quit()
         if cls.failed_test and cls.config['logs']:
             cmd = subprocess.Popen(
                 ['docker-compose', 'logs'],
@@ -170,8 +175,9 @@ def test_topology_graph(self):
         self.action_on_resource(label, path, 'delete_selected')
         self.assertNotIn('<li>Nodes: ', self.base_driver.page_source)
         self.action_on_resource(label, path, 'update_selected')
-        time.sleep(4)  # Wait for nodes to be fetched!
+
         self.action_on_resource(label, path, 'delete_selected')
+        self._wait_for_element()
         self.assertIn('<li>Nodes: ', self.base_driver.page_source)
 
     def test_admin_login(self):
@@ -194,21 +200,21 @@ def test_create_prefix_users(self):
         self.base_driver.get(
             f"{self.config['app_url']}/admin/openwisp_radius/radiusbatch/add/"
         )
-        self.base_driver.find_element_by_name('strategy').find_element_by_xpath(
-            '//option[@value="prefix"]'
+        self.base_driver.find_element(By.NAME, 'strategy').find_element(
+            By.XPATH, '//option[@value="prefix"]'
         ).click()
-        self.base_driver.find_element_by_name('organization').find_element_by_xpath(
-            '//option[text()="default"]'
+        self.base_driver.find_element(By.NAME, 'organization').find_element(
+            By.XPATH, '//option[text()="default"]'
         ).click()
-        self.base_driver.find_element_by_name('name').send_keys(prefix_objname)
-        self.base_driver.find_element_by_name('prefix').send_keys('automated-prefix')
-        self.base_driver.find_element_by_name('number_of_users').send_keys('1')
-        self.base_driver.find_element_by_name('_save').click()
+        self.base_driver.find_element(By.NAME, 'name').send_keys(prefix_objname)
+        self.base_driver.find_element(By.NAME, 'prefix').send_keys('automated-prefix')
+        self.base_driver.find_element(By.NAME, 'number_of_users').send_keys('1')
+        self.base_driver.find_element(By.NAME, '_save').click()
         # Check PDF available
         self.get_resource(prefix_objname, '/admin/openwisp_radius/radiusbatch/')
         self.objects_to_delete.append(self.base_driver.current_url)
-        prefix_pdf_file_path = self.base_driver.find_element_by_xpath(
-            '//a[text()="Download User Credentials"]'
+        prefix_pdf_file_path = self.base_driver.find_element(
+            By.XPATH, '//a[text()="Download User Credentials"]'
         ).get_property('href')
         reqHeader = {
             'Cookie': f"sessionid={self.base_driver.get_cookies()[0]['value']}"
@@ -314,6 +320,7 @@ def test_forgot_password(self):
         self.base_driver.get(f"{self.config['app_url']}/accounts/password/reset/")
         self.base_driver.find_element(By.NAME, 'email').send_keys('[email protected]')
         self.base_driver.find_element(By.XPATH, '//input[@type="submit"]').click()
+        self._wait_for_element()
         self.assertIn(
             'We have sent you an e-mail. If you have not received '
             'it please check your spam folder. Otherwise contact us '
@@ -355,13 +362,14 @@ def test_celery(self):
             "openwisp_monitoring.check.tasks.run_checks",
             "openwisp_monitoring.device.tasks.delete_wifi_clients_and_sessions",
             "openwisp_monitoring.device.tasks.offline_device_close_session",
-            "openwisp_monitoring.device.tasks.save_wifi_clients_and_sessions",
             "openwisp_monitoring.device.tasks.trigger_device_checks",
             "openwisp_monitoring.device.tasks.write_device_metrics",
+            "openwisp_monitoring.device.tasks.handle_disabled_organization",
             "openwisp_monitoring.monitoring.tasks.delete_timeseries",
             "openwisp_monitoring.monitoring.tasks.migrate_timeseries_database",
             "openwisp_monitoring.monitoring.tasks.timeseries_batch_write",
             "openwisp_monitoring.monitoring.tasks.timeseries_write",
+            "openwisp_monitoring.monitoring.tasks.delete_timeseries",
             "openwisp_notifications.tasks.delete_ignore_object_notification",
             "openwisp_notifications.tasks.delete_notification",
             "openwisp_notifications.tasks.delete_obsolete_objects",
@@ -392,8 +400,8 @@ def _test_celery_task_registered(container_name):
             for expected_output in expected_output_list:
                 if expected_output not in output:
                     self.fail(
-                        'Not all celery / celery-beat tasks are registered\n'
-                        f'Output:\n{output}\n'
+                        'Not all celery / celery-beat tasks are registered.\n'
+                        f'Expected celery task not found:\n{expected_output}'
                     )
 
         with self.subTest('Test celery container'):

tests/utils.py

@@ -8,9 +8,11 @@
 from selenium.common.exceptions import NoAlertPresentException
 from selenium.webdriver import ActionChains
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
 
 
-class TestConfig(object):
+class TestConfig:
     """
     Get the configurations that are to be used for all the tests.
     """
@@ -96,6 +98,7 @@ def create_superuser(
         if not driver:
             driver = self.base_driver
         driver.get(f"{self.config['app_url']}/admin/openwisp_users/user/add/")
+        self._wait_for_element()
         driver.find_element(By.NAME, 'username').send_keys(username)
         driver.find_element(By.NAME, 'email').send_keys(email)
         driver.find_element(By.NAME, 'password1').send_keys(password)
@@ -104,6 +107,7 @@ def create_superuser(
         self._click_save_btn(driver)
         self.objects_to_delete.append(driver.current_url)
         self._click_save_btn(driver)
+        self._wait_for_element()
 
     def get_resource(self, resource_name, path, select_field='field-name', driver=None):
         """
@@ -195,8 +199,10 @@ def create_mobile_location(self, location_name, driver=None):
         self._click_save_btn(driver)
         # Add to delete list
         self.get_resource(location_name, '/admin/geo/location/', driver=driver)
+        self._wait_for_element()
         self.objects_to_delete.append(driver.current_url)
         driver.get(f"{self.config['app_url']}/admin/geo/location/")
+        self._wait_for_element()
 
     def add_mobile_location_point(self, location_name, driver=None):
         """
@@ -261,5 +267,12 @@ def create_network_topology(
         self.get_resource(
             label, '/admin/topology/topology/', 'field-label', driver=driver
         )
+        self._wait_for_element()
         self.objects_to_delete.append(driver.current_url)
         driver.get(f"{self.config['app_url']}/admin/topology/topology/")
+        self._wait_for_element()
+
+    def _wait_for_element(self, element_id='content'):
+        WebDriverWait(self.base_driver, 10).until(
+            EC.visibility_of_element_located((By.ID, element_id))
+        )