From 52bd2a06079b0b80b207bf3361aa732b0060b1a1 Mon Sep 17 00:00:00 2001 From: Christoph Gasser Date: Wed, 23 Sep 2026 10:17:18 +0200 Subject: [PATCH] Add persistent PostgreSQL stack and SQLite migration --- .gitignore | 1 + README-Docker.md | 103 +++++++++++++-------------- TelcoPortal-README.md | 18 ++--- TelcoPortal-source/Dockerfile | 2 +- TelcoPortal-source/app.py | 26 ++++++- TelcoPortal-source/migrate_sqlite.py | 95 ++++++++++++++++++++++++ TelcoPortal-source/requirements.txt | 1 + docker-compose.yml | 38 ++++++++-- 8 files changed, 214 insertions(+), 70 deletions(-) create mode 100644 TelcoPortal-source/migrate_sqlite.py diff --git a/.gitignore b/.gitignore index 32263a7..6e418bf 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Live application data - real phone numbers, employee names, login credentials, session key. # Never commit these. The Docker volume recreates this folder at deploy time. /TelcoPortal-data/ +/data/ # Leftovers not meant to be tracked (see chat history for context). /_to_delete/ diff --git a/README-Docker.md b/README-Docker.md index cd3157d..5d71e70 100755 --- a/README-Docker.md +++ b/README-Docker.md @@ -1,65 +1,62 @@ -# Queensway Europe Telco Portal - Docker (runs locally on this PC) +# Queensway Europe Telco Portal - Docker -A Docker container you run directly on your own PC, same setup as TostaSklad. Requires -**Docker Desktop for Windows** (with WSL2 backend, the default) - download from -https://www.docker.com/products/docker-desktop/ if you don't have it yet. +The Docker stack contains two services: -## First run +- `telcoportal` - the Flask web application +- `db` - PostgreSQL 16, stored in a persistent Docker volume -Open a terminal (PowerShell) in this `QWE Telco` folder and run: +## First start and SQLite import -``` -docker compose up -d --build +1. Put the existing SQLite database at `data/telco.db` next to `docker-compose.yml`. +2. Optionally set a database password in a `.env` file: + + ```env + TELCO_DB_PASSWORD=replace-with-a-long-random-password + ``` + +3. Build and start the stack: + + ```powershell + docker compose up -d --build + ``` + +4. Open . + +Before the web server starts, `migrate_sqlite.py` creates the PostgreSQL schema and copies all +rows from `users`, `phones`, and `internet_connections`. IDs, password hashes, dates, comments, +and timestamps are retained. The import runs only while PostgreSQL has no application rows; all +later starts skip it without changing existing data. + +If `data/telco.db` is missing on the first start, the app exits instead of creating an empty +production database. Check the reason with `docker compose logs telcoportal`. + +After a successful import, the SQLite file is no longer used by the application. Keep an +offline backup until the migrated data has been checked. + +## Persistent data and backups + +PostgreSQL stores its files in the Docker named volume `telco-db-data`. Rebuilding or replacing +the containers does not delete this volume. The generated Flask session key is kept separately +in `telco-app-data`. + +Create a portable SQL backup: + +```powershell +docker compose exec -T db pg_dump -U telco -d telco -Fc > telco-backup.dump ``` -This builds the image from `TelcoPortal-source/` (Python/Flask app) and starts the container in -the background. - -Open **http://localhost:8092/** in your browser. - -The Mobile Phones table is pre-loaded from the one-time Excel import (see -`TelcoPortal-source/import_excel.py` and the main README). There is no login - anyone who can -reach the URL can view and edit. - -## Data persistence - -The `TelcoPortal-data/` folder next to this file is bind-mounted into the container so the -database survives container restarts, rebuilds, and updates: - -- `TelcoPortal-data/telco.db` - the SQLite database (both Mobile Phones and Internet - Connectivity tables live in this one file) - -**Back up by simply copying `TelcoPortal-data/telco.db`.** Nothing important lives inside the -container itself. +`docker compose down` keeps both volumes. Do **not** use `docker compose down -v` unless you +intend to delete the PostgreSQL database and application state. ## Common commands -``` -docker compose logs -f # follow app logs -docker compose ps # check container status -docker compose down # stop and remove the container (data folder is untouched) -docker compose up -d # start it again -``` - -## Updating after you change the source code - -``` +```powershell +docker compose ps +docker compose logs -f telcoportal +docker compose logs -f db +docker compose down +docker compose up -d docker compose up -d --build ``` -This rebuilds the image with your latest changes from `TelcoPortal-source/` and replaces the -running container. Your data in `TelcoPortal-data/` is not affected. - -## Changing the port - -Edit the `ports:` line in `docker-compose.yml`, e.g. `"80:8000"` to use the standard HTTP port -instead of 8092, then run `docker compose up -d` again. - -## Note on this setup - -This was written and validated by running the Flask app directly (not inside Docker - Docker -itself was not available where these files were generated), so the app logic, database, and -import script are confirmed working end-to-end. The Docker build/run itself could not be -executed from where these files were generated. Please run the "First run" command above and -let me know if anything doesn't come up cleanly (e.g. build errors, port conflicts) so it can be -fixed. +To use another web port, change `"8092:8000"` in `docker-compose.yml` and recreate the stack. diff --git a/TelcoPortal-README.md b/TelcoPortal-README.md index 5cefa10..ecb993b 100755 --- a/TelcoPortal-README.md +++ b/TelcoPortal-README.md @@ -6,9 +6,8 @@ Management web portal for: eligibility, current owner, venue) - Internet connectivity at each restaurant venue -Same deployment approach as TostaSklad/TostaPortal: a Docker container you run locally on this -PC with Docker Desktop, with all data kept in a bind-mounted folder outside the container. See -`README-Docker.md` for how to start/stop it. +The production Docker stack runs the app with PostgreSQL and retains its database in a named +Docker volume. See `README-Docker.md` for the initial SQLite import, startup, and backup steps. ## Folder layout @@ -18,8 +17,8 @@ QWE Telco/ README-Docker.md <- Docker instructions (start here) TelcoPortal-README.md <- this file TelcoPortal-source/ <- Flask app source (rebuilt into the Docker image) - TelcoPortal-data/ - telco.db <- the SQLite database (both tables live in this one file) + data/ + telco.db <- legacy SQLite file, used for the first import only ``` ## Data model @@ -31,13 +30,13 @@ Replacement, Current Owner, Venue. Expiration, Notes. This table starts empty - the source Excel file only had mobile phone data - add rows with the "+ Add Connection" button on the Internet Connectivity page as you go. -There is no login. Anyone who can reach the URL (`http://localhost:8092/` by default) can view -and edit both tables. +The portal is available at `http://localhost:8092/` by default and requires a configured user +login. ## How the mobile phone data was loaded The one-off migration script `TelcoPortal-source/import_excel.py` reads the source Excel file -(`Mobilne telefony 20260901.xlsx`) and loads it into `TelcoPortal-data/telco.db`. This has +(`Mobilne telefony 20260901.xlsx`) and loads it into the legacy SQLite database. This has **already been run once** to populate the database - the Excel file is not needed by the running app and can be archived/deleted. @@ -77,4 +76,5 @@ the Edit button in the portal itself. ## Backups -Copy or zip `TelcoPortal-data/telco.db` any time - that single file is the entire database. +The production database is PostgreSQL. Use the `pg_dump` command documented in +`README-Docker.md` to create a portable backup. diff --git a/TelcoPortal-source/Dockerfile b/TelcoPortal-source/Dockerfile index c2fe9af..5f9a603 100755 --- a/TelcoPortal-source/Dockerfile +++ b/TelcoPortal-source/Dockerfile @@ -10,7 +10,7 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -# The SQLite database lives outside the container - see docker-compose.yml volumes. +# Persistent application state (the generated Flask session key) lives in this volume. VOLUME ["/app/data"] ENV TELCO_DATA_DIR=/app/data diff --git a/TelcoPortal-source/app.py b/TelcoPortal-source/app.py index efc4e96..4821e55 100755 --- a/TelcoPortal-source/app.py +++ b/TelcoPortal-source/app.py @@ -4,7 +4,7 @@ from datetime import date, datetime, timedelta from functools import wraps from flask import Flask, jsonify, redirect, render_template, request, session, url_for -from sqlalchemy import text +from sqlalchemy import URL, inspect, text from werkzeug.security import check_password_hash, generate_password_hash from models import InternetConnection, Phone, User, db @@ -59,7 +59,7 @@ def get_or_create_secret_key(): def create_app(): app = Flask(__name__) - app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{DB_PATH}" + app.config["SQLALCHEMY_DATABASE_URI"] = get_database_uri() app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False app.config["JSON_SORT_KEYS"] = False app.config["PERMANENT_SESSION_LIFETIME"] = timedelta(days=30) @@ -79,7 +79,7 @@ def create_app(): def run_migrations(): """Lightweight in-place migrations for columns added after the DB was first created. db.create_all() only creates missing tables, not missing columns on existing tables.""" - existing_columns = {row[1] for row in db.session.execute(text("PRAGMA table_info(phones)")).fetchall()} + existing_columns = {column["name"] for column in inspect(db.engine).get_columns("phones")} if "comment" not in existing_columns: db.session.execute(text("ALTER TABLE phones ADD COLUMN comment TEXT")) db.session.commit() @@ -88,6 +88,26 @@ def run_migrations(): db.session.commit() +def get_database_uri(): + """Use the Docker database service when configured, with SQLite as a local-dev fallback.""" + database_url = os.environ.get("DATABASE_URL") + if database_url: + return database_url + + database_host = os.environ.get("TELCO_DB_HOST") + if database_host: + return URL.create( + drivername="postgresql+psycopg2", + username=os.environ.get("TELCO_DB_USER", "telco"), + password=os.environ.get("TELCO_DB_PASSWORD", "telco"), + host=database_host, + port=int(os.environ.get("TELCO_DB_PORT", "5432")), + database=os.environ.get("TELCO_DB_NAME", "telco"), + ) + + return f"sqlite:///{DB_PATH}" + + def seed_users(): """First-run only: create the initial login accounts if no users exist yet at all.""" if User.query.count() > 0: diff --git a/TelcoPortal-source/migrate_sqlite.py b/TelcoPortal-source/migrate_sqlite.py new file mode 100644 index 0000000..f4cdc5e --- /dev/null +++ b/TelcoPortal-source/migrate_sqlite.py @@ -0,0 +1,95 @@ +"""Import the legacy SQLite database into an empty PostgreSQL database. + +The import is intentionally idempotent: if any application rows already exist in the +destination, nothing is changed. This makes it safe to run before every container start. +""" + +import os +import sys +from pathlib import Path + +from sqlalchemy import URL, MetaData, Table, create_engine, func, inspect, select, text + +from models import InternetConnection, Phone, User, db + + +TABLES = (User.__table__, Phone.__table__, InternetConnection.__table__) + + +def destination_url(): + database_url = os.environ.get("DATABASE_URL") + if database_url: + return database_url + return URL.create( + drivername="postgresql+psycopg2", + username=os.environ.get("TELCO_DB_USER", "telco"), + password=os.environ.get("TELCO_DB_PASSWORD", "telco"), + host=os.environ.get("TELCO_DB_HOST", "db"), + port=int(os.environ.get("TELCO_DB_PORT", "5432")), + database=os.environ.get("TELCO_DB_NAME", "telco"), + ) + + +def main(): + sqlite_path = Path(os.environ.get("TELCO_LEGACY_SQLITE_PATH", "/app/legacy-data/telco.db")) + target_engine = create_engine(destination_url()) + + # Create the current schema first. The data copy below then runs as one transaction. + db.metadata.create_all(target_engine) + + try: + with target_engine.connect() as connection: + existing_rows = sum( + connection.execute(select(func.count()).select_from(table)).scalar_one() + for table in TABLES + ) + if existing_rows: + print(f"PostgreSQL already contains {existing_rows} application rows; SQLite import skipped.") + return + + if not sqlite_path.is_file(): + print( + f"Legacy SQLite database not found at {sqlite_path}. " + "Place it there before the first start.", + file=sys.stderr, + ) + raise SystemExit(1) + + source_engine = create_engine(f"sqlite:///{sqlite_path.as_posix()}") + source_metadata = MetaData() + imported = {} + try: + source_table_names = set(inspect(source_engine).get_table_names()) + with target_engine.begin() as destination, source_engine.connect() as source: + for target_table in TABLES: + if target_table.name not in source_table_names: + imported[target_table.name] = 0 + continue + source_table = Table(target_table.name, source_metadata, autoload_with=source_engine) + target_columns = {column.name for column in target_table.columns} + rows = [ + {key: value for key, value in row.items() if key in target_columns} + for row in source.execute(select(source_table)).mappings() + ] + if rows: + destination.execute(target_table.insert(), rows) + imported[target_table.name] = len(rows) + + # Explicit IDs were retained, so move PostgreSQL's sequences past them. + if destination.dialect.name == "postgresql": + for table in TABLES: + destination.execute(text( + "SELECT setval(pg_get_serial_sequence(:table_name, 'id'), " + f"COALESCE(MAX(id), 1), MAX(id) IS NOT NULL) FROM {table.name}" + ), {"table_name": table.name}) + finally: + source_engine.dispose() + + summary = ", ".join(f"{name}={count}" for name, count in imported.items()) + print(f"SQLite import complete: {summary}") + finally: + target_engine.dispose() + + +if __name__ == "__main__": + main() diff --git a/TelcoPortal-source/requirements.txt b/TelcoPortal-source/requirements.txt index 907ff99..e40e6b0 100755 --- a/TelcoPortal-source/requirements.txt +++ b/TelcoPortal-source/requirements.txt @@ -2,3 +2,4 @@ Flask==3.0.3 Flask-SQLAlchemy==3.1.1 openpyxl==3.1.5 gunicorn==22.0.0 +psycopg2-binary==2.9.10 diff --git a/docker-compose.yml b/docker-compose.yml index 911fb92..4ea366e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,45 @@ services: + db: + image: postgres:16-alpine + container_name: telco-db + restart: unless-stopped + environment: + POSTGRES_DB: telco + POSTGRES_USER: telco + POSTGRES_PASSWORD: ${TELCO_DB_PASSWORD:-telco} + volumes: + - telco-db-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U telco -d telco"] + interval: 5s + timeout: 5s + retries: 10 + telcoportal: build: context: ./TelcoPortal-source dockerfile: Dockerfile container_name: telcoportal restart: unless-stopped + depends_on: + db: + condition: service_healthy ports: - "8092:8000" volumes: - # Host-side folder - keeps the SQLite database across rebuilds/updates, and makes it - # easy to back up (just copy/zip this folder). - - ./TelcoPortal-data:/app/data + - telco-app-data:/app/data + # Read-only source for the automatic first-run SQLite import. + - ./data:/app/legacy-data:ro environment: - - TELCO_DATA_DIR=/app/data + TELCO_DATA_DIR: /app/data + TELCO_DB_HOST: db + TELCO_DB_PORT: 5432 + TELCO_DB_NAME: telco + TELCO_DB_USER: telco + TELCO_DB_PASSWORD: ${TELCO_DB_PASSWORD:-telco} + TELCO_LEGACY_SQLITE_PATH: /app/legacy-data/telco.db + command: ["sh", "-c", "python migrate_sqlite.py && exec gunicorn --bind 0.0.0.0:8000 --workers 2 app:app"] + +volumes: + telco-db-data: + telco-app-data: