From 19edc9df144411c6f573a188fdd6d59d5f9ede3d Mon Sep 17 00:00:00 2001 From: Alexey Olendor <137941487+Lolendor@users.noreply.github.com> Date: Sat, 27 Dec 2025 10:41:45 +0300 Subject: [PATCH] feat: add packed archive support with brotli compression, add configuration and more Add support for serving assets from packed binary archives with brotli compression. New features: - `packer_brotli.py`: Pack vcsky/vcbr folders into single .bin archive with brotli compression (quality 11) - `downloader_brotli.py`: Stream unpack archives during download - `--packed` flag: Serve files directly from archive (supports local files and URLs) - `--unpacked` flag: Extract archive to `unpacked/{hash}/` and serve from there - `--vcsky_local` / `--vcbr_local`: Now accept optional custom paths - MD5 hash detection: Pass existing folder hash directly to `--unpacked` - `--pack` for packing Archive format: - ULEB128 length encoding for compact size - Folder/file deduplication (identical content stored once) - Parallel compression with ProcessPoolExecutor and much more, like ?configurable=1 and auto language detection fix, ... --- .gitignore | 2 + README.md | 84 +- additions/packed.py | 411 +++++++++ colab_reVCDOS.ipynb | 56 +- dist/game.js | 275 ++++-- dist/index.html | 63 ++ dist/index.js | 4 +- docker-compose.yml | 14 +- requirements.txt | 3 +- server.py | 363 +++++++- utils/downloader_brotli.py | 457 ++++++++++ utils/packer_brotli.py | 1757 ++++++++++++++++++++++++++++++++++++ 12 files changed, 3360 insertions(+), 129 deletions(-) create mode 100644 additions/packed.py create mode 100644 utils/downloader_brotli.py create mode 100644 utils/packer_brotli.py diff --git a/.gitignore b/.gitignore index 924d785..6c52516 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ __pycache__/ vcbr/ vcsky/ saves/ +unpacked/ +*.bin \ No newline at end of file diff --git a/README.md b/README.md index ef48867..72a34c7 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,12 @@ Web-based port of GTA: Vice City running in browser via WebAssembly. 2. **Configure Assets** (Optional): - By default, the project uses the **DOS Zone CDN**. For local hosting, download and place assets in [(see structure)](#project-structure): - * **Resources:** `vcsky/fetched/` (or `fetched-ru/`) — `data`, `audio`, `anim`, `models` folders. - * **Binaries:** `vcbr/` — `.wasm.br` and `.data.br` files for your chosen language. -4. **Launch the Application**: + By default, the project uses **DOS Zone CDN** — no local assets needed. For offline hosting you can use: + * **Packed archive** (`--packed` or `--unpacked`) — single `.bin` file with all assets + * **Local folders** (`--vcsky_local`, `--vcbr_local`) — unpacked asset directories + * **Cache mode** (`--vcsky_cache`, `--vcbr_cache`) — download from CDN once, serve locally after + +3. **Launch the Application**: Choose one of the setup methods below: * **Docker** (Recommended for most users) — fast and isolated. * **PHP** — Simply upload the folder to your web server (FTP/Hosting). @@ -36,7 +38,7 @@ Web-based port of GTA: Vice City running in browser via WebAssembly. The easiest way to get started is using Docker Compose: ```bash -VCSKY_CACHE=1 VCBR_CACHE=1 docker compose up -d --build +PACKED=https://folder.morgen.monster/revcdos.bin docker compose up -d --build ``` To configure server options via environment variables: @@ -54,12 +56,15 @@ IN_PORT=3000 AUTH_LOGIN=admin AUTH_PASSWORD=secret CUSTOM_SAVES=1 docker compose | `AUTH_LOGIN` | HTTP Basic Auth username | | `AUTH_PASSWORD` | HTTP Basic Auth password | | `CUSTOM_SAVES` | Enable local saves (set to `1`) | -| `VCSKY_LOCAL` | Serve vcsky from local directory (set to `1`) | -| `VCBR_LOCAL` | Serve vcbr from local directory (set to `1`) | +| `VCSKY_LOCAL` | Serve vcsky from local directory (set to `1`, or path like `/data/vcsky`) | +| `VCBR_LOCAL` | Serve vcbr from local directory (set to `1`, or path like `/data/vcbr`) | | `VCSKY_URL` | Custom vcsky proxy URL | | `VCBR_URL` | Custom vcbr proxy URL | | `VCSKY_CACHE` | Cache vcsky files locally while proxying (set to `1`) | | `VCBR_CACHE` | Cache vcbr files locally while proxying (set to `1`) | +| `PACKED` | Serve from packed archive (filename or URL, e.g., `revcdos.bin`) | +| `UNPACKED` | Unpack archive to local folders (filename or URL, auto-sets vcsky/vcbr paths) | +| `PACK` | Pack a folder and serve from resulting archive (folder path or MD5 hash) | ### Option 2: Local Installation @@ -70,7 +75,7 @@ pip install -r requirements.txt 2. Start the server: ```bash -python server.py --vcsky_cache --vcbr_cache +python server.py --packed https://folder.morgen.monster/revcdos.bin ``` Server starts at `http://localhost:8000` @@ -87,12 +92,15 @@ By default the `index.php` and `.htaccess` will get the job done. | `--custom_saves` | flag | disabled | Enable local save files (saves router) | | `--login` | string | none | HTTP Basic Auth username | | `--password` | string | none | HTTP Basic Auth password | -| `--vcsky_local` | flag | disabled | Serve vcsky from local `vcsky/` directory | -| `--vcbr_local` | flag | disabled | Serve vcbr from local `vcbr/` directory | +| `--vcsky_local` | string/flag | disabled | Serve vcsky from local directory. Use flag for `vcsky/` or specify path | +| `--vcbr_local` | string/flag | disabled | Serve vcbr from local directory. Use flag for `vcbr/` or specify path | | `--vcsky_url` | string | `https://cdn.dos.zone/vcsky/` | Custom vcsky proxy URL | | `--vcbr_url` | string | `https://br.cdn.dos.zone/vcsky/` | Custom vcbr proxy URL | | `--vcsky_cache` | flag | disabled | Cache vcsky files locally while proxying | | `--vcbr_cache` | flag | disabled | Cache vcbr files locally while proxying | +| `--packed` | string | disabled | Serve from packed archive file. Accepts file path or URL | +| `--unpacked` | string | disabled | Unpack archive to `unpacked/{hash}/` and serve from there. Accepts file path or URL | +| `--pack` | string | disabled | Pack a folder and serve from resulting `{hash}.bin` archive. Accepts folder path or MD5 hash | **Examples:** ```bash @@ -105,25 +113,45 @@ python server.py --custom_saves # Enable HTTP Basic Authentication python server.py --login admin --password secret123 -# Use local vcsky and vcbr files (fully offline mode) +# Use local vcsky and vcbr files python server.py --vcsky_local --vcbr_local -# Use custom proxy URLs -python server.py --vcsky_url https://my-cdn.example.com/vcsky/ --vcbr_url https://my-cdn.example.com/vcbr/ - # Cache files locally while proxying (hybrid mode) (recommended) python server.py --vcsky_cache --vcbr_cache -# All options combined -python server.py --port 3000 --custom_saves --login admin --password secret123 --vcsky_local --vcbr_local +# Serve from packed archive (local file) +python server.py --packed revcdos.bin + +# Serve from packed archive (download from URL if not present) +python server.py --packed https://example.com/revcdos.bin + +# Unpack archive and serve from unpacked files +python server.py --unpacked revcdos.bin + +# Stream-unpack from URL (downloads and unpacks simultaneously) +python server.py --unpacked https://example.com/revcdos.bin + +# Pack a folder and serve from the resulting archive +python server.py --pack /path/to/assets # Creates {folder_hash}.bin + +# Pack from existing unpacked folder by MD5 hash +python server.py --pack a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6 # Uses unpacked/{hash}/ ``` > **Note:** HTTP Basic Auth is only enabled when both `--login` and `--password` are provided. -> **Note:** By default, vcsky and vcbr are proxied from DOS Zone CDN. Use `--vcsky_local` and `--vcbr_local` flags to serve files from local directories instead. +> **Note:** By default, vcsky and vcbr are proxied from DOS Zone CDN. Use `--vcsky_local` and `--vcbr_local` flags to serve files from local directories instead. You can optionally specify a custom path. > **Note:** Use `--vcsky_cache` and `--vcbr_cache` to cache proxied files locally. Files are downloaded once and served from local storage on subsequent requests. +> **Note:** `--packed` serves files directly from a compressed archive without unpacking (faster and more compressed). `--unpacked` extracts the archive once and serves from local files (if you want edit assets). + +> **Note:** When using URL with `--unpacked`, the archive is streamed and unpacked simultaneously during download using `downloader_brotli.py`. + +> **Note:** You can pass a raw MD5 hash (32 hex characters) to `--unpacked` to use an existing unpacked folder without needing the original archive. Example: if you have `unpacked/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6/`, you can start the server with `--unpacked a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6`. + +> **Note:** `--pack` creates a packed archive from a folder containing subfolders (like `vcsky/` and `vcbr/`). Each subfolder is packed sequentially: the first creates the archive, subsequent ones are appended. After packing, the server automatically uses the created archive via `--packed` mode. + ## URL Parameters | Parameter | Values | Description | @@ -133,23 +161,35 @@ python server.py --port 3000 --custom_saves --login admin --password secret123 - | `request_original_game` | `1` | Request original game files before play | | `fullscreen` | `0` | Disable auto-fullscreen | | `max_fps` | `1-240` | Limit frame rate (e.g., `60` for 60 FPS) | +| `configurable` | `1` | Show configuration UI before play button | **Examples:** - `http://localhost:8000/?lang=ru` — Russian version - `http://localhost:8000/?lang=en&cheats=1` — English + cheats +- `http://localhost:8000/?configurable=1` — Show settings UI before play ## Project Structure ``` ├── server.py # FastAPI proxy server -├── index.php # php proxy server -├── .htaccess # apache config for php +├── index.php # PHP proxy server +├── .htaccess # Apache config for PHP ├── requirements.txt # Python dependencies +├── packer_brotli.py # Archive packer with brotli compression +├── downloader_brotli.py # Stream unpacker for packed archives +├── revcdos.bin # Packed archive (optional, created by packer_brotli.py) ├── additions/ # Server extensions │ ├── auth.py # HTTP Basic Auth middleware │ ├── cache.py # Proxy caching and brotli decompression +│ ├── packed.py # Packed archive serving module │ └── saves.py # Local saves router +├── utils/ # Utility modules +│ └── packer_brotli.py # Packer module (imported by server) +├── unpacked/ # Auto-created by --unpacked flag +│ └── {md5_hash}/ # Unpacked files organized by source hash +│ ├── vcsky/ # Decompressed game assets +│ └── vcbr/ # Brotli-compressed binaries ├── dist/ # Game client files │ ├── index.html # Main page │ ├── game.js # Game loader @@ -180,13 +220,13 @@ python server.py --port 3000 --custom_saves --login admin --password secret123 - │ ├── vc-sky-en-v6.wasm.br │ ├── vc-sky-ru-v6.data.br │ └── vc-sky-ru-v6.wasm.br -└── vcsky/ # Decompressed assets (optional) - ├── fetched/ # English version files +└── vcsky/ # Decompressed assets (optional) + ├── fetched/ # English version files │ ├── data/ │ ├── audio/ │ ├── models/ │ └── anim/ - └── fetched-ru/ # Russian version files + └── fetched-ru/ # Russian version files ├── data/ ├── audio/ └── ... diff --git a/additions/packed.py b/additions/packed.py new file mode 100644 index 0000000..bdb250a --- /dev/null +++ b/additions/packed.py @@ -0,0 +1,411 @@ +""" +Module for serving files from a PackedArchive. +Provides similar interface to cache.py but reads from packed .bin archives. + +Supports: +- Serving files from vcsky/ and vcbr/ paths inside the archive +- Brotli compression passthrough when client supports it (Accept-Encoding: br) +- On-the-fly decompression when client doesn't support brotli +- Proper handling of .br files (stored without additional compression) +- Auto-download from URL if archive file is not present locally +""" + +import os +import sys +from typing import Optional +from urllib.parse import urlparse + +import httpx +import brotli +from fastapi import Request +from fastapi.responses import Response, StreamingResponse + +# Import PackedArchive from utils +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'utils')) +from utils.packer_brotli import PackedArchive + +# Global archive instance (initialized by init_packed_archive) +_archive: Optional[PackedArchive] = None + + +def _is_url(path: str) -> bool: + """Check if the path is a URL.""" + return path.startswith("http://") or path.startswith("https://") + + +def _get_filename_from_url(url: str) -> str: + """Extract filename from URL.""" + parsed = urlparse(url) + path = parsed.path + filename = os.path.basename(path) + if not filename: + filename = "packed.bin" + return filename + + +async def _download_file(url: str, dest_path: str) -> bool: + """ + Download a file from URL to destination path. + + Args: + url: URL to download from + dest_path: Local path to save the file + + Returns: + True if download succeeded, False otherwise + """ + print(f"Downloading archive from {url}...") + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(300.0), follow_redirects=True) as client: + async with client.stream('GET', url) as response: + response.raise_for_status() + + content_length = response.headers.get('content-length') + total_size = int(content_length) if content_length else 0 + downloaded = 0 + + with open(dest_path, 'wb') as f: + async for chunk in response.aiter_bytes(65536): + f.write(chunk) + downloaded += len(chunk) + if total_size > 0: + percent = (downloaded / total_size) * 100 + print(f"\r Downloaded: {downloaded / 1024 / 1024:.1f} MB ({percent:.1f}%)", end="", flush=True) + else: + print(f"\r Downloaded: {downloaded / 1024 / 1024:.1f} MB", end="", flush=True) + + print() # New line after download complete + print(f" Saved to: {dest_path}") + return True + except httpx.HTTPStatusError as e: + print(f"Failed to download: HTTP {e.response.status_code}") + return False + except Exception as e: + print(f"Error downloading file: {e}") + return False + + +async def resolve_packed_source(source: str) -> Optional[str]: + """ + Resolve packed archive source to local file path. + + If source is a URL: + - Extract filename from URL + - Check if file exists locally and has size > 0 + - If not, download it from URL + - Return local file path + + If source is a local path: + - Return it as-is + + Args: + source: URL or local file path + + Returns: + Local file path, or None if download failed + """ + if not _is_url(source): + # Local file path + return source + + # It's a URL - extract filename and check if we have it locally + filename = _get_filename_from_url(source) + local_path = filename # Save in current directory + + # Check if file exists and has size > 0 + if os.path.isfile(local_path) and os.path.getsize(local_path) > 0: + print(f"Using existing archive: {local_path} ({os.path.getsize(local_path)} bytes)") + return local_path + + # File doesn't exist or is empty - download it + if await _download_file(source, local_path): + return local_path + + return None + + +async def init_packed_archive(source: str) -> Optional[PackedArchive]: + """ + Initialize the packed archive. + Must be called before using get_packed_file(). + + Supports both local file paths and URLs. + If a URL is provided, the file will be downloaded if not present locally. + + Args: + source: Path to the .bin archive file or URL to download from + + Returns: + Initialized PackedArchive instance, or None if failed + """ + global _archive + + # Resolve source to local path (download if needed) + archive_path = await resolve_packed_source(source) + if archive_path is None: + print(f"Failed to resolve packed archive source: {source}") + return None + + if not os.path.isfile(archive_path): + print(f"Archive file not found: {archive_path}") + return None + + _archive = PackedArchive(archive_path) + await _archive.init() + print(f"Loaded packed archive: {archive_path}") + print(f" Folders: {len(_archive.list_folders())}") + print(f" Files: {len(_archive.list_files())}") + return _archive + + +def get_archive() -> Optional[PackedArchive]: + """Get the global archive instance.""" + return _archive + + +def is_initialized() -> bool: + """Check if the archive is initialized.""" + return _archive is not None and _archive._initialized + + +def _client_accepts_brotli(request: Request) -> bool: + """Check if client accepts brotli encoding.""" + accept_encoding = request.headers.get("accept-encoding", "") + return "br" in accept_encoding.lower() + + +def _get_response_headers(use_brotli: bool, media_type: str) -> dict: + """Get response headers, optionally with brotli encoding.""" + headers = { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", + "Content-Type": media_type + } + + if use_brotli: + headers["Content-Encoding"] = "br" + + return headers + + +def _is_br_file(path: str) -> bool: + """Check if the file is a .br (pre-compressed brotli) file.""" + return path.lower().endswith(".br") + + +def _get_media_type(path: str) -> str: + """ + Get appropriate media type based on file extension. + For .br files, returns the media type of the underlying content. + """ + lower_path = path.lower() + + # Handle .br files - get media type of what's inside + if lower_path.endswith(".wasm.br"): + return "application/wasm" + if lower_path.endswith(".js.br"): + return "application/javascript" + if lower_path.endswith(".json.br"): + return "application/json" + if lower_path.endswith(".html.br"): + return "text/html" + if lower_path.endswith(".css.br"): + return "text/css" + if lower_path.endswith(".br"): + # Generic .br file - use octet-stream + return "application/octet-stream" + + # Non-.br files + if lower_path.endswith(".wasm"): + return "application/wasm" + if lower_path.endswith(".js"): + return "application/javascript" + if lower_path.endswith(".json"): + return "application/json" + if lower_path.endswith(".html"): + return "text/html" + if lower_path.endswith(".css"): + return "text/css" + if lower_path.endswith(".png"): + return "image/png" + if lower_path.endswith(".jpg") or lower_path.endswith(".jpeg"): + return "image/jpeg" + if lower_path.endswith(".gif"): + return "image/gif" + if lower_path.endswith(".svg"): + return "image/svg+xml" + if lower_path.endswith(".mp3"): + return "audio/mpeg" + if lower_path.endswith(".wav"): + return "audio/wav" + if lower_path.endswith(".ogg"): + return "audio/ogg" + + return "application/octet-stream" + + +async def get_packed_file(path: str, request: Request) -> Optional[Response]: + """ + Get a file from the packed archive. + + How .br files work: + - .br files are stored in the archive WITHOUT additional brotli compression + - archive.open(path) returns the raw .br file content (already brotli-compressed) + - If client accepts br: send .br data with Content-Encoding: br + - If client doesn't accept br: decompress .br data and send plain + + How regular files work: + - Regular files are stored with brotli compression in the archive + - If client accepts br: keep_brotli=True returns compressed data, send with Content-Encoding: br + - If client doesn't accept br: keep_brotli=False decompresses, send plain + + Args: + path: Path to the file inside the archive (e.g., "vcsky/fetched/model.txd") + request: FastAPI request object to check Accept-Encoding header + + Returns: + Response with file data, or None if file not found or archive not initialized + """ + if not is_initialized(): + return None + + # Check if file exists in archive + if not _archive.exists(path): + return None + + # Check if client accepts brotli + client_accepts_br = _client_accepts_brotli(request) + + # Check if file is a .br file + is_br_file = _is_br_file(path) + + # Get media type based on file extension + media_type = _get_media_type(path) + + try: + if is_br_file: + # .br files are stored as-is (no archive compression) + # archive.open() returns the raw .br content + async with _archive.open(path, keep_brotli=False) as f: + br_data = f.read() + + if client_accepts_br: + # Send the .br data with Content-Encoding: br + headers = _get_response_headers(use_brotli=True, media_type=media_type) + return Response(content=br_data, headers=headers) + else: + # Decompress .br for client + decompressed_data = brotli.decompress(br_data) + headers = _get_response_headers(use_brotli=False, media_type=media_type) + return Response(content=decompressed_data, headers=headers) + else: + # Regular files: use archive's brotli compression + if client_accepts_br: + async with _archive.open(path, keep_brotli=True) as f: + data = f.read() + headers = _get_response_headers(use_brotli=True, media_type=media_type) + else: + async with _archive.open(path, keep_brotli=False) as f: + data = f.read() + headers = _get_response_headers(use_brotli=False, media_type=media_type) + + return Response(content=data, headers=headers) + except FileNotFoundError: + return None + except Exception as e: + print(f"Error reading file from archive: {path} - {e}") + return None + + +async def get_packed_file_streaming(path: str, request: Request, chunk_size: int = 65536) -> Optional[StreamingResponse]: + """ + Get a file from the packed archive as a streaming response. + + Args: + path: Path to the file inside the archive + request: FastAPI request object to check Accept-Encoding header + chunk_size: Size of chunks for streaming (default: 64KB) + + Returns: + StreamingResponse with file data, or None if file not found + """ + if not is_initialized(): + return None + + if not _archive.exists(path): + return None + + client_accepts_br = _client_accepts_brotli(request) + is_br = _is_br_file(path) + media_type = _get_media_type(path) + + async def generate(): + try: + if is_br: + # .br file: stored as-is, open returns raw .br data + async with _archive.open(path, keep_brotli=False) as f: + br_data = f.data + + if client_accepts_br: + # Send .br data directly + for i in range(0, len(br_data), chunk_size): + yield br_data[i:i + chunk_size] + else: + # Decompress for client + decompressed_data = brotli.decompress(br_data) + for i in range(0, len(decompressed_data), chunk_size): + yield decompressed_data[i:i + chunk_size] + else: + # Regular file + async with _archive.open(path, keep_brotli=client_accepts_br) as f: + data = f.data + for i in range(0, len(data), chunk_size): + yield data[i:i + chunk_size] + except Exception as e: + print(f"Error streaming file from archive: {path} - {e}") + + headers = _get_response_headers(use_brotli=client_accepts_br, media_type=media_type) + + return StreamingResponse(generate(), headers=headers) + + +def file_exists(path: str) -> bool: + """ + Check if a file exists in the packed archive. + + Args: + path: Path to the file inside the archive + + Returns: + True if file exists, False otherwise + """ + if not is_initialized(): + return False + return _archive.exists(path) + + +def list_files(folder: Optional[str] = None) -> list: + """ + List files in the archive. + + Args: + folder: Optional folder path to filter by + + Returns: + List of file paths + """ + if not is_initialized(): + return [] + return _archive.list_files(folder) + + +def list_folders() -> list: + """ + List all folders in the archive. + + Returns: + List of folder paths + """ + if not is_initialized(): + return [] + return _archive.list_folders() diff --git a/colab_reVCDOS.ipynb b/colab_reVCDOS.ipynb index 47c7241..2b7a832 100644 --- a/colab_reVCDOS.ipynb +++ b/colab_reVCDOS.ipynb @@ -50,7 +50,7 @@ "# Start server\n", "print(\"🚀 Starting server...\")\n", "server = subprocess.Popen(\n", - " [sys.executable, \"server.py\", \"--vcsky_cache\", \"--vcbr_cache\"],\n", + " [sys.executable, \"server.py\", \"--packed\", \"https://folder.morgen.monster/revcdos.bin\"],\n", " stdout=subprocess.DEVNULL,\n", " stderr=subprocess.DEVNULL\n", ")\n", @@ -82,11 +82,11 @@ "if tunnel_url:\n", " # Display interactive button that copies password and opens URL\n", " html = f'''\n", - "
\n", + "
\n", "
Server is ready
\n", "
The tunnel is active. Configure options and launch the game.
\n", " \n", - "
\n", + "
\n", "
\n", " \n", "
\n", @@ -109,6 +109,44 @@ "
\n", "
\n", "
\n", + " \n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", " \n", "