Add tests

.github/workflows/python-lib.yml

@@ -0,0 +1,38 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on: ['push', 'pull_request']
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Fetch test files
+      uses: actions/checkout@v4
+      with:
+        repository: SiegeEngineers/dat-files
+        ssh-key: ${{ secrets.DEPLOY_KEY }}
+        path: tests/testdata/
+    - name: List test files
+      run: |
+        find tests/testdata/ -iname '*.dat'
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.11"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements-dev.txt
+        pip install -e .
+    - name: Run test script
+      run: |
+        ./test

.gitignore

@@ -1,4 +1,5 @@
 .idea
+tests/testdata/*
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -7,7 +7,8 @@ This library can be used to read and write `empires2_x2_p1.dat` files for Age of
 
 ## Supported dat versions
 
-Currently, only the latest version used in Age of Empires II Definitive Edition is supported (`GV_LatestDE2`/`GV_C20`).
+Currently, only the recent versions used in Age of Empires II Definitive Edition are supported
+(`GV_C20` and above, corresponding to FileVersion 7.7 and above).
 
 
 ## Installation
@@ -48,6 +49,19 @@ for civ in data.civs:
 data.save('path/to/modded/empires2_x2_p1.dat')
 ```
 
+## Running tests
+
+**Before running tests, you need to add sample `empires2_x2_p1.dat` files into the `tests/testdata` subfolder.**
+
+1. Create a virtual environment  
+   `python3 -m venv venv`
+2. Activate the virtual environment  
+   `source venv/bin/activate`
+3. Install the dev dependencies  
+   `pip install -r requirements-dev.txt`
+4. Run the test script  
+   `./test`
+
 
 ## Authors
 

pyproject.toml

@@ -26,3 +26,9 @@ Issues = "https://github.com/SiegeEngineers/genieutils-py/issues"
 
 [project.scripts]
 dat-to-json = "genieutils.scripts:dat_to_json"
+
+[tool.ruff]
+line-length = 120
+
+[tool.coverage.run]
+omit = ['tests/*']

requirements-dev.txt

@@ -0,0 +1,4 @@
+pytest
+pytest-cov
+ruff
+mypy

src/genieutils/unit.py

@@ -829,19 +829,19 @@ def to_bytes(self, version: Version) -> bytes:
         building = b''
         if self.type != UnitType.AoeTrees:
             if self.type >= UnitType.Flag:
-                speed = self.write_float(self.speed)
+                speed = self.write_float(self.speed) if self.speed is not None else b''
                 if self.type >= UnitType.DeadFish:
-                    dead_fish = self.write_class(self.dead_fish, version)
+                    dead_fish = self.write_class(self.dead_fish, version) if self.dead_fish is not None else b''
                 if self.type >= UnitType.Bird:
-                    bird = self.write_class(self.bird, version)
+                    bird = self.write_class(self.bird, version) if self.bird is not None else b''
                 if self.type >= UnitType.Combatant:
-                    type_50 = self.write_class(self.type_50, version)
+                    type_50 = self.write_class(self.type_50, version) if self.type_50 is not None else b''
                 if self.type == UnitType.Projectile:
-                    projectile = self.write_class(self.projectile, version)
+                    projectile = self.write_class(self.projectile, version) if self.projectile is not None else b''
                 if self.type >= UnitType.Creatable:
-                    creatable = self.write_class(self.creatable, version)
+                    creatable = self.write_class(self.creatable, version) if self.creatable is not None else b''
                 if self.type == UnitType.Building:
-                    building = self.write_class(self.building, version)
+                    building = self.write_class(self.building, version) if self.building is not None else b''
         return b''.join([
             self.write_int_8(self.type),
             self.write_int_16(self.id),

src/genieutils/unitheaders.py

@@ -26,6 +26,6 @@ def from_bytes(cls, content: ByteHandler) -> 'UnitHeaders':
     def to_bytes(self, version: Version) -> bytes:
         return b''.join([
             self.write_int_8(self.exists),
-            self.write_int_16(len(self.task_list)) if self.exists else b'',
-            self.write_class_array(self.task_list, version) if self.exists else b'',
+            self.write_int_16(len(self.task_list)) if self.exists and self.task_list is not None else b'',
+            self.write_class_array(self.task_list, version) if self.exists and self.task_list is not None else b'',
         ])

test

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+python -m pytest --cov=src --cov-report=html --cov-config=pyproject.toml ./
+ruff check src
+python -m mypy ./src

tests/test_dat_compatibility.py

@@ -0,0 +1,33 @@
+import zlib
+from pathlib import Path
+
+import pytest
+
+from genieutils.common import ByteHandler
+from genieutils.datfile import DatFile
+
+TESTDATA_DIR = Path(__file__).with_name('testdata')
+
+
+class TestDatCompatibility:
+    def test_there_are_actually_dat_files_for_testing(self):
+        assert len(list(TESTDATA_DIR.rglob('*.dat')))
+
+    @pytest.mark.parametrize('datfile', sorted(TESTDATA_DIR.rglob('*.dat'), reverse=True))
+    def test_compatibility_with_dat_files(self, datfile: Path):
+        version, data = self.get_version(datfile)
+        print(datfile)
+        print(version)
+        if version in ('VER 7.8', 'VER 7.7'):
+            byte_handler = ByteHandler(memoryview(data))
+            content = DatFile.from_bytes(byte_handler)
+            re_encoded = content.to_bytes()
+            assert data == re_encoded
+        else:
+            pytest.skip(f'version {version} is currently not supported')
+
+    def get_version(self, datfile: Path) -> tuple[str, bytes]:
+        content = datfile.read_bytes()
+        data = zlib.decompress(content, wbits=-15)
+        version = data[:8].rstrip(b'\0').decode()
+        return version, data