Initial commit

.gitignore

@@ -1,3 +1,4 @@
+.idea
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,55 @@
 # genieutils-py
-Python implementation of genieutils
+
+Python implementation of [genieutils](https://github.com/Tapsa/genieutils).
+
+This library can be used to read and write `empires2_x2_p1.dat` files for Age of Empires II Definitive Edition.
+
+
+## Supported dat versions
+
+Currently, only the latest version used in Age of Empires II Definitive Edition is supported (`GV_LatestDE2`/`GV_C20`).
+
+
+## Installation
+
+```shell
+pip install genieutils-py
+```
+
+## Usage examples
+
+### Dump the whole dat file as json
+
+The package comes with a handy command line tool that does that for you.
+
+```shell
+dat-to-json path/to/empires2_x2_p1.dat
+```
+
+
+### Change cost of Loom to 69 Gold
+
+```python
+from genieutils.datfile import DatFile
+
+data = DatFile.parse('path/to/empires2_x2_p1.dat')
+data.techs[22].resource_costs[0].amount = 69
+data.save('path/to/modded/empires2_x2_p1.dat')
+```
+
+### Prevent Kings from garrisoning
+
+```python
+from genieutils.datfile import DatFile
+
+data = DatFile.parse('path/to/empires2_x2_p1.dat')
+for civ in data.civs:
+    civ.units[434].bird.task_size -= 1
+    civ.units[434].bird.tasks.pop()
+data.save('path/to/modded/empires2_x2_p1.dat')
+```
+
+
+## Authors
+
+[HSZemi](https://github.com/hszemi) - Original Author

pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/genieutils"]
+
+[project]
+name = "genieutils-py"
+version = "0.0.1"
+authors = [
+    { name = "SiegeEngineers", email = "[email protected]" },
+]
+description = "Re-implementation of genieutils in Python"
+readme = "README.md"
+requires-python = ">=3.11"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/SiegeEngineers/genieutils-py"
+Issues = "https://github.com/SiegeEngineers/genieutils-py/issues"
+
+[project.scripts]
+dat-to-json = "genieutils.scripts:dat_to_json"

src/genieutils/civ.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+from genieutils.common import ByteHandler, GenieClass
+from genieutils.unit import Unit
+
+
+@dataclass
+class Civ(GenieClass):
+    player_type: int
+    name: str
+    resources_size: int
+    tech_tree_id: int
+    team_bonus_id: int
+    resources: list[float]
+    icon_set: int
+    units_size: int
+    unit_pointers: list[int]
+    units: list[Unit | None]
+
+    @classmethod
+    def from_bytes(cls, content: ByteHandler) -> 'Civ':
+        player_type = content.read_int_8()
+        name = content.read_debug_string()
+        resources_size = content.read_int_16()
+        tech_tree_id = content.read_int_16()
+        team_bonus_id = content.read_int_16()
+        resources = content.read_float_array(resources_size)
+        icon_set = content.read_int_8()
+        units_size = content.read_int_16()
+        unit_pointers = content.read_int_32_array(units_size)
+        units = content.read_class_array_with_pointers(Unit, units_size, unit_pointers)
+        return cls(
+            player_type=player_type,
+            name=name,
+            resources_size=resources_size,
+            tech_tree_id=tech_tree_id,
+            team_bonus_id=team_bonus_id,
+            resources=resources,
+            icon_set=icon_set,
+            units_size=units_size,
+            unit_pointers=unit_pointers,
+            units=units,
+        )
+
+    def to_bytes(self) -> bytes:
+        return b''.join([
+            self.write_int_8(self.player_type),
+            self.write_debug_string(self.name),
+            self.write_int_16(self.resources_size),
+            self.write_int_16(self.tech_tree_id),
+            self.write_int_16(self.team_bonus_id),
+            self.write_float_array(self.resources),
+            self.write_int_8(self.icon_set),
+            self.write_int_16(self.units_size),
+            self.write_int_32_array(self.unit_pointers),
+            self.write_class_array_with_pointers(self.unit_pointers, self.units),
+        ])

src/genieutils/common.py

@@ -0,0 +1,172 @@
+from abc import ABC
+from enum import IntEnum
+from typing import TypeVar
+
+from genieutils.datatypes import Int, Float, String
+
+TILE_TYPE_COUNT = 19
+TERRAIN_COUNT = 200
+TERRAIN_UNITS_SIZE = 30
+
+
+class UnitType(IntEnum):
+    EyeCandy = 10
+    Trees = 15
+    Flag = 20
+    DeadFish = 30
+    Bird = 40
+    Combatant = 50
+    Projectile = 60
+    Creatable = 70
+    Building = 80
+    AoeTrees = 90
+
+
+class GenieClass(ABC):
+    @classmethod
+    def from_bytes(cls, data: 'ByteHandler'):
+        raise NotImplementedError
+
+    @classmethod
+    def from_bytes_with_count(cls, data: 'ByteHandler', terrains_used_1: int):
+        raise NotImplementedError
+
+    def to_bytes(self) -> bytes:
+        raise NotImplementedError
+
+    def write_debug_string(self, value: str) -> bytes:
+        return (self.write_int_16(0x0A60, signed=False)
+                + self.write_int_16(len(value), signed=False)
+                + value.encode('utf-8'))
+
+    def write_string(self, length: int, value: str) -> bytes:
+        return String.to_bytes(value, length)
+
+    def write_int_8(self, value: int) -> bytes:
+        return Int.to_bytes(value, length=1, signed=False)
+
+    def write_int_8_array(self, value: list[int]) -> bytes:
+        return b''.join(self.write_int_8(v) for v in value)
+
+    def write_int_16(self, value: int, signed=True) -> bytes:
+        return Int.to_bytes(value, length=2, signed=signed)
+
+    def write_int_16_array(self, value: list[int]) -> bytes:
+        return b''.join(self.write_int_16(v) for v in value)
+
+    def write_int_32(self, value: int, signed=True) -> bytes:
+        return Int.to_bytes(value, length=4, signed=signed)
+
+    def write_int_32_array(self, value: list[int]) -> bytes:
+        return b''.join(self.write_int_32(v) for v in value)
+
+    def write_float(self, value: float) -> bytes:
+        return Float.to_bytes(value)
+
+    def write_float_array(self, value: list[float]) -> bytes:
+        return b''.join(self.write_float(v) for v in value)
+
+    def write_class(self, value: 'GenieClass') -> bytes:
+        retval = value.to_bytes()
+        if retval:
+            return retval
+        return b''
+
+    def write_class_array(self, value: list['GenieClass']) -> bytes:
+        retval = b''.join(self.write_class(v) for v in value)
+        if retval:
+            return retval
+        return b''
+
+    def write_class_array_with_pointers(self, pointers: list[int], value: list['GenieClass']) -> bytes:
+        retval = b''.join(self.write_class(v) for i, v in enumerate(value) if pointers[i])
+        if retval:
+            return retval
+        return b''
+
+
+C = TypeVar('C', bound=GenieClass)
+
+
+class ByteHandler:
+    def __init__(self, content: memoryview):
+        self.content = content
+        self.offset = 0
+
+    def consume_range(self, length: int) -> memoryview:
+        start = self.offset
+        end = start + length
+        self.offset = end
+        return self.content[start:end]
+
+    def read_debug_string(self) -> str:
+        tmp_size = self.read_int_16(signed=False)
+        assert tmp_size == 0x0A60
+        size = self.read_int_16(signed=False)
+        return String.from_bytes(self.consume_range(size))
+
+    def read_string(self, length: int) -> str:
+        return String.from_bytes(self.consume_range(length))
+
+    def read_int_8(self) -> int:
+        return Int.from_bytes(self.consume_range(1), signed=False)
+
+    def read_int_8_array(self, size: int) -> list[int]:
+        elements = []
+        for i in range(size):
+            elements.append(self.read_int_8())
+        return elements
+
+    def read_int_16(self, signed=True) -> int:
+        return Int.from_bytes(self.consume_range(2), signed=signed)
+
+    def read_int_16_array(self, size: int) -> list[int]:
+        elements = []
+        for i in range(size):
+            elements.append(self.read_int_16())
+        return elements
+
+    def read_int_32(self, signed=True) -> int:
+        return Int.from_bytes(self.consume_range(4), signed=signed)
+
+    def read_int_32_array(self, size: int) -> list[int]:
+        elements = []
+        for i in range(size):
+            elements.append(self.read_int_32())
+        return elements
+
+    def read_float(self) -> float:
+        return Float.from_bytes(self.consume_range(4))
+
+    def read_float_array(self, size: int) -> list[float]:
+        elements = []
+        for i in range(size):
+            elements.append(self.read_float())
+        return elements
+
+    def read_class(self, class_: type[C]) -> C:
+        element = class_.from_bytes(self)
+        return element
+
+    def read_class_array(self, class_: type[C], size: int) -> list[C]:
+        elements = []
+        for i in range(size):
+            element = class_.from_bytes(self)
+            elements.append(element)
+        return elements
+
+    def read_class_array_with_pointers(self, class_: type[C], size: int, pointers: list[int]) -> list[C | None]:
+        elements = []
+        for i in range(size):
+            element = None
+            if pointers[i]:
+                element = class_.from_bytes(self)
+            elements.append(element)
+        return elements
+
+    def read_class_array_with_param(self, class_: type[C], size: int, terrains_used_1: int) -> list[C]:
+        elements = []
+        for i in range(size):
+            terrain_restriction = class_.from_bytes_with_count(self, terrains_used_1)
+            elements.append(terrain_restriction)
+        return elements

src/genieutils/datatypes.py

@@ -0,0 +1,35 @@
+import struct
+
+
+class String:
+    @staticmethod
+    def from_bytes(content: memoryview) -> str:
+        return bytes(content).rstrip(b'\0').decode()
+
+    @staticmethod
+    def to_bytes(content: str, length=None) -> bytes:
+        encoded = content.encode()
+        if not length:
+            length = len(encoded) + 1
+        zfill = length - len(encoded)
+        return encoded + (b'\0' * zfill)
+
+
+class Int:
+    @staticmethod
+    def from_bytes(content: memoryview, signed=True) -> int:
+        return int.from_bytes(content, byteorder='little', signed=signed)
+
+    @staticmethod
+    def to_bytes(content: int, length=2, signed=True) -> bytes:
+        return content.to_bytes(length, 'little', signed=signed)
+
+
+class Float:
+    @staticmethod
+    def from_bytes(content: memoryview) -> float:
+        return struct.unpack('f', content)[0]
+
+    @staticmethod
+    def to_bytes(content: float) -> bytes:
+        return struct.pack('f', content)