94 comp_m = re.search(
r"compressed\[\] = \{(.*?)\};", text, re.S)
96 raise SystemExit(f
"no compressed[] array in {BLOB_PATH}")
98 int(x, 16)
for x
in re.findall(
r"0x([0-9a-fA-F]+)", comp_m.group(1))
100 us = int(re.search(
r"uncompressed_size = (\d+)", text).group(1))
102 m.group(1): int(m.group(2))
103 for m
in re.finditer(
r"constexpr size_t off_(\w+) = (\d+);", text)
106 m.group(1): int(m.group(2))
107 for m
in re.finditer(
r"constexpr size_t count_(\w+) = (\d+);", text)
109 plain = zlib.decompress(compressed, -15)
112 f
"blob size mismatch: decompressed {len(plain)} != declared {us}"
122 """Load every table section from the current compressed blob."""
123 text = path.read_text()
125 plain = info[
"plain"]
127 counts = info[
"counts"]
128 sections: dict[str, list[int]] = {}
129 for name, kind
in SECTION_ORDER:
135 fmt =
"<" + PACK[kind] * count
136 raw = plain[off : off + count * w]
137 if len(raw) != count * w:
138 raise SystemExit(f
"truncated read of {name}")
139 vals = list(struct.unpack(fmt, raw))
140 sections[name] = vals
144def write_blob(sections: dict[str, Any], path: Path = BLOB_PATH) ->
None:
145 """Write sections (name -> list[int]) as src/table_blob.inc."""
147 layout: list[tuple[str, int, int, str, int]] = []
149 def align(a: int) ->
None:
153 for name, kind
in SECTION_ORDER:
154 if name
not in sections:
155 raise SystemExit(f
"write_blob missing section {name}")
156 vals = sections[name]
160 fmt =
"<" + PACK[kind] * len(vals)
161 blob.extend(struct.pack(fmt, *vals))
162 layout.append((name, off, len(vals) * WIDTH[kind], kind, len(vals)))
164 uncompressed = bytes(blob)
166 uncompressed_crc32 = zlib.crc32(uncompressed) & 0xFFFFFFFF
167 co = zlib.compressobj(9, zlib.DEFLATED, -15)
168 compressed = co.compress(uncompressed) + co.flush()
169 assert zlib.decompress(compressed, -15) == uncompressed
173 def block_rows(name: str, cols: int) -> int:
174 rows, rest = divmod(len(sections[name]), cols)
176 raise SystemExit(f
"{name} is not a whole number of rows of {cols}")
180 "decomposition_block_rows": block_rows(
"decomposition_block", 257),
181 "decomposition_block_cols": 257,
182 "ccc_block_rows": block_rows(
"ccc_block", 256),
183 "ccc_block_cols": 256,
184 "composition_block_rows": block_rows(
"composition_block", 257),
185 "composition_block_cols": 257,
186 "id_continue_count": len(sections[
"id_continue_flat"]) // 2,
187 "id_start_count": len(sections[
"id_start_flat"]) // 2,
188 "dir_table_count": len(sections[
"dir_start"]),
189 "combining_range_count": len(sections[
"combining_flat"]) // 2,
190 "joining_table_count": len(sections[
"joining_start"]),
193 def c_bytes(data: bytes, per: int = 16) -> str:
195 for i
in range(0, len(data), per):
196 chunk = data[i : i + per]
197 lines.append(
",".join(f
"0x{b:02x}" for b
in chunk) +
",")
198 return "\n".join(lines)
201 "// Auto-generated by scripts/pack_tables.py - do not edit.",
202 "// Compressed Unicode/IDNA tables (raw DEFLATE).",
203 "// clang-format off",
204 "#ifndef ADA_IDNA_TABLE_BLOB_H",
205 "#define ADA_IDNA_TABLE_BLOB_H",
206 "#include <cstdint>",
207 "#include <cstddef>",
208 "namespace ada::idna::table_blob {",
209 f
"constexpr size_t uncompressed_size = {len(uncompressed)};",
210 f
"constexpr size_t compressed_size = {len(compressed)};",
211 f
"constexpr uint32_t uncompressed_crc32 = 0x{uncompressed_crc32:08X}u;",
213 for k, v
in meta.items():
214 out.append(f
"constexpr size_t {k} = {v};")
215 out.append(
"// Offsets into the decompressed buffer:")
216 for name, off, _nbytes, _kind, count
in layout:
217 out.append(f
"constexpr size_t off_{name} = {off};")
218 out.append(f
"constexpr size_t count_{name} = {count};")
219 out.append(
"alignas(8) inline constexpr uint8_t compressed[] = {")
220 out.append(c_bytes(compressed))
222 out.append(
"} // namespace ada::idna::table_blob")
225 path.write_text(
"\n".join(out) +
"\n")
227 f
"Wrote {path.relative_to(ROOT)}: "
228 f
"{len(uncompressed)} raw -> {len(compressed)} compressed "
229 f
"({100 * len(compressed) / len(uncompressed):.1f}%)"
None write_mapping_constants(*, str version, int block_bits, int block_size, int block_mask, int sentinel_valid, int sentinel_disallowed, int ignored_idx, int bool_flag, int low_range_end, int high_ignored_start, int high_ignored_end, list[int] stage1, list[int] mixed_data, list[int] bool_words, list[int] utf8_table, Path path=MAPPING_CPP)