Ada
4.1.1
Fast spec-compliant URL parser
Toggle main menu visibility
Loading...
Searching...
No Matches
derived_table.py
Go to the documentation of this file.
1
import
os.path
2
import
requests
3
import
re
4
import
textwrap
5
import
sys
6
if
sys.version_info[0] < 3:
7
print(
'You need to run this with Python 3'
)
8
sys.exit(1)
9
10
indent =
' '
* 4
11
12
def
fill
(text):
13
tmp = textwrap.fill(text)
14
return
textwrap.indent(tmp, indent)
15
16
def
filltab
(text):
17
tmp = textwrap.fill(text, width=120)
18
return
textwrap.indent(tmp,
'\t'
)
19
20
def
cpp_array_initializer
(arr):
21
return
'{%s}'
% (
', '
.join(map(str, arr)))
22
23
def
multiline_cpp_array_initializer
(arr):
24
alllines =
'{\n'
25
current_line =
'\t'
26
for
i
in
arr:
27
if(len(current_line) > 80):
28
if(current_line[-1] ==
' '
):
29
current_line = current_line[:-1]
30
alllines += current_line +
'\n'
31
current_line =
'\t'
32
current_line += str(i)
33
if(i != len(arr)-1): current_line +=
', '
34
if(len(current_line) > 0):
35
if(current_line[-1] ==
' '
):
36
current_line = current_line[:-1]
37
alllines += current_line +
'\n'
38
alllines +=
'\n}'
39
return
alllines
40
41
def
compose2
(f, g):
42
return
lambda
x: f(g(x))
43
44
def
cpp_arrayarray_initializer
(arr):
45
alllines =
'{'
46
for
i
in
range(0,len(arr), 4):
47
alllines +=
'\n\t'
+
', '
.join(map(cpp_array_initializer,arr[i:i+4]))
48
if(i + 4 < len(arr)): alllines +=
','
49
alllines +=
'\n}'
50
return
alllines
51
52
53
# Keep in step with UNICODE_VERSION in scripts/idna_table.py.
54
UNICODE_VERSION =
"18.0.0"
55
urlderived = f
"https://www.unicode.org/Public/{UNICODE_VERSION}/ucd/DerivedCoreProperties.txt"
56
derivedfilename = f
"DerivedCoreProperties-{UNICODE_VERSION}.txt"
57
def
get_derived_table
():
58
if(
not
os.path.exists(derivedfilename)):
59
tablefile = requests.get(urlderived)
60
tablefile.raise_for_status()
61
with
open(derivedfilename,
'wb'
)
as
file:
62
file.write(tablefile.content)
63
with
open(derivedfilename,
'r'
)
as
file:
64
table_data = file.read()
65
version =
get_version
(table_data)
66
if
version != UNICODE_VERSION:
67
raise
SystemExit(f
"{derivedfilename} is version {version}, expected "
68
f
"{UNICODE_VERSION}; delete it to download again"
)
69
return
table_data
70
71
def
get_version
(table_data):
72
return
re.search(
r"# DerivedCoreProperties-(.*)\.txt"
, table_data).group(1)
73
74
75
def
parse_id_start_section
(text):
76
start_pattern = re.compile(
r'# Derived Property: ID_Start'
)
77
end_pattern = re.compile(
r'# Derived Property: ID_Continue'
)
78
79
# Find the start and end indices of the section
80
start = start_pattern.search(text)
81
end = end_pattern.search(text)
82
83
if
start
is
None
or
end
is
None
:
84
raise
ValueError(
"Section markers not found in the text."
)
85
86
# Extract the relevant section
87
section = text[start.end():end.start()].strip().split(
'\n'
)
88
result = []
89
for
line
in
section:
90
line = line.strip()
91
if
not
line
or
line.startswith(
'#'
):
92
continue
93
94
parts = line.split(
';'
)[0].strip().split(
'..'
)
95
96
# Convert hex to int and handle single value
97
if
len(parts) == 1:
98
start_value = int(parts[0], 16)
99
result.append([start_value, start_value])
100
elif
len(parts) == 2:
101
start_value = int(parts[0], 16)
102
end_value = int(parts[1], 16)
103
result.append([start_value, end_value])
104
else
:
105
print(f
"Warning: Line format not recognized: {line}"
)
106
107
return
result
108
109
110
def
parse_id_continue_section
(text):
111
start_pattern = re.compile(
r'# Derived Property: ID_Continue'
)
112
end_pattern = re.compile(
r'# Derived Property: XID_Start'
)
113
114
# Find the start and end indices of the section
115
start = start_pattern.search(text)
116
end = end_pattern.search(text)
117
118
if
start
is
None
or
end
is
None
:
119
raise
ValueError(
"Section markers not found in the text."
)
120
121
# Extract the relevant section
122
section = text[start.end():end.start()].strip().split(
'\n'
)
123
result = []
124
for
line
in
section:
125
line = line.strip()
126
if
not
line
or
line.startswith(
'#'
):
127
continue
128
129
parts = line.split(
';'
)[0].strip().split(
'..'
)
130
131
# Convert hex to int and handle single value
132
if
len(parts) == 1:
133
start_value = int(parts[0], 16)
134
result.append([start_value, start_value])
135
elif
len(parts) == 2:
136
start_value = int(parts[0], 16)
137
end_value = int(parts[1], 16)
138
result.append([start_value, end_value])
139
else
:
140
print(f
"Warning: Line format not recognized: {line}"
)
141
142
return
result
143
144
def
generate_derived
():
145
"""Build identifier range tables from DerivedCoreProperties.txt."""
146
table_data =
get_derived_table
()
147
version =
get_version
(table_data)
148
id_start =
parse_id_start_section
(table_data)
149
id_continue =
parse_id_continue_section
(table_data)
150
return
{
151
"version"
: version,
152
"id_start"
: id_start,
153
"id_continue"
: id_continue,
154
}
155
156
157
def
print_derived
():
158
"""Print expanded C++ identifier tables to stdout (debug / legacy)."""
159
data =
generate_derived
()
160
print(
"// IDNA "
, data[
"version"
])
161
id_start = data[
"id_start"
]
162
id_continue = data[
"id_continue"
]
163
print(
"""
164
// clang-format off
165
#ifndef ADA_IDNA_IDENTIFIER_TABLES_H
166
#define ADA_IDNA_IDENTIFIER_TABLES_H
167
#include <cstdint>
168
169
namespace ada::idna {
170
"""
)
171
print(
"const uint32_t id_continue["
+str(len(id_continue))+
"][2] ="
)
172
print(
cpp_arrayarray_initializer
(id_continue), end=
";\n"
)
173
print(
"const uint32_t id_start["
+str(len(id_start))+
"][2] ="
)
174
print(
cpp_arrayarray_initializer
(id_start), end=
";\n"
)
175
print(
"""
176
177
} // namespace ada::idna
178
#endif // ADA_IDNA_IDENTIFIER_TABLES_H
179
"""
)
180
181
182
def
write_derived
():
183
"""Write id_tables stub and update table_blob.inc with new ranges."""
184
script_dir = os.path.dirname(os.path.abspath(__file__))
185
if
script_dir
not
in
sys.path:
186
sys.path.insert(0, script_dir)
187
import
pack_tables
188
189
data =
generate_derived
()
190
pack_tables.write_id_tables_stub
(
191
version=data[
"version"
],
192
id_continue=data[
"id_continue"
],
193
id_start=data[
"id_start"
],
194
)
195
pack_tables.update_id_in_blob
(data[
"id_continue"
], data[
"id_start"
])
196
print(
"Identifier tables regenerated and packed into table_blob.inc"
)
197
198
199
if
__name__ ==
"__main__"
:
200
if
len(sys.argv) > 1
and
sys.argv[1]
in
(
"--write"
,
"-w"
):
201
write_derived
()
202
elif
len(sys.argv) > 1
and
sys.argv[1]
in
(
"-h"
,
"--help"
):
203
print(
"Usage: python3 scripts/derived_table.py [--write]"
)
204
print(
" (default) Print expanded C++ identifier tables to stdout"
)
205
print(
" --write Write stub + update src/table_blob.inc"
)
206
else
:
207
print_derived
()
derived_table.multiline_cpp_array_initializer
multiline_cpp_array_initializer(arr)
Definition
derived_table.py:23
derived_table.write_derived
write_derived()
Definition
derived_table.py:182
derived_table.compose2
compose2(f, g)
Definition
derived_table.py:41
derived_table.cpp_array_initializer
cpp_array_initializer(arr)
Definition
derived_table.py:20
derived_table.get_derived_table
get_derived_table()
Definition
derived_table.py:57
derived_table.filltab
filltab(text)
Definition
derived_table.py:16
derived_table.cpp_arrayarray_initializer
cpp_arrayarray_initializer(arr)
Definition
derived_table.py:44
derived_table.fill
fill(text)
Definition
derived_table.py:12
derived_table.generate_derived
generate_derived()
Definition
derived_table.py:144
derived_table.parse_id_start_section
parse_id_start_section(text)
Definition
derived_table.py:75
derived_table.parse_id_continue_section
parse_id_continue_section(text)
Definition
derived_table.py:110
derived_table.print_derived
print_derived()
Definition
derived_table.py:157
derived_table.get_version
get_version(table_data)
Definition
derived_table.py:71
pack_tables.write_id_tables_stub
None write_id_tables_stub(*, str version, list[list[int]] id_continue, list[list[int]] id_start, Path path=ID_CPP)
Definition
pack_tables.py:313
pack_tables.update_id_in_blob
None update_id_in_blob(list[list[int]] id_continue, list[list[int]] id_start)
Definition
pack_tables.py:353