1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
import pathlib
import re
import unicodedata
def list(xs):
return '[' + ', '.join([f'"{x}"' for x in xs]) + ']'
def path_part(name):
simplified = ''.join([alnum_or_space(c) for c in unaccent(name.lower())])
return '-'.join(simplified.split())
def unaccent(s):
return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')
def alnum_or_space(c):
if c.isalnum():
return c
else:
return ' '
def extension(path):
return pathlib.Path(path).suffix
def cleanup_text(s, lang):
s = re.sub('\'', '’', s)
s = re.sub(r'\.\.\.', '…', s)
s = re.sub(r'\. \. \.', '…', s)
s = cleanup_double_quotes(s, lang)
if lang == 'fr':
s = re.sub('“', '«', s)
s = re.sub('”', '»', s)
# Replace space by insecable spaces
s = re.sub(r' ([:?\!»])', r' \1', s)
s = re.sub('« ', '« ', s)
# Add missing insecable spaces
s = re.sub(r'([^ ]):', r'\1 :', s)
s = re.sub(r'([^ ])\?', r'\1 ?', s)
s = re.sub(r'([^ ])\!', r'\1 !', s)
s = re.sub(r'([^ ])»', r'\1 »', s)
s = re.sub(r'«([^ ])', r'« \1', s)
elif lang == 'en':
s = re.sub('«', '“', s)
s = re.sub('»', '”', s)
return s
def cleanup_double_quotes(s, lang):
res = ''
quoted = False
for c in s:
if c == '"':
if quoted:
quoted = False
if lang == 'fr':
res += '»'
elif lang == 'en':
res += '”'
else:
quoted = True
if lang == 'fr':
res += '«'
elif lang == 'en':
res += '“'
else:
res += c
return res
|
