Repository navigation
Conversation
|
Ran some perf tests perftest.pyimport time
a = {"apricot": "test", "apple": "test2", "pear": "test3"}
b = {1: "test", 3: "test2", 2: "test3"}
c = {1.398: "test", 1.38: "test2", 1.3729: "test3"}
d = {"1233":"test", "1838":"test2", "1234-1837":"test3"}
e = {134:"test", "1234-1837":"test2", 777:"test3", "hello":"test3", 9999:"test4", 124.213:"test5"}
test_dicts = [a, b, c, d, e]
start = time.time()
from natsort import natsorted
end = time.time()
exec_time = end - start
print(f'0) {exec_time:.7f}s importing `natsort` package')
# Check types before converting all types to string, only use sorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'1) {exec_time:.7f}s string conversion only when mixed types, use only sorted')
# Check types before converting all types to string, use natsorted only when mixed types
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
else:
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'2) {exec_time:.7f}s string conversion only when mixed types, use natsorted if so else use sorted')
# Check types before converting all types to string, use only natsorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
end = time.time()
exec_time = end - start
print(f'3) {exec_time:.7f}s string conversion only when mixed types, use only natsorted')
# Systematically convert types to strings, use only sorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'4) {exec_time:.7f}s always convert types to strings, use only sorted')
# Systematically convert types to strings, use only natsorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = natsorted(keys)
end = time.time()
exec_time = end - start
print(f'5) {exec_time:.7f}s always convert types to strings, use only natsorted')So performance-wise, I'd recommend option 1 (let's not convert types when not mixed, as it might avoid literal sort behaviors I have not thought of with |
79d86d1 to
e832f15
Compare
e832f15 to
18a0363
Compare
|
I am more comfortable with Here is an alternative approach to It therefore works well with dict e = {134:"test", "1234-1837":"test2", 777:"test3", 9999:"test4", 1233: "test5"}
f = {134:"test", "1234-1837":"test2", 777:"test3", "hello":"test3", 9999:"test4", 124.213:"test5", 1233: "test6"}perftest.pyimport time
a = {"apricot": "test", "apple": "test2", "pear": "test3"}
b = {1: "test", 3: "test2", 2: "test3"}
c = {1.398: "test", 1.38: "test2", 1.3729: "test3"}
d = {"1233":"test", "1838":"test2", "1234-1837":"test3"}
e = {134:"test", "1234-1837":"test2", 777:"test3", 9999:"test4", 1233: "test5"}
f = {134:"test", "1234-1837":"test2", 777:"test3", "hello":"test3", 9999:"test4", 124.213:"test5", 1233: "test6"}
test_dicts = [a, b, c, d, e, f]
start = time.time()
from natsort import natsorted
end = time.time()
exec_time = end - start
print(f'0) {exec_time:.7f}s importing `natsort` package')
start = time.time()
import re
end = time.time()
exec_time = end - start
print(f'0) {exec_time:.7f}s importing `re` package')
# Check types before converting all types to string, only use sorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'1) {exec_time:.7f}s string conversion only when mixed types, use only sorted')
# Check types before converting all types to string, use natsorted only when mixed types
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
else:
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'2) {exec_time:.7f}s string conversion only when mixed types, use natsorted if so else use sorted')
# Check types before converting all types to string, use only natsorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
end = time.time()
exec_time = end - start
print(f'3) {exec_time:.7f}s string conversion only when mixed types, use only natsorted')
# Systematically convert types to strings, use only sorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'4) {exec_time:.7f}s always convert types to strings, use only sorted')
# Systematically convert types to strings, use only natsorted
start = time.time()
for test_dict in test_dicts:
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = natsorted(keys)
end = time.time()
exec_time = end - start
print(f'5) {exec_time:.7f}s always convert types to strings, use only natsorted')
# Check types before converting all types to string, use regex (re.sub) to manage port range case
start = time.time()
for test_dict in test_dicts:
keys = list(test_dict.keys())
keys_types = set(type(k) for k in keys)
# Keys are assumed to be of integer, float or/and string type
if len(keys_types) > 1:
#import re
replacements = {}
# For ports range (e.g. "150-194") case, sort using only start value
for k in keys:
if isinstance(k, str) and re.search(r"^([0-9]+)-[0-9]+$", k):
keys.remove(k)
k_sort = int(re.sub(r"^([0-9]+)-[0-9]+$",r"\1", k))
keys.append(k_sort)
replacements[k_sort] = k
# Make sure all keys are now of the same type
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
#raise MoulinetteError(f"Can't prettify dictionnary: one or more key are of an unexpected type", raw_msg=True)
print(f'MoulinetteError for {test_dict}')
else:
keys = sorted(keys)
keys = [replacements[k] if k in replacements else k for k in keys]
else:
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'6) {exec_time:.7f}s string conversion only when mixed types, manage port range case with re.sub')
# Check types before converting all types to string, use regex (re.search) and split() to manage port range case
start = time.time()
for test_dict in test_dicts:
keys = list(test_dict.keys())
keys_types = set(type(k) for k in keys)
# Keys are assumed to be of integer, float or/and string type
if len(keys_types) > 1:
#import re
replacements = {}
# For ports range (e.g. "150-194") case, sort using only start value
for k in keys:
if isinstance(k, str) and re.search(r"^([0-9]+)-[0-9]+$", k):
k_sort = int(k.split("-")[0])
keys.remove(k)
replacements[k_sort] = k
keys.append(k_sort)
# Make sure all keys are now of the same type
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
#raise MoulinetteError(f"Can't prettify dictionnary: one or more key are of an unexpected type", raw_msg=True)
print(f'MoulinetteError for {test_dict}')
else:
keys = sorted(keys)
keys = [replacements[k] if k in replacements else k for k in keys]
else:
keys = sorted(keys)
end = time.time()
exec_time = end - start
print(f'7) {exec_time:.7f}s string conversion only when mixed types, manage port range case with re.search + split')
Note that |
|
Hello, Thanks for the PR and this important fix. In my opinion I would prefer to not add natsort as the dependency and try to only use the native sort function, because this lib seem not well maintained, the last releases seem to be 3 year old. So even if the perf is a bit lower I prefer to use the native Python function. But it's still my opinion, let's see maybe what other people think about this. |
Perf actually seems to be better with native Python, but it seems more likely to break when facing new unforeseen cases.
To be fair regarding the lib's maintenance history, there seems to be new releases on the way: https://github.com/SethMMorton/natsort/pulls |
| keys = [str(k) for k in keys] | ||
| # `natsorted()` does proper numeric sorting when dealing with | ||
| # numbers even when the data is of string type | ||
| from natsort import natsorted | ||
| keys = natsorted(keys) |
There was a problem hiding this comment.
In my opinion we can do just something like this:
| keys = [str(k) for k in keys] | |
| # `natsorted()` does proper numeric sorting when dealing with | |
| # numbers even when the data is of string type | |
| from natsort import natsorted | |
| keys = natsorted(keys) | |
| keys = sorted(keys, key=lambda k: str(k)) |
There was a problem hiding this comment.
That is enough to fix the bug, however the sort becomes literal and it would look weird in various cases, like typically in the linked issue (firewall port list), e.g.:
{134:"test", "1234-1837":"test2", 111111:"test3", 9999:"test4", 1233: "test5"}will become:
[111111, 1233, '1234-1837', 134, 9999]instead of expected:
[134, 1233, '1234-1837', 9999, 111111]This is basically the same approach as in my test 4) above, even if the use of lambda function is probably better (saves ~20 tens of microsecond!).
I solved this issue for this specific case in tests 6 and 7 (with the help of additional re module, but that's almost native python I guess), cf. second perftest.py (in folded code block) above.
|
Note that i am not totally sure that yunohost firewall support |
I ran the regen-conf code using the And port ranges indicated with a dash do seem supported in |
|
We are using the notation like |
On the UI side only though since migration to |
|
you are right, it's legacy stuff that would need to be updated. |
|
To summarize, there is a decision left to take here with mainly 3 choices:
|
Co-authored-by: zamentur <4080016+zamentur@users.noreply.github.com> Co-authored-by: sitegui <2801575+sitegui@users.noreply.github.com>
|
@josue @Salamandar In the contributors' meeting, we've been thinking to keep natural sort order, but using a much simpler sorting function than Sorting the following sample:
[124, '1113-1837', 777, 'hello', 999.98, 999.94, '1.2.3~yn4', '1.2.3~yn1', '1.2.3~yn12', '1.2.1~yn15', 'dom2', 'dom15', 'DOM1']
0) 0.0485466s importing `natsort` package
0) 0.0000012s importing `re` package
1) string conversion only when mixed types, use only sorted
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn12', '1.2.3~yn4', '1113-1837', '124', '777', '999.94', '999.98', 'DOM1', 'dom15', 'dom2', 'hello']
0.0000246s
2) string conversion only when mixed types, use natsorted if so else use sorted
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn4', '1.2.3~yn12', '124', '777', '999.94', '999.98', '1113-1837', 'DOM1', 'dom2', 'dom15', 'hello']
0.0002000s
3) string conversion only when mixed types, use only natsorted
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn4', '1.2.3~yn12', '124', '777', '999.94', '999.98', '1113-1837', 'DOM1', 'dom2', 'dom15', 'hello']
0.0001700s
4) always convert types to strings, use only sorted
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn12', '1.2.3~yn4', '1113-1837', '124', '777', '999.94', '999.98', 'DOM1', 'dom15', 'dom2', 'hello']
0.0000148s
5) always convert types to strings, use only natsorted
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn4', '1.2.3~yn12', '124', '777', '999.94', '999.98', '1113-1837', 'DOM1', 'dom2', 'dom15', 'hello']
0.0001364s
MoulinetteError for {124: 'test', '1113-1837': 'test2', 777: 'test3', 'hello': 'test3', 999.98: 'test4', 999.94: 'test5', '1.2.3~yn4': 'test6', '1.2.3~yn1': 'test7', '1.2.3~yn12': 'test8', '1.2.1~yn15': 'test9', 'dom2': 'test10', 'dom15': 'test11', 'DOM1': 'test12'}
6) string conversion only when mixed types, manage port range case with re.sub
[124, 777, 'hello', 999.98, 999.94, '1.2.3~yn4', '1.2.3~yn1', '1.2.3~yn12', '1.2.1~yn15', 'dom2', 'dom15', 'DOM1', 1113]
0.0001557s
MoulinetteError for {124: 'test', '1113-1837': 'test2', 777: 'test3', 'hello': 'test3', 999.98: 'test4', 999.94: 'test5', '1.2.3~yn4': 'test6', '1.2.3~yn1': 'test7', '1.2.3~yn12': 'test8', '1.2.1~yn15': 'test9', 'dom2': 'test10', 'dom15': 'test11', 'DOM1': 'test12'}
7) string conversion only when mixed types, manage port range case with re.search + split
[124, 777, 'hello', 999.98, 999.94, '1.2.3~yn4', '1.2.3~yn1', '1.2.3~yn12', '1.2.1~yn15', 'dom2', 'dom15', 'DOM1', 1113]
0.0000572s
8) always convert types to strings, use only custom natural_sort
['1.2.1~yn15', '1.2.3~yn1', '1.2.3~yn4', '1.2.3~yn12', 124, 777, 999.94, 999.98, '1113-1837', 'DOM1', 'dom2', 'dom15', 'hello']
0.0001249sperftest.pyimport time
a = {"apricot": "test", "apple": "test2", "pear": "test3"}
b = {1: "test", 3: "test2", 2: "test3"}
c = {1.398: "test", 1.38: "test2", 1.3729: "test3"}
d = {"1233":"test", "1838":"test2", "1234-1837":"test3"}
e = {134:"test", "1234-1837":"test2", 777:"test3", 9999:"test4", 1233: "test5"}
f = {134:"test", "1234-1837":"test2", 777:"test3", "hello":"test3", 9999:"test4", 124.213:"test5", 1233: "test6"}
g = {124:"test", "1113-1837":"test2", 777:"test3", "hello":"test3", 999.98:"test4", 999.94:"test5", "1.2.3~yn4": "test6", "1.2.3~yn1": "test7", "1.2.3~yn12": "test8", "1.2.1~yn15": "test9", "dom2": "test10", "dom15": "test11", "DOM1": "test12"}
#test_dicts = [a, b, c, d, e, f, g]
test_dicts = [g]
print('Sorting the following sample:')
for test_dict in test_dicts:
print (" " + str(list(test_dict.keys())))
start = time.time()
from natsort import natsorted
end = time.time()
exec_time = end - start
print(f'0) {exec_time:.7f}s importing `natsort` package')
start = time.time()
import re
end = time.time()
exec_time = end - start
print(f'0) {exec_time:.7f}s importing `re` package')
# 1. Check types before converting all types to string, only use sorted
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = sorted(keys)
print(f'1) string conversion only when mixed types, use only sorted')
print(" " + str(keys)) if test_dict is g else ''
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 2. Check types before converting all types to string, use natsorted only when mixed types
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
else:
keys = sorted(keys)
if i == 0:
print(f'2) string conversion only when mixed types, use natsorted if so else use sorted')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 3. Check types before converting all types to string, use only natsorted
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
keys = [str(k) for k in keys]
keys = natsorted(keys)
if i == 0:
print(f'3) string conversion only when mixed types, use only natsorted')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 4. Systematically convert types to strings, use only sorted
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = sorted(keys)
if i == 0:
print(f'4) always convert types to strings, use only sorted')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 5. Systematically convert types to strings, use only natsorted
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys = [str(k) for k in keys]
keys = natsorted(keys)
if i == 0:
print(f'5) always convert types to strings, use only natsorted')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 6. Check types before converting all types to string, use regex (re.sub) to manage port range case
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = list(test_dict.keys())
keys_types = set(type(k) for k in keys)
# Keys are assumed to be of integer, float or/and string type
if len(keys_types) > 1:
#import re
replacements = {}
# For ports range (e.g. "150-194") case, sort using only start value
for k in keys:
if isinstance(k, str) and re.search(r"^([0-9]+)-[0-9]+$", k):
keys.remove(k)
k_sort = int(re.sub(r"^([0-9]+)-[0-9]+$",r"\1", k))
keys.append(k_sort)
replacements[k_sort] = k
# Make sure all keys are now of the same type
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
#raise MoulinetteError(f"Can't prettify dictionnary: one or more key are of an unexpected type", raw_msg=True)
print(f'MoulinetteError for {test_dict}')
else:
keys = sorted(keys)
keys = [replacements[k] if k in replacements else k for k in keys]
else:
keys = sorted(keys)
if i == 0:
print(f'6) string conversion only when mixed types, manage port range case with re.sub')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 7. Check types before converting all types to string, use regex (re.search) and split() to manage port range case
start = time.time()
for i, test_dict in enumerate(test_dicts):
keys = list(test_dict.keys())
keys_types = set(type(k) for k in keys)
# Keys are assumed to be of integer, float or/and string type
if len(keys_types) > 1:
#import re
replacements = {}
# For ports range (e.g. "150-194") case, sort using only start value
for k in keys:
if isinstance(k, str) and re.search(r"^([0-9]+)-[0-9]+$", k):
k_sort = int(k.split("-")[0])
keys.remove(k)
replacements[k_sort] = k
keys.append(k_sort)
# Make sure all keys are now of the same type
keys_types = set(type(k) for k in keys)
if len(keys_types) > 1:
#raise MoulinetteError(f"Can't prettify dictionnary: one or more key are of an unexpected type", raw_msg=True)
print(f'MoulinetteError for {test_dict}')
else:
keys = sorted(keys)
keys = [replacements[k] if k in replacements else k for k in keys]
else:
keys = sorted(keys)
if i == 0:
print(f'7) string conversion only when mixed types, manage port range case with re.search + split')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s')
# 8. Systematically convert types to strings, use only custom natural_sort
start = time.time()
def natural_sort(l):
# borrowed from https://stackoverflow.com/questions/4836710/is-there-a-built-in-function-for-string-natural-sort/4836734#4836734
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', str(key))]
return sorted(l, key=alphanum_key)
for i, test_dict in enumerate(test_dicts):
keys = test_dict.keys()
keys = natural_sort(keys)
if i == 0:
print(f'8) always convert types to strings, use only custom natural_sort')
print(" " + str(keys))
end = time.time()
exec_time = end - start
print(f'{exec_time:.7f}s') |
Fixes YunoHost/issues#2910
sorted()crashes when keys are of mixed types so this PR fixes this by converting all types to strings when mixed types are detected.This PR assumes that keys' original types would only be string, integer or float.
It uses additional module
natsortinstead ofsorted()because it performs better at sorting mixed types (once converted to string) as show below tests:These two lines are optional since it should also work well when systematically converting keys to string type. But performances might be slightly lower (untested) ?