build_pycompat_tokenizer.py

#!/usr/bin/python3
from dateutil.parser import _timelex

from build_pycompat import tests

def main():
    with open('src/tests/pycompat_tokenizer.rs', 'w+') as handle:
        handle.write(TEST_HEADER)

        counter = 0
        for _, test_strings in tests.items():
            for s in test_strings:
                handle.write(build_test(counter, s))
                counter += 1

def build_test(i, test_string):
    python_tokens = list(_timelex(test_string))
    formatted_tokens = 'vec!["' + '", "'.join(python_tokens) + '"]'
    return f'''
#[test]
fn test_tokenize{i}() {{
    let comp = {formatted_tokens};
    tokenize_assert("{test_string}", comp);
}}\n'''


TEST_HEADER = '''
//! This code has been generated by running the `build_pycompat_tokenizer.py` script
//! in the repository root. Please do not edit it, as your edits will be destroyed
//! upon re-running code generation.

use tokenize::Tokenizer;

fn tokenize_assert(test_str: &str, comparison: Vec<&str>) {
    let tokens: Vec<String> = Tokenizer::new(test_str).collect();
    assert_eq!(tokens, comparison, "Tokenizing mismatch for `{}`", test_str);
}\n'''

if __name__ == '__main__':
    main()