From ebb7c3a3d1f3fb76232d69530db5eb44fbdd1ea9 Mon Sep 17 00:00:00 2001 From: Eli Uriegas <1700823+seemethere@users.noreply.github.com> Date: Thu, 18 Feb 2021 15:03:05 -0800 Subject: [PATCH] .circleci: Add Python 3.9 support (#1139) --- .circleci/config.yml | 205 +++++++++++++++++- .circleci/config.yml.in | 33 ++- .circleci/regenerate.py | 2 +- .../unittest/linux/scripts/environment.yml | 3 +- .circleci/unittest/linux/scripts/install.sh | 2 +- .../unittest/windows/scripts/environment.yml | 3 +- .circleci/unittest/windows/scripts/install.sh | 2 +- packaging/pkg_helpers.bash | 6 +- test/common/torchtext_test_case.py | 2 +- torchtext/datasets/iwslt.py | 6 +- torchtext/datasets/wmt14.py | 6 +- 11 files changed, 240 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f18b086090..853ca37d24 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,16 @@ commands: our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -71,7 +81,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -92,6 +102,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -127,12 +138,13 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: build command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install ${CONDA_CHANNEL_FLAGS} -yq conda-build "conda-package-handling!=1.5.0" bash packaging/build_conda.sh rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - store_artifacts: @@ -175,6 +187,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -232,13 +245,14 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly - conda install -v -y -c file://$HOME/workspace/conda-bld torchtext + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-${UPLOAD_CHANNEL} pytorch + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c file://$HOME/workspace/conda-bld torchtext - run: name: smoke test command: | @@ -291,6 +305,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -299,8 +314,8 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-"${UPLOAD_CHANNEL}" pytorch cpuonly - conda install -v -y -c ~/workspace/conda-bld torchtext + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-"${UPLOAD_CHANNEL}" pytorch + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c ~/workspace/conda-bld torchtext - run: name: smoke test command: | @@ -335,11 +350,12 @@ jobs: unittest_linux: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -392,6 +408,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -440,7 +457,7 @@ jobs: stylecheck: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: medium steps: - checkout @@ -538,6 +555,9 @@ workflows: - binary_linux_wheel: name: binary_linux_wheel_py3.8 python_version: '3.8' + - binary_linux_wheel: + name: binary_linux_wheel_py3.9 + python_version: '3.9' - binary_macos_wheel: name: binary_macos_wheel_py3.6 python_version: '3.6' @@ -547,6 +567,9 @@ workflows: - binary_macos_wheel: name: binary_macos_wheel_py3.8 python_version: '3.8' + - binary_macos_wheel: + name: binary_macos_wheel_py3.9 + python_version: '3.9' - binary_windows_wheel: name: binary_windows_wheel_py3.6 python_version: '3.6' @@ -556,6 +579,9 @@ workflows: - binary_windows_wheel: name: binary_windows_wheel_py3.8 python_version: '3.8' + - binary_windows_wheel: + name: binary_windows_wheel_py3.9 + python_version: '3.9' - binary_linux_conda: name: binary_linux_conda_py3.6 python_version: '3.6' @@ -565,6 +591,9 @@ workflows: - binary_linux_conda: name: binary_linux_conda_py3.8 python_version: '3.8' + - binary_linux_conda: + name: binary_linux_conda_py3.9 + python_version: '3.9' - binary_macos_conda: name: binary_macos_conda_py3.6 python_version: '3.6' @@ -574,6 +603,9 @@ workflows: - binary_macos_conda: name: binary_macos_conda_py3.8 python_version: '3.8' + - binary_macos_conda: + name: binary_macos_conda_py3.9 + python_version: '3.9' - binary_windows_conda: name: binary_windows_conda_py3.6 python_version: '3.6' @@ -583,6 +615,9 @@ workflows: - binary_windows_conda: name: binary_windows_conda_py3.8 python_version: '3.8' + - binary_windows_conda: + name: binary_windows_conda_py3.9 + python_version: '3.9' - build_docs: name: build_docs python_version: '3.8' @@ -613,6 +648,9 @@ workflows: - unittest_linux: name: unittest_linux_py3.8 python_version: '3.8' + - unittest_linux: + name: unittest_linux_py3.9 + python_version: '3.9' - unittest_windows: name: unittest_windows_py3.6 python_version: '3.6' @@ -622,6 +660,9 @@ workflows: - unittest_windows: name: unittest_windows_py3.8 python_version: '3.8' + - unittest_windows: + name: unittest_windows_py3.9 + python_version: '3.9' nightly: jobs: - circleci_consistency: @@ -712,6 +753,34 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_wheel_py3.8_upload + - binary_linux_wheel: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_upload + requires: + - nightly_binary_linux_wheel_py3.9 + - smoke_test_linux_pip: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_linux_wheel_py3.9_upload - binary_macos_wheel: filters: branches: @@ -766,6 +835,24 @@ workflows: name: nightly_binary_macos_wheel_py3.8_upload requires: - nightly_binary_macos_wheel_py3.8 + - binary_macos_wheel: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_wheel_py3.9_upload + requires: + - nightly_binary_macos_wheel_py3.9 - binary_windows_wheel: filters: branches: @@ -850,6 +937,34 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_wheel_py3.8_upload + - binary_windows_wheel: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9 + python_version: '3.9' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_upload + requires: + - nightly_binary_windows_wheel_py3.9 + - smoke_test_windows_pip: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_wheel_py3.9_smoke_test_pip + python_version: '3.9' + requires: + - nightly_binary_windows_wheel_py3.9_upload - binary_linux_conda: filters: branches: @@ -934,6 +1049,34 @@ workflows: python_version: '3.8' requires: - nightly_binary_linux_conda_py3.8_upload + - binary_linux_conda: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_upload + requires: + - nightly_binary_linux_conda_py3.9 + - smoke_test_linux_conda: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_linux_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_linux_conda_py3.9_upload - binary_macos_conda: filters: branches: @@ -988,6 +1131,24 @@ workflows: name: nightly_binary_macos_conda_py3.8_upload requires: - nightly_binary_macos_conda_py3.8 + - binary_macos_conda: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_macos_conda_py3.9_upload + requires: + - nightly_binary_macos_conda_py3.9 - binary_windows_conda: filters: branches: @@ -1072,6 +1233,34 @@ workflows: python_version: '3.8' requires: - nightly_binary_windows_conda_py3.8_upload + - binary_windows_conda: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9 + python_version: '3.9' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_upload + requires: + - nightly_binary_windows_conda_py3.9 + - smoke_test_windows_conda: + filters: + branches: + only: nightly + tags: + only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ + name: nightly_binary_windows_conda_py3.9_smoke_test_conda + python_version: '3.9' + requires: + - nightly_binary_windows_conda_py3.9_upload docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 12c4c242bd..c2b56e78e5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -29,6 +29,16 @@ commands: our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} + load_conda_channel_flags: + description: "Determines whether we need extra conda channels" + steps: + - run: + name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV + command: | + CONDA_CHANNEL_FLAGS="" + if [[ "${PYTHON_VERSION}" = *3.9* ]]; then + echo "export CONDA_CHANNEL_FLAGS=-c=conda-forge" >> ${BASH_ENV} + fi binary_common: &binary_common parameters: @@ -71,7 +81,7 @@ jobs: binary_linux_wheel: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout @@ -92,6 +102,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 @@ -127,12 +138,13 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: build command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base - conda install -yq conda-build "conda-package-handling!=1.5.0" + conda install ${CONDA_CHANNEL_FLAGS} -yq conda-build "conda-package-handling!=1.5.0" bash packaging/build_conda.sh rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - store_artifacts: @@ -175,6 +187,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh @@ -232,13 +245,14 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly - conda install -v -y -c file://$HOME/workspace/conda-bld torchtext + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-${UPLOAD_CHANNEL} pytorch + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c file://$HOME/workspace/conda-bld torchtext - run: name: smoke test command: | @@ -291,6 +305,7 @@ jobs: - attach_workspace: at: ~/workspace - designate_upload_channel + - load_conda_channel_flags - run: name: install binaries command: | @@ -299,8 +314,8 @@ jobs: conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} - conda install -v -y -c pytorch-"${UPLOAD_CHANNEL}" pytorch cpuonly - conda install -v -y -c ~/workspace/conda-bld torchtext + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-"${UPLOAD_CHANNEL}" pytorch + conda install -v -y ${CONDA_CHANNEL_FLAGS} -c ~/workspace/conda-bld torchtext - run: name: smoke test command: | @@ -335,11 +350,12 @@ jobs: unittest_linux: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: 2xlarge+ steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -392,6 +408,7 @@ jobs: steps: - checkout - designate_upload_channel + - load_conda_channel_flags - run: name: Generate cache key # This will refresh cache on Sundays, nightly build should generate new cache. @@ -440,7 +457,7 @@ jobs: stylecheck: <<: *binary_common docker: - - image: "pytorch/manylinux-cuda100" + - image: "pytorch/manylinux-cuda102" resource_class: medium steps: - checkout diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 6ab710bdf6..fc19cae89d 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -19,7 +19,7 @@ import os.path -PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] +PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] DOC_VERSION = ('linux', '3.8') diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index 282877c207..e1e9f2cda8 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -1,7 +1,8 @@ channels: - defaults + - conda-forge dependencies: - - flake8==3.7.9 + - flake8>=3.7.9 - codecov - pip - pip: diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh index 87e027150e..e9201b266b 100755 --- a/.circleci/unittest/linux/scripts/install.sh +++ b/.circleci/unittest/linux/scripts/install.sh @@ -11,7 +11,7 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env printf "* Installing PyTorch\n" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch cpuonly +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch cpuonly printf "* Installing torchtext\n" git submodule update --init --recursive diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 2d7c790b91..9716f09114 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -1,7 +1,8 @@ channels: - defaults + - conda-forge dependencies: - - flake8==3.7.9 + - flake8>=3.7.9 - codecov - pip - pip: diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh index 5ef74ea6cc..622ebc1cd1 100644 --- a/.circleci/unittest/windows/scripts/install.sh +++ b/.circleci/unittest/windows/scripts/install.sh @@ -16,7 +16,7 @@ eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" conda activate ./env printf "* Installing PyTorch\n" -conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch cpuonly +conda install -y -c "pytorch-${UPLOAD_CHANNEL}" ${CONDA_CHANNEL_FLAGS} pytorch cpuonly printf "* Installing torchtext\n" git submodule update --init --recursive diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash index e2e411b2ee..8f1e24f1da 100644 --- a/packaging/pkg_helpers.bash +++ b/packaging/pkg_helpers.bash @@ -142,6 +142,7 @@ setup_wheel_python() { 3.6) python_abi=cp36-cp36m ;; 3.7) python_abi=cp37-cp37m ;; 3.8) python_abi=cp38-cp38 ;; + 3.9) python_abi=cp39-cp39 ;; *) echo "Unrecognized PYTHON_VERSION=$PYTHON_VERSION" exit 1 @@ -178,11 +179,12 @@ setup_pip_pytorch_version() { # # You MUST have populated PYTORCH_VERSION_SUFFIX before hand. setup_conda_pytorch_constraint() { + CONDA_CHANNEL_FLAGS=${CONDA_CHANNEL_FLAGS:-} if [[ -z "$PYTORCH_VERSION" ]]; then - export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch-nightly" export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | python -c "import sys, json, re; print(re.sub(r'\\+.*$', '', json.load(sys.stdin)['pytorch'][-1]['version']))")" else - export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}" + export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c pytorch -c pytorch-${UPLOAD_CHANNEL}" fi if [[ "$CU_VERSION" == cpu ]]; then export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}" diff --git a/test/common/torchtext_test_case.py b/test/common/torchtext_test_case.py index 322eb32292..c456cd705b 100644 --- a/test/common/torchtext_test_case.py +++ b/test/common/torchtext_test_case.py @@ -108,7 +108,7 @@ def make_mock_dataset(self, num_examples=30, num_labels=3): texts = [str(i) for i in range(num_examples)] labels = list(range(num_labels)) * num_repetitions - labels = [str(l) for l in labels[:num_examples]] + labels = [str(line) for line in labels[:num_examples]] dict_dataset = [ {'text': t, 'label': l} for t, l in zip(texts, labels) diff --git a/torchtext/datasets/iwslt.py b/torchtext/datasets/iwslt.py index 8426b84bc5..b32773669b 100644 --- a/torchtext/datasets/iwslt.py +++ b/torchtext/datasets/iwslt.py @@ -43,13 +43,13 @@ def _clean_tags_file(f_orig): f_txt = f_orig.replace('.tags', '') with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \ io.open(f_orig, mode='r', encoding='utf-8') as fd_orig: - for l in fd_orig: - if not any(tag in l for tag in xml_tags): + for line in fd_orig: + if not any(tag in line for tag in xml_tags): # TODO: Fix utf-8 next line mark # fd_txt.write(l.strip() + '\n') # fd_txt.write(l.strip() + u"\u0085") # fd_txt.write(l.lstrip()) - fd_txt.write(l.strip() + '\n') + fd_txt.write(line.strip() + '\n') def _construct_filenames(filename, languages): diff --git a/torchtext/datasets/wmt14.py b/torchtext/datasets/wmt14.py index dcda72c212..d84f9b5a98 100644 --- a/torchtext/datasets/wmt14.py +++ b/torchtext/datasets/wmt14.py @@ -43,13 +43,13 @@ def _clean_tags_file(f_orig): f_txt = f_orig.replace('.tags', '') with codecs.open(f_txt, mode='w', encoding='utf-8') as fd_txt, \ io.open(f_orig, mode='r', encoding='utf-8') as fd_orig: - for l in fd_orig: - if not any(tag in l for tag in xml_tags): + for line in fd_orig: + if not any(tag in line for tag in xml_tags): # TODO: Fix utf-8 next line mark # fd_txt.write(l.strip() + '\n') # fd_txt.write(l.strip() + u"\u0085") # fd_txt.write(l.lstrip()) - fd_txt.write(l.strip() + '\n') + fd_txt.write(line.strip() + '\n') def _construct_filenames(filename, languages):