diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94b5339c5cab89edd2e5a11d685cbbc7eea6feda..1052ddfb48e16559a78d3c00fff8f7006238bc05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,7 @@ pytest: only: [merge_requests] <<: *before_script script: + - export LANG=C # Hopefully detect anything relying on locale - python3 -m pip install ".[test]" - python3 -m pytest --color yes --verbose --cov=cal_tools --cov=xfel_calibrate # Nope... https://docs.gitlab.com/12.10/ee/user/project/merge_requests/test_coverage_visualization.html#enabling-the-feature diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10959e513ce6bd8b9799fcc8f21eff646624958d..bb618c33c4a2ca9e2621ce182a302f27d0158b1a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: hooks: - id: identity - repo: https://github.com/nbQA-dev/nbQA - rev: 0.5.9 + rev: 0.13.0 hooks: - id: nbqa-isort additional_dependencies: [isort==5.7.0] diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh index 4cfa4484a88c8fd36919325616df74458ef51856..d1baf806b1bd531e47366892118e5226037b496a 100755 --- a/bin/slurm_calibrate.sh +++ b/bin/slurm_calibrate.sh @@ -33,6 +33,9 @@ module load texlive/2019 # make sure we use agg backend export MPLBACKEND=AGG +# Ensure Python uses UTF-8 for files by default +export LANG=en_US.UTF-8 + # start an ip cluster if requested if [ "${ipcluster_profile}" != "NO_CLUSTER" ] then diff --git a/notebooks/test/test-cli.ipynb b/notebooks/test/test-cli.ipynb index 2874fa87593b5d841e964d74920156a1896e1758..4367676b193dee076662a4142870f031d3988d83 100644 --- a/notebooks/test/test-cli.ipynb +++ b/notebooks/test/test-cli.ipynb @@ -56,6 +56,22 @@ "\n", "(in_folder / \"touch\").touch()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Include some non-ascii characters to check that files are reliably processed as UTF-8. 🤖" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"🥼\")" + ] } ], "metadata": { diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py index 4bc573bea5ac2807dcdfa5cb91a3ab94b558cfcf..17e4aa0ca81776f63bf7ef53cb8ae0e46c689b24 100755 --- a/src/xfel_calibrate/calibrate.py +++ b/src/xfel_calibrate/calibrate.py @@ -3,6 +3,7 @@ import argparse import ast import inspect +import locale import math import os import pprint @@ -859,6 +860,9 @@ def make_pipeline_yaml(parms, version, report_path, output_dir): def run(): """ Run a calibration task with parser arguments """ + # Ensure files are opened as UTF-8 by default, regardless of environment. + locale.setlocale(locale.LC_CTYPE, ('en_US', 'UTF-8')) + parser = make_extended_parser() args = deconsolize_args(vars(parser.parse_args())) detector = args["detector"].upper() diff --git a/webservice/webservice.py b/webservice/webservice.py index 76cd3d2a92c9f0456c3cc214d05bcf31519a6762..2a2c7a0621f9464f1860ef3ab4784cfba5f46eb8 100644 --- a/webservice/webservice.py +++ b/webservice/webservice.py @@ -6,6 +6,7 @@ import getpass import glob import inspect import json +import locale import logging import os import sqlite3 @@ -1097,6 +1098,9 @@ def main(argv: Optional[List[str]] = None): if argv is None: argv = sys.argv[1:] + # Ensure files are opened as UTF-8 by default, regardless of environment. + locale.setlocale(locale.LC_CTYPE, ('en_US', 'UTF-8')) + parser = argparse.ArgumentParser( description='Start the calibration webservice' )