diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..c25bdcf --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ + +# Docs: https://coverage.readthedocs.org/en/latest/config.html + +[run] +branch = False + +# If True, stores relative file paths in data file (needed for Github Actions). +# Using this parameter requires coverage>=5.0 +relative_files = True diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..1f3c598 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,77 @@ +name: Tests + +on: + push: + branches: [ master ] + paths-ignore: + - '.gitignore' + - '*.md' + - '*.rst' + - 'LICENSE' + - 'requirements.txt' + - 'vdf2json/**' + pull_request: + branches: [ master ] + paths-ignore: + - '.gitignore' + - '*.md' + - '*.rst' + - 'LICENSE' + - 'requirements.txt' + - 'vdf2json/**' + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10'] + no-coverage: [0] + include: + - os: ubuntu-latest + python-version: pypy-2.7 + no-coverage: 1 + - os: ubuntu-latest + python-version: pypy-3.6 + no-coverage: 1 + steps: + - uses: actions/checkout@v2 + - name: Set up Python Env + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Display Python version + run: python -c "import sys; print(sys.version)" + - name: Install dependencies + run: | + make init + - name: Run Tests + env: + NOCOV: ${{ matrix.no-coverage }} + run: | + make test + - name: Upload to Coveralls + # pypy + concurrenct=gevent not supported in coveragepy. See https://github.com/nedbat/coveragepy/issues/560 + if: matrix.no-coverage == 0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: "${{ matrix.os }}_${{ matrix.python-version }}" + run: | + coveralls --service=github + + coveralls: + name: Finish Coveralls + needs: test + runs-on: ubuntu-latest + container: python:3-slim + steps: + - name: Install coveralls + run: | + pip3 install --upgrade coveralls + - name: Send coverage finish to coveralls.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls --finish diff --git a/.scrutinizer.yml b/.scrutinizer.yml deleted file mode 100644 index 1f5aea0..0000000 --- a/.scrutinizer.yml +++ /dev/null @@ -1,17 +0,0 @@ -filter: - excluded_paths: - - 'tests/*' - - 'vdf2json/*' -tools: - external_code_coverage: - timeout: 200 - runs: 8 - -build: - nodes: - analysis: - tests: - override: - - command: py-scrutinizer-run - idle_timeout: 300 - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0e3b33b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,62 +0,0 @@ -language: python -os: linux - -jobs: - include: - - python: 2.7 - - python: 3.4 - - python: 3.5 - - python: 3.6 - - python: 3.7 - - python: 3.8 - - python: "pypy" - - python: "pypy3" -# OSX - - name: OSX Python 2.7 - os: osx - language: shell - before_install: - - cp -fv `which python2` `which python` || true - - cp -fv `which pip2` `which pip` || true - - pip install --upgrade pip - - name: OSX Python 3.7 - os: osx - language: shell - before_install: - - cp -fv `which python3` `which python` || true - - cp -fv `which pip3` `which pip` || true - - pip install --upgrade pip -# Windows - - name: Win Python 3.6 - language: shell - os: windows - before_install: - - choco install python --version 3.6.8 - - python -m pip install --upgrade pip - env: PATH=/c/Python36:/c/Python36/Scripts:$PATH - - name: Win Python 3.7 - language: shell - os: windows - before_install: - - choco install python --version 3.7.4 - - python -m pip install --upgrade pip - env: PATH=/c/Python37:/c/Python37/Scripts:$PATH - - name: Win Python 3.8 - language: shell - os: windows - before_install: - - choco install python --version 3.8.2 - - python -m pip install --upgrade pip - env: PATH=/c/Python38:/c/Python38/Scripts:$PATH - -install: - - pip install mock pytest==3.2.1 pytest-cov==2.5.1 - - pip install codecov - - pip install coveralls - - pip install scrutinizer-ocular -script: - - PYTHONHASHSEED=0 pytest --cov=vdf tests -after_success: - - codecov - - coveralls - - ocular --data-file ".coverage" diff --git a/Makefile b/Makefile index 458427e..914f5ac 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,17 @@ help: @echo "$$HELPBODY" init: - pip install -r requirements.txt + pip install -r dev_requirements.txt + +COVOPTS = --cov-config .coveragerc --cov=vdf + +ifeq ($(NOCOV), 1) + COVOPTS = +endif test: rm -f .coverage vdf/*.pyc tests/*.pyc - PYTHONHASHSEED=0 python -m pytest --cov=vdf tests + PYTHONHASHSEED=0 pytest --tb=short $(COVOPTS) tests pylint: pylint -r n -f colorized vdf || true diff --git a/README.rst b/README.rst index bba7455..c80704e 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,9 @@ -|pypi| |license| |coverage| |scru| |master_build| +| |pypi| |license| |coverage| |master_build| +| |sonar_maintainability| |sonar_reliability| |sonar_security| Pure python module for (de)serialization to and from VDF that works just like ``json``. -Tested and works on ``python2.7``, ``python3.3+``, ``pypy`` and ``pypy3``. +Tested and works on ``py2.7``, ``py3.3+``, ``pypy`` and ``pypy3``. VDF is Valve's KeyValue text file format @@ -35,8 +36,8 @@ Problems & solutions that can be used as mapper instead of ``dict``. See the example section for details. - By default de-serialization will return a ``dict``, which doesn't preserve nor guarantee - key order due to `hash randomization`_. If key order is important then - I suggest using ``collections.OrderedDict``, or ``vdf.VDFDict``. + key order on Python versions prior to 3.6, due to `hash randomization`_. If key order is + important on old Pythons, I suggest using ``collections.OrderedDict``, or ``vdf.VDFDict``. Example usage ------------- @@ -146,12 +147,20 @@ of reassign the value to the existing key. :target: https://coveralls.io/r/ValvePython/vdf?branch=master :alt: Test coverage -.. |scru| image:: https://scrutinizer-ci.com/g/ValvePython/vdf/badges/quality-score.png?b=master - :target: https://scrutinizer-ci.com/g/ValvePython/vdf/?branch=master - :alt: Scrutinizer score +.. |sonar_maintainability| image:: https://sonarcloud.io/api/project_badges/measure?project=ValvePython_vdf&metric=sqale_rating + :target: https://sonarcloud.io/dashboard?id=ValvePython_vdf + :alt: SonarCloud Rating -.. |master_build| image:: https://img.shields.io/travis/ValvePython/vdf/master.svg?style=flat&label=master%20build - :target: http://travis-ci.org/ValvePython/vdf +.. |sonar_reliability| image:: https://sonarcloud.io/api/project_badges/measure?project=ValvePython_vdf&metric=reliability_rating + :target: https://sonarcloud.io/dashboard?id=ValvePython_vdf + :alt: SonarCloud Rating + +.. |sonar_security| image:: https://sonarcloud.io/api/project_badges/measure?project=ValvePython_vdf&metric=security_rating + :target: https://sonarcloud.io/dashboard?id=ValvePython_vdf + :alt: SonarCloud Rating + +.. |master_build| image:: https://github.com/ValvePython/vdf/workflows/Tests/badge.svg?branch=master + :target: https://github.com/ValvePython/vdf/actions?query=workflow%3A%22Tests%22+branch%3Amaster :alt: Build status of master branch .. _DuplicateOrderedDict: https://github.com/rossengeorgiev/dota2_notebooks/blob/master/DuplicateOrderedDict_for_VDF.ipynb diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..ded0252 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,8 @@ +mock; python_version < '3.3' + +coverage>=5.0; python_version == '2.7' or python_version>= '3.5' +pytest-cov>=2.7.0; python_version == '2.7' or python_version>= '3.5' + +# coveralls 2.0 has removed support for Python 2.7 and 3.4 +git+https://github.com/andy-maier/coveralls-python.git@andy/add-py27#egg=coveralls; python_version == '2.7' +coveralls>=2.1.2; python_version>= '3.5' diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 27054f2..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -mock -pytest -pytest-cov diff --git a/setup.py b/setup.py index a45b13e..b40cf4b 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,12 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: PyPy', ], keywords='valve keyvalue vdf tf2 dota2 csgo', diff --git a/tests/test_vdf.py b/tests/test_vdf.py index f041578..00a245c 100644 --- a/tests/test_vdf.py +++ b/tests/test_vdf.py @@ -1,7 +1,11 @@ import unittest -import mock import sys +try: + from unittest import mock +except ImportError: + import mock + try: from StringIO import StringIO except ImportError: @@ -124,7 +128,7 @@ def test_parse_bom_removal(self): result = vdf.loads(vdf.BOMS + '"asd" "123"') self.assertEqual(result, {'asd': '123'}) - if sys.version_info[0] is 2: + if sys.version_info[0] == 2: result = vdf.loads(vdf.BOMS_UNICODE + '"asd" "123"') self.assertEqual(result, {'asd': '123'}) @@ -319,16 +323,27 @@ def test_hash_key(self): self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) def test_wierd_symbols_for_unquoted(self): - INPUT = 'a asd.vdf\nb language_*lol*\nc zxc_-*.sss//' + INPUT = 'a asd.vdf\nb language_*lol*\nc zxc_-*.sss//\nd<2?$% /cde/$fgh/' EXPECTED = { 'a': 'asd.vdf', 'b': 'language_*lol*', 'c': 'zxc_-*.sss', + 'd<2?$%': '/cde/$fgh/', } self.assertEqual(vdf.loads(INPUT), EXPECTED) self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) + def test_space_for_unquoted(self): + INPUT = 'a b c d \n efg h i\t // j k\n' + EXPECTED= { + 'a': 'b c d', + 'efg': 'h i', + } + + self.assertEqual(vdf.loads(INPUT), EXPECTED) + self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) + def test_merge_multiple_keys_on(self): INPUT = ''' a @@ -393,6 +408,101 @@ def test_escape_before_last_unescaped(self): self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) + def test_single_line_empty_block(self): + INPUT = ''' + "root1" + { + "key1" {} + key2 "value2" + "key3" value3 + } + root2 { } + root3 + { + "key1" "value1" + key2 { } + "key3" value3 + } + ''' + + EXPECTED = { + 'root1': { + 'key1': {}, + 'key2': 'value2', + 'key3': 'value3', + }, + 'root2': {}, + 'root3': { + 'key1': 'value1', + 'key2': {}, + 'key3': 'value3', + } + } + + self.assertEqual(vdf.loads(INPUT), EXPECTED) + self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) + + def test_inline_opening_bracker(self): + INPUT = ''' + "root1" { + "key1" { + } + key2 "value2" + "key3" value3 + } + root2 { } + root3 { + "key1" "value1" + key2 { + + } + "key3" value3 + } + ''' + + EXPECTED = { + 'root1': { + 'key1': {}, + 'key2': 'value2', + 'key3': 'value3', + }, + 'root2': {}, + 'root3': { + 'key1': 'value1', + 'key2': {}, + 'key3': 'value3', + } + } + + self.assertEqual(vdf.loads(INPUT), EXPECTED) + self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED) + + def test_duplicate_key_with_value_from_str_to_mapper(self): + INPUT = r''' + level1 + { + key1 text1 + key2 text2 + } + level1 + { + key2 + { + key3 text3 + } + } + ''' + + EXPECTED = { + "level1": { + "key1": "text1", + "key2": { + "key3": "text3" + } + } + } + + self.assertEqual(vdf.loads(INPUT), EXPECTED) class testcase_VDF_other(unittest.TestCase): def test_dumps_pretty_output(self): diff --git a/vdf/__init__.py b/vdf/__init__.py index 7bcd11c..6b47213 100644 --- a/vdf/__init__.py +++ b/vdf/__init__.py @@ -1,7 +1,7 @@ """ Module for deserializing/serializing to and from VDF """ -__version__ = "3.3" +__version__ = "3.4" __author__ = "Rossen Georgiev" import re @@ -86,10 +86,11 @@ def parse(fp, mapper=dict, merge_duplicate_keys=True, escaped=True): stack = [mapper()] expect_bracket = False - re_keyvalue = re.compile(r'^("(?P(?:\\.|[^\\"])+)"|(?P#?[a-z0-9\-\_\\\?]+))' + re_keyvalue = re.compile(r'^("(?P(?:\\.|[^\\"])*)"|(?P#?[a-z0-9\-\_\\\?$%]+))' r'([ \t]*(' r'"(?P(?:\\.|[^\\"])*)(?P")?' - r'|(?P[a-z0-9\-\_\\\?\*\.]+)' + r'|(?P(?:(? ])+)' + r'|(?P{[ \t]*)(?P})?' r'))?', flags=re.I) @@ -134,7 +135,13 @@ def parse(fp, mapper=dict, merge_duplicate_keys=True, escaped=True): (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line)) key = match.group('key') if match.group('qkey') is None else match.group('qkey') - val = match.group('val') if match.group('qval') is None else match.group('qval') + val = match.group('qval') + if val is None: + val = match.group('val') + if val is not None: + val = val.rstrip() + if val == "": + val = None if escaped: key = _unescape(key) @@ -143,12 +150,18 @@ def parse(fp, mapper=dict, merge_duplicate_keys=True, escaped=True): if val is None: if merge_duplicate_keys and key in stack[-1]: _m = stack[-1][key] + # we've descended a level deeper, if value is str, we have to overwrite it to mapper + if not isinstance(_m, mapper): + _m = stack[-1][key] = mapper() else: _m = mapper() stack[-1][key] = _m - stack.append(_m) - expect_bracket = True + if match.group('eblock') is None: + # only expect a bracket if it's not already closed or on the same line + stack.append(_m) + if match.group('sblock') is None: + expect_bracket = True # we've matched a simple keyvalue pair, map it to the last dict obj in the stack else: diff --git a/vdf/vdict.py b/vdf/vdict.py index ee71d65..e67afca 100644 --- a/vdf/vdict.py +++ b/vdf/vdict.py @@ -5,7 +5,7 @@ _iter_values = 'values' _range = range _string_type = str - import collections as _c + import collections.abc as _c class _kView(_c.KeysView): def __iter__(self): return self._mapping.iterkeys()

AltStyle によって変換されたページ (->オリジナル) /